Fix#9041 YAML input list-of-maps reading and null-path failures

plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInput.java

@@ -204,28 +204,24 @@ private void addFileToResultFilesName(FileObject file) {
 
   private boolean openNextFile() {
     try {
-      if (data.fileIndex >= data.files.nrOfFiles()) {
-        // finished processing!
-        if (isDetailed()) {
-          logDetailed(BaseMessages.getString(PKG, "YamlInput.Log.FinishedProcessing"));
-        }
-        return false;
-      }
-      // Get file to process from list
-      data.file = data.files.getFile(data.fileIndex);
-
-      // Move file pointer ahead!
-      data.fileIndex++;
-
-      if (meta.isIgnoringEmptyFile() && data.file.getContent().getSize() == 0) {
-        if (isBasic()) {
-          logBasic(
-              BaseMessages.getString(PKG, "YamlInput.Error.FileSizeZero", data.file.getName()));
+      while (data.fileIndex < data.files.nrOfFiles()) {
+        // Get file to process from list
+        data.file = data.files.getFile(data.fileIndex);
+
+        // Move file pointer ahead!
+        data.fileIndex++;
+
+        if (meta.isIgnoringEmptyFile() && data.file.getContent().getSize() == 0) {
+          // Do not keep a reader from a previous file, and do not report success: there is
+          // nothing to read until a non-empty file is opened.
+          data.yaml = null;
+          if (isBasic()) {
+            logBasic(
+                BaseMessages.getString(PKG, "YamlInput.Error.FileSizeZero", data.file.getName()));
+          }
+          continue;
         }
-        // Let's open the next file
-        openNextFile();
 
-      } else {
         if (isDetailed()) {
           logDetailed(
               BaseMessages.getString(PKG, "YamlInput.Log.OpeningFile", data.file.toString()));
@@ -246,7 +242,14 @@ private boolean openNextFile() {
           logDetailed(
               BaseMessages.getString(PKG, "YamlInput.Log.FileOpened", data.file.toString()));
         }
+        return true;
       }
+
+      // finished processing!
+      if (isDetailed()) {
+        logDetailed(BaseMessages.getString(PKG, "YamlInput.Log.FinishedProcessing"));
+      }
+      return false;
     } catch (Exception e) {
       logError(
           BaseMessages.getString(
@@ -260,7 +263,6 @@ private boolean openNextFile() {
       logError(Const.getStackTracker(e));
       return false;
     }
-    return true;
   }
 
   @Override
@@ -321,11 +323,11 @@ private Object[] getOneRow() throws HopException {
 
   private Object[] getOneRowFromFileMode() throws HopException {
     while (true) {
-      if (data.file == null && !openNextFile()) {
+      if ((data.file == null || data.yaml == null) && !openNextFile()) {
         return new Object[0];
       }
 
-      if (data.file == null) {
+      if (data.file == null || data.yaml == null) {
         continue;
       }
 

plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReader.java

@@ -86,6 +86,12 @@ public class YamlReader {
 
   private final Deque<Object[]> pendingRows = new ArrayDeque<>();
 
+  /**
+   * Rows queued from one element of a list document. Draining them must not finish the document, or
+   * the remaining elements are skipped.
+   */
+  private boolean pendingRowsFromListElement;
+
   /** Bytes read from the last loaded file (for data volume tracking). */
   @Getter private long bytesReadFromFile;
 
@@ -148,11 +154,7 @@ public boolean isMapUsed() {
   /** get row */
   public Object[] getRow(IRowMeta rowMeta) {
     if (!pendingRows.isEmpty()) {
-      Object[] row = pendingRows.poll();
-      if (pendingRows.isEmpty()) {
-        finishDocument();
-      }
-      return row;
+      return pollPendingRow();
     }
 
     if (document == null) {
@@ -164,7 +166,9 @@ public Object[] getRow(IRowMeta rowMeta) {
     }
 
     Object[] row = fetchRow(rowMeta);
-    if (row.length == 0 && !isFinishedDocument()) {
+    // An empty row finishes the current list document. Keep going when another document
+    // remains: YamlInput treats an empty row as the end of the file.
+    if (row.length == 0 && (!isFinishedDocument() || documentIt.hasNext())) {
       return getRow(rowMeta);
     }
     return row;
@@ -224,13 +228,27 @@ private Object[] processListRow(IRowMeta rowMeta) {
   private Object[] processMapRow(IRowMeta rowMeta, Object root, boolean listElement) {
     if (pendingRows.isEmpty()) {
       prepareRowsFromPaths(rowMeta, root, listElement);
+      pendingRowsFromListElement = listElement;
     }
 
+    Object[] row = pollPendingRow();
+    return row == null ? new Object[0] : row;
+  }
+
+  /**
+   * Returns the next prepared row. A map document is finished when its rows are consumed. A list
+   * element is not: the list iterator still has to walk the remaining elements.
+   */
+  private Object[] pollPendingRow() {
     Object[] row = pendingRows.poll();
     if (pendingRows.isEmpty()) {
-      finishDocument();
+      if (pendingRowsFromListElement) {
+        pendingRowsFromListElement = false;
+      } else {
+        finishDocument();
+      }
     }
-    return row == null ? new Object[0] : row;
+    return row;
   }
 
   private void prepareRowsFromPaths(IRowMeta rowMeta, Object root, boolean listElement) {
@@ -283,32 +301,39 @@ private static Object pickValue(List<Object> values, int rowIndex, int rowCount)
 
   private List<Object> readPathAsList(Object root, JsonPath path) {
     if (root == null) {
-      return List.of((Object) null);
+      return listWithNull();
     }
 
     try {
       ReadContext context = JsonPath.using(JSON_PATH_CONFIG).parse(root);
       Object result = context.read(path);
       return toValueList(result);
     } catch (Exception e) {
-      return List.of((Object) null);
+      return listWithNull();
     }
   }
 
   private static List<Object> toValueList(Object result) {
     if (result == null) {
-      return List.of((Object) null);
+      return listWithNull();
     }
 
     if (result instanceof List<?> list) {
       if (list.isEmpty()) {
-        return List.of((Object) null);
+        return listWithNull();
       }
       return new ArrayList<>(list);
     }
     return List.of(result);
   }
 
+  /** {@link List#of} rejects null elements. A missing path is a null cell, not an error. */
+  private static List<Object> listWithNull() {
+    List<Object> values = new ArrayList<>(1);
+    values.add(null);
+    return values;
+  }
+
   private Object[] advanceListIterator() {
     if (!dataListIt.hasNext()) {
       finishDocument();

plugins/transforms/yamlinput/src/main/resources/org/apache/hop/pipeline/transforms/yamlinput/messages/messages_en_US.properties

@@ -37,7 +37,7 @@ YamlInput.Log.UnableToOpenFile=Could not open file \#{0} \: {1} --> {2}
 YamlInput.Log.UnexpectedError=Unexpected Error \: {0}
 YamlInput.Log.YAMLStream=The field [{0}] contain [{1}]
 YamlInput.Name=YAML input
-YamlInputDialog.AddResult.Label=Add files to result filesname
+YamlInputDialog.AddResult.Label=Add files to result filenames
 YamlInputDialog.AddResult.Tooltip=Add files to result filenames
 YamlInputDialog.Button.PreviewRows=\  &Preview rows
 YamlInputDialog.Content.Tab=Content
@@ -80,12 +80,12 @@ YamlInputDialog.FilesReadSelection.DialogMessage=Files read\:
 YamlInputDialog.FilesReadSelection.DialogTitle=Files read
 YamlInputDialog.GetFields.Button=\ &Get fields
 YamlInputDialog.IgnoreEmptyFile.Label=Ignore empty file
-YamlInputDialog.IgnoreEmptyFile.Tooltip=Check this option if you want to ignore (skip) empty files.Otherwise, Apache Hop will fail making the process stop\!
+YamlInputDialog.IgnoreEmptyFile.Tooltip=Skip empty files. Without this option an empty file produces no rows and does not stop the pipeline.
 YamlInputDialog.InclFilename.Label=Include filename in output
 YamlInputDialog.InclFilename.Tooltip=Check this to add a field (String) containing the filename.
 YamlInputDialog.InclFilenameField.Label=Filename fieldname
 YamlInputDialog.InclRownum.Label=Rownum in output
-YamlInputDialog.InclRownum.Tooltip=Check this to add a field (String) containing the row number.
+YamlInputDialog.InclRownum.Tooltip=Check this to add a field (Integer) containing the row number.
 YamlInputDialog.InclRownumField.Label=Rownum fieldname
 YamlInputDialog.IncludeSubDirs.Column=Include subfolders
 YamlInputDialog.IncludeSubDirs.Tooltip=Check this option if you want to fetch subfolders
@@ -96,7 +96,7 @@ YamlInputDialog.NumberRows.DialogMessage=Enter the number of rows you would like
 YamlInputDialog.NumberRows.DialogTitle=Enter preview size
 YamlInputDialog.RegExp.Label=Regular Expression
 YamlInputDialog.Required.Column=Required
-YamlInputDialog.Required.Tooltip=Is this file required? \nOnly used for files without wildcards.
+YamlInputDialog.Required.Tooltip=Is this file or folder required?\nA required file must exist. A required folder must exist and match at least one file.
 YamlInputDialog.ShowFiles.Button=&Show filename(s)...
 YamlInputDialog.wAddFileResult.Label=Add to result filename
 YamlInputDialog.wAdditionalFields.Label=Additional fields

plugins/transforms/yamlinput/src/main/resources/org/apache/hop/pipeline/transforms/yamlinput/messages/messages_zh_CN.properties

@@ -81,12 +81,12 @@ YamlInputDialog.FilesReadSelection.DialogMessage=\u8BFB\u6587\u4EF6\:
 YamlInputDialog.FilesReadSelection.DialogTitle=\u8BFB\u53D6\u6587\u4EF6
 YamlInputDialog.GetFields.Button=\u83B7\u53D6\u5B57\u6BB5(&G)
 YamlInputDialog.IgnoreEmptyFile.Label=\u5FFD\u7565\u7A7A\u6587\u4EF6\:
-YamlInputDialog.IgnoreEmptyFile.Tooltip=\u9009\u4E2D\u8BE5\u9009\u9879\u5C06\u5FFD\u7565(\u8DF3\u8FC7)\u7A7A\u6587\u4EF6, \u5426\u5219\u7EC8\u6B62\u5904\u7406\u8FDB\u7A0B!
+YamlInputDialog.IgnoreEmptyFile.Tooltip=\u9009\u4E2D\u8BE5\u9009\u9879\u5C06\u5FFD\u7565(\u8DF3\u8FC7)\u7A7A\u6587\u4EF6\u3002\u4E0D\u9009\u4E2D\u65F6\uFF0C\u7A7A\u6587\u4EF6\u4E0D\u4F1A\u4EA7\u751F\u6570\u636E\u884C\uFF0C\u4E5F\u4E0D\u4F1A\u7EC8\u6B62\u5904\u7406\u3002
 YamlInputDialog.InclFilename.Label=\u8F93\u51FA\u6587\u4EF6\u540D:
 YamlInputDialog.InclFilename.Tooltip=\u6DFB\u52A0\u6587\u4EF6\u540D\u5230\u8F93\u51FA\u5B57\u6BB5\u5217\u8868\u4E2D
 YamlInputDialog.InclFilenameField.Label=\u5B57\u6BB5\u540D\u79F0\:
 YamlInputDialog.InclRownum.Label=\u8F93\u51FA\u884C\u53F7:
-YamlInputDialog.InclRownum.Tooltip=\u6DFB\u52A0\u884C\u53F7\u5230\u8F93\u51FA\u5B57\u6BB5\u5217\u8868\u4E2D
+YamlInputDialog.InclRownum.Tooltip=\u6DFB\u52A0\u884C\u53F7\uFF08\u6574\u6570\uFF09\u5230\u8F93\u51FA\u5B57\u6BB5\u5217\u8868\u4E2D
 YamlInputDialog.InclRownumField.Label=\u5B57\u6BB5\u540D\u79F0:
 YamlInputDialog.IncludeSubDirs.Column=\u5305\u542B\u5B50\u76EE\u5F55
 YamlInputDialog.IncludeSubDirs.Tooltip=\u904D\u5386\u5B50\u76EE\u5F55
@@ -97,7 +97,7 @@ YamlInputDialog.NumberRows.DialogMessage=\u8F93\u5165\u60A8\u60F3\u9884\u89C8\u7
 YamlInputDialog.NumberRows.DialogTitle=\u9884\u89C8\u884C\u6570
 YamlInputDialog.RegExp.Label=\u6B63\u5219\u8868\u8FBE\u5F0F:
 YamlInputDialog.Required.Column=\u5FC5\u9700
-YamlInputDialog.Required.Tooltip=\u6307\u5B9A\u6587\u4EF6\u5FC5\u9700\u5B58\u5728(\u4F7F\u7528\u901A\u914D\u7B26\u65F6, \u8BE5\u9009\u9879\u65E0\u6548)
+YamlInputDialog.Required.Tooltip=\u8BE5\u6587\u4EF6\u6216\u76EE\u5F55\u662F\u5426\u5FC5\u9700\uFF1F\n\u5FC5\u9700\u7684\u6587\u4EF6\u5FC5\u987B\u5B58\u5728\u3002\u5FC5\u9700\u7684\u76EE\u5F55\u5FC5\u987B\u5B58\u5728\uFF0C\u5E76\u4E14\u81F3\u5C11\u5339\u914D\u4E00\u4E2A\u6587\u4EF6\u3002
 YamlInputDialog.ShowFiles.Button=\u663E\u793A\u6587\u4EF6\u540D(&S)
 YamlInputDialog.wAddFileResult.Label=Transform \u8F93\u51FA\u6587\u4EF6
 YamlInputDialog.wAdditionalFields.Label=\u9644\u52A0\u5B57\u6BB5