convert/merge: add --strip-empty-lists

src/cyclonedx/CliUtils.cs

@@ -22,6 +22,7 @@
 using CycloneDX.Spdx.Interop;
 using CycloneDX.Cli.Commands;
 using CycloneDX.Cli.Serialization;
+using CycloneDX.Utils;
 
 namespace CycloneDX.Cli
 {
@@ -174,7 +175,7 @@ public static async Task<int> OutputBomHelper(Bom bom, CycloneDXBomFormat format
             return 0;
         }
 
-        public static async Task<int> OutputBomHelper(Bom bom, ConvertFormat format, SpecificationVersion? outputVersion, string filename)
+        public static async Task<int> OutputBomHelper(Bom bom, ConvertFormat format, SpecificationVersion? outputVersion, string filename, bool stripEmptyLists = false)
         {
             if (filename == null && format == ConvertFormat.autodetect)
             {
@@ -193,6 +194,16 @@ public static async Task<int> OutputBomHelper(Bom bom, ConvertFormat format, Spe
 
             bom.SpecVersion = outputVersion.HasValue ? outputVersion.Value : SpecificationVersionHelpers.CurrentVersion;
 
+            // Downgrading to an older spec version already collapses empty
+            // (non-null, zero-count) lists to null as a side effect of the
+            // protobuf deep-copy CopyBomAndDowngrade uses -- but the current
+            // spec version is serialized without going through that copy,
+            // so its empty lists survive unless pruned here explicitly.
+            if (stripEmptyLists)
+            {
+                CycloneDXUtils.CleanupEmptyListsDeep(bom);
+            }
+
             using var stream = filename == null ? Console.OpenStandardOutput() : File.Create(filename);
 
             switch (format)

src/cyclonedx/Commands/ConvertCommand.cs

@@ -35,6 +35,7 @@ internal static void Configure(RootCommand rootCommand)
             subCommand.Add(new Option<ConvertFormat>("--input-format", "Specify input file format."));
             subCommand.Add(new Option<ConvertFormat>("--output-format", "Specify output file format."));
             subCommand.Add(new Option<SpecificationVersion>("--output-version", "Specify output BOM specification version. (ignored for CSV and SPDX formats)"));
+            subCommand.Add(new Option<bool>("--strip-empty-lists", "Omit empty list properties (e.g. \"licenses\": [], \"dependsOn\": []) from the output instead of writing them out. Schema-valid either way; this just avoids redundant clutter."));
             subCommand.Handler = CommandHandler.Create<ConvertCommandOptions>(Convert);
             rootCommand.Add(subCommand);
         }
@@ -62,7 +63,7 @@ public static async Task<int> Convert(ConvertCommandOptions options)
                 }
             }
 
-            return await CliUtils.OutputBomHelper(inputBom, options.OutputFormat, options.OutputVersion, options.OutputFile).ConfigureAwait(false);
+            return await CliUtils.OutputBomHelper(inputBom, options.OutputFormat, options.OutputVersion, options.OutputFile, options.StripEmptyLists).ConfigureAwait(false);
         }
     }
 }

src/cyclonedx/Commands/ConvertCommandOptions.cs

@@ -24,5 +24,6 @@ internal class ConvertCommandOptions
         public ConvertFormat InputFormat { get; set; }
         public ConvertFormat OutputFormat { get; set; }
         public SpecificationVersion? OutputVersion { get; set; }
+        public bool StripEmptyLists { get; set; }
     }
 }

src/cyclonedx/Commands/MergeCommand.cs

@@ -19,6 +19,7 @@
 using System.Diagnostics.Contracts;
 using System.CommandLine;
 using System.CommandLine.Invocation;
+using System.IO;
 using System.Threading.Tasks;
 using CycloneDX.Models;
 using CycloneDX.Utils;
@@ -34,14 +35,24 @@ public static void Configure(RootCommand rootCommand)
             var subCommand = new System.CommandLine.Command("merge", "Merge two or more BOMs")
             {
                 new Option<List<string>>("--input-files", "Input BOM filenames (separate filenames with a space).") { AllowMultipleArgumentsPerToken = true },
+                new Option<List<string>>("--input-files-list", "One or more text file(s) with input BOM filenames (one per line). Combined with --input-files, useful to exceed OS/shell command-line length limits when merging many BOMs.") { AllowMultipleArgumentsPerToken = true },
+                new Option<List<string>>("--input-files-nul-list", "One or more text-like file(s) with input BOM filenames (separated by 0x00 characters, e.g. from `find -print0`).") { AllowMultipleArgumentsPerToken = true },
                 new Option<string>("--output-file", "Output BOM filename, will write to stdout if no value provided."),
                 new Option<CycloneDXBomFormat>("--input-format", "Specify input file format."),
                 new Option<CycloneDXBomFormat>("--output-format", "Specify output file format."),
                 new Option<SpecificationVersion>("--output-version", "Specify output BOM specification version."),
                 new Option<bool>("--hierarchical", "Perform a hierarchical merge."),
                 new Option<string>("--group", "Provide the group of software the merged BOM describes."),
                 new Option<string>("--name", "Provide the name of software the merged BOM describes (required for hierarchical merging)."),
-                new Option<string>("--version", "Provide the version of software the merged BOM describes (required for hierarchical merging).")
+                new Option<string>("--version", "Provide the version of software the merged BOM describes (required for hierarchical merging)."),
+                new Option<bool>("--validate-output", "Validate the merged document before writing it, and do not write it if validation fails."),
+                new Option<bool>("--validate-output-relaxed", "Validate the merged document, but still write it (for troubleshooting) even if validation fails."),
+                new Option<bool>("--strip-empty-lists", "Omit empty list properties (e.g. \"licenses\": [], \"dependsOn\": []) from the output instead of writing them out. Schema-valid either way; this just avoids redundant clutter."),
+#if NET8_0_OR_GREATER
+                new Option<ComponentConflictResolution>("--component-conflict-resolution", "How to resolve two equivalent (same type/name/version/group/purl) but not-identical Components, e.g. differing only by Scope. Default: squash, preferring the more permissive Scope."),
+                new Option<bool>("--attach-dangling-components", "Attach any components no dependsOn edge reaches (grouped by Scope into synthetic components) so consumers that walk the dependency graph from the subject, rather than scanning the flat components list, don't silently miss them."),
+                new Option<string>("--attach-dangling-components-ref", "Existing bom-ref to attach dangling components under (with --attach-dangling-components). Defaults to the merge subject if not given or not found."),
+#endif
             };
             subCommand.Handler = CommandHandler.Create<MergeCommandOptions>(Merge);
             rootCommand.Add(subCommand);
@@ -65,7 +76,7 @@ public static async Task<int> Merge(MergeCommandOptions options)
                 return (int)ExitCode.ParameterValidationError;
             }
 
-            var inputBoms = await InputBoms(options.InputFiles, options.InputFormat, outputToConsole).ConfigureAwait(false);
+            var inputBoms = await InputBoms(DetermineInputFiles(options), options.InputFormat, outputToConsole).ConfigureAwait(false);
 
             Component bomSubject = null;
             if (options.Group != null || options.Name != null || options.Version != null)
@@ -77,23 +88,37 @@ public static async Task<int> Merge(MergeCommandOptions options)
                     Version = options.Version,
                 };
 
+#if NET8_0_OR_GREATER
+            var mergeStrategy = MergeStrategy.Default();
+            if (options.ComponentConflictResolution.HasValue)
+            {
+                mergeStrategy.ComponentConflictResolution = options.ComponentConflictResolution.Value;
+            }
+#endif
+
             Bom outputBom;
             if (options.Hierarchical)
             {
+#if NET8_0_OR_GREATER
+                outputBom = CycloneDXUtils.HierarchicalMerge(inputBoms, bomSubject, mergeStrategy);
+#else
                 outputBom = CycloneDXUtils.HierarchicalMerge(inputBoms, bomSubject);
+#endif
             }
             else
             {
-                outputBom = CycloneDXUtils.FlatMerge(inputBoms);
+#if NET8_0_OR_GREATER
+                outputBom = CycloneDXUtils.FlatMerge(inputBoms, bomSubject, mergeStrategy);
+#else
+                outputBom = CycloneDXUtils.FlatMerge(inputBoms, bomSubject);
+#endif
                 if (outputBom.Metadata is null) outputBom.Metadata = new Metadata();
-                if (bomSubject != null)
-                {
-                    // use the params provided if possible
-                    outputBom.Metadata.Component = bomSubject;
-                }
-                else
+                if (bomSubject is null)
                 {
-                    // otherwise use the first non-null component from the input BOMs as the default
+                    // otherwise use the first non-null component from the input
+                    // BOMs as the default; note CleanupMetadataComponent below,
+                    // since that same component may also already be present
+                    // in outputBom.Components.
                     foreach (var bom in inputBoms)
                     {
                         if(bom.Metadata != null && bom.Metadata.Component != null)
@@ -105,6 +130,43 @@ public static async Task<int> Merge(MergeCommandOptions options)
                 }
             }
 
+#if NET8_0_OR_GREATER
+            outputBom = CycloneDXUtils.CleanupMetadataComponent(outputBom, mergeStrategy);
+            outputBom = CycloneDXUtils.CleanupEmptyLists(outputBom);
+
+            if (options.AttachDanglingComponents)
+            {
+                var attached = outputBom.AttachDanglingComponents(options.AttachDanglingComponentsRef);
+                if (attached.Count > 0)
+                {
+                    var totalAttached = 0;
+                    foreach (var bucket in attached.Values) totalAttached += bucket.Count;
+                    Console.WriteLine($"Attached {totalAttached} component(s) unreachable from the dependency graph, in {attached.Count} scope bucket(s):");
+                    foreach (var bucket in attached)
+                    {
+                        Console.WriteLine($"    {bucket.Key}: {bucket.Value.Count} component(s)");
+                    }
+                }
+            }
+#endif
+
+            // FlatMerge/HierarchicalMerge never set SpecVersion on their
+            // result, so it defaults to v1_0 unless assigned here. Apply the
+            // requested --output-version (or the library's current version,
+            // matching OutputBomHelper's own default) before the
+            // --validate-output check below, so validation reflects the
+            // spec version that will actually be written -- rather than
+            // OutputBomHelper silently overriding it afterwards, by which
+            // point validation has already run against the wrong target.
+            outputBom.SpecVersion = options.OutputVersion ?? SpecificationVersionHelpers.CurrentVersion;
+
+            // Ensure that the merged document has its own identity (new
+            // SerialNumber, Version=1, Timestamp...) and that its Tools
+            // collection records the library and program that produced it.
+#if NET8_0_OR_GREATER
+            outputBom.BomMetadataUpdate(true);
+            outputBom.BomMetadataReferThisToolkit();
+#else
             outputBom.Version = 1;
             outputBom.SerialNumber = "urn:uuid:" + System.Guid.NewGuid().ToString();
             if (outputBom.Metadata == null)
@@ -115,14 +177,112 @@ public static async Task<int> Merge(MergeCommandOptions options)
             {
                 outputBom.Metadata.Timestamp = DateTime.Now;
             }
+#endif
+
+            if (options.StripEmptyLists)
+            {
+                // Applied before validation so the validated document
+                // matches what OutputBomHelper actually writes below.
+                CycloneDXUtils.CleanupEmptyListsDeep(outputBom);
+            }
+
+            ValidationResult validationResult = null;
+            if (options.ValidateOutput || options.ValidateOutputRelaxed)
+            {
+                Console.WriteLine("Validating merged BOM...");
+                validationResult = Json.Validator.Validate(Json.Serializer.Serialize(outputBom), outputBom.SpecVersion);
+
+                if (validationResult.Messages != null)
+                {
+                    foreach (var message in validationResult.Messages)
+                    {
+                        Console.WriteLine(message);
+                    }
+                }
+
+                if (validationResult.Valid)
+                {
+                    Console.WriteLine("Merged BOM validated successfully.");
+                }
+                else
+                {
+                    Console.WriteLine("Merged BOM is not valid.");
+                    if (!options.ValidateOutputRelaxed)
+                    {
+                        Console.WriteLine("NOT writing output file...");
+                        return (int)ExitCode.SignatureFailedVerification;
+                    }
+                }
+            }
 
             if (!outputToConsole)
             {
                 Console.WriteLine("Writing output file...");
                 Console.WriteLine($"    Total {outputBom.Components?.Count ?? 0} components");
             }
 
-            return await CliUtils.OutputBomHelper(outputBom, (ConvertFormat)options.OutputFormat, options.OutputVersion, options.OutputFile).ConfigureAwait(false);
+            var res = await CliUtils.OutputBomHelper(outputBom, (ConvertFormat)options.OutputFormat, options.OutputVersion, options.OutputFile, options.StripEmptyLists).ConfigureAwait(false);
+            if (validationResult != null && !validationResult.Valid)
+            {
+                // Relaxed mode: the file was still written above, but the
+                // command as a whole should still report failure.
+                return (int)ExitCode.SignatureFailedVerification;
+            }
+            return res;
+        }
+
+        /// <summary>
+        /// Combines --input-files with any filenames listed inside
+        /// --input-files-list (one per line) and --input-files-nul-list
+        /// (0x00-separated) files, deduplicating as it goes. Lets callers
+        /// exceed OS/shell command-line length or argument-count limits
+        /// when merging many BOMs, by passing a generated list file
+        /// instead of one --input-files argument per BOM.
+        /// </summary>
+        private static List<string> DetermineInputFiles(MergeCommandOptions options)
+        {
+            var inputFiles = options.InputFiles != null ? new List<string>(options.InputFiles) : new List<string>();
+
+            if (options.InputFilesList != null)
+            {
+                foreach (var oneList in options.InputFilesList)
+                {
+                    Console.WriteLine($"Adding to input file list from {oneList}");
+                    var count = 0;
+                    foreach (var line in File.ReadAllLines(oneList))
+                    {
+                        if (string.IsNullOrEmpty(line) || inputFiles.Contains(line))
+                        {
+                            continue;
+                        }
+                        inputFiles.Add(line);
+                        count++;
+                    }
+                    Console.WriteLine($"Got {count} new entries from {oneList}");
+                }
+            }
+
+            if (options.InputFilesNulList != null)
+            {
+                foreach (var oneList in options.InputFilesNulList)
+                {
+                    Console.WriteLine($"Adding to input file list from {oneList}");
+                    var count = 0;
+                    foreach (var line in File.ReadAllText(oneList).Split('\0'))
+                    {
+                        if (string.IsNullOrEmpty(line) || inputFiles.Contains(line))
+                        {
+                            continue;
+                        }
+                        inputFiles.Add(line);
+                        count++;
+                    }
+                    Console.WriteLine($"Got {count} new entries from {oneList}");
+                }
+            }
+
+            Console.WriteLine($"Determined {inputFiles.Count} input file(s) to merge");
+            return inputFiles;
         }
 
         private static async Task<IEnumerable<Bom>> InputBoms(IEnumerable<string> inputFilenames, CycloneDXBomFormat inputFormat, bool outputToConsole)

src/cyclonedx/Commands/MergeCommandOptions.cs

@@ -15,12 +15,15 @@
 // SPDX-License-Identifier: Apache-2.0
 // Copyright (c) OWASP Foundation. All Rights Reserved.
 using System.Collections.Generic;
+using CycloneDX.Models;
 
 namespace CycloneDX.Cli.Commands
 {
     internal class MergeCommandOptions
     {
         public IList<string> InputFiles { get; set; }
+        public IList<string> InputFilesList { get; set; }
+        public IList<string> InputFilesNulList { get; set; }
         public string OutputFile { get; set; }
         public CycloneDXBomFormat InputFormat { get; set; }
         public CycloneDXBomFormat OutputFormat { get; set; }
@@ -29,5 +32,13 @@ internal class MergeCommandOptions
         public string Group { get; set; }
         public string Name { get; set; }
         public string Version { get; set; }
+        public bool ValidateOutput { get; set; }
+        public bool ValidateOutputRelaxed { get; set; }
+        public bool StripEmptyLists { get; set; }
+#if NET8_0_OR_GREATER
+        public ComponentConflictResolution? ComponentConflictResolution { get; set; }
+        public bool AttachDanglingComponents { get; set; }
+        public string AttachDanglingComponentsRef { get; set; }
+#endif
     }
 }