From 168bde00b6ae26a655b01b42c5bb6fb5ff6c576b Mon Sep 17 00:00:00 2001 From: Adam Plaskitt Date: Sun, 23 Nov 2025 18:06:29 +0000 Subject: [PATCH 1/5] Ref: Enable SA1101 as warning SA1101 - Prefix local calls with `this` --- .editorconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/.editorconfig b/.editorconfig index 66d093589..ab7abd364 100644 --- a/.editorconfig +++ b/.editorconfig @@ -464,6 +464,7 @@ dotnet_diagnostic.SA1623.severity = none ########################################## dotnet_diagnostic.SA1009.severity = none +dotnet_diagnostic.SA1101.severity = warning dotnet_diagnostic.SA1111.severity = none # https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1600.md From c5344c60c74dddbabf8d98dd6c3b2e7faadbebc8 Mon Sep 17 00:00:00 2001 From: Adam Plaskitt Date: Sun, 23 Nov 2025 18:15:14 +0000 Subject: [PATCH 2/5] Ref: Format StyleCop.Analyzers into a group --- .editorconfig | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.editorconfig b/.editorconfig index ab7abd364..aa0d438c9 100644 --- a/.editorconfig +++ b/.editorconfig @@ -463,10 +463,26 @@ dotnet_diagnostic.SA1623.severity = none # https://github.com/DotNetAnalyzers/StyleCopAnalyzers ########################################## +# https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1009.md +# Closing parenthesis should be spaced correctly dotnet_diagnostic.SA1009.severity = none + +# https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1101.md +# Local calls should be prefixed with `this` dotnet_diagnostic.SA1101.severity = warning + +# https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1111.md +# Closing parenthesis should be on the line of last parameter dotnet_diagnostic.SA1111.severity = none +# https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1201.md +# Elements should be in the correct order +dotnet_diagnostic.SA1201.severity = warning + +# https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1202.md +# Elements should be ordered by access +dotnet_diagnostic.SA1202.severity = none + # https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1600.md # Elements should be documented dotnet_diagnostic.SA1600.severity = suggestion @@ -613,9 +629,6 @@ dotnet_diagnostic.CA1837.severity = suggestion # CA1024: Use properties where appropriate dotnet_diagnostic.CA1024.severity = suggestion -# SA1202: Elements should be ordered by access -dotnet_diagnostic.SA1202.severity = suggestion - # IDE0022: Use expression body for methods dotnet_diagnostic.IDE0022.severity = suggestion From 4204e694cbf41017b6c51c4a330023f02dde90c2 Mon Sep 17 00:00:00 2001 From: Adam Plaskitt Date: Sun, 23 Nov 2025 19:07:39 +0000 Subject: [PATCH 3/5] Ref: Address SA1202 warning SA1202 - Elements should be ordered by access --- .editorconfig | 2 +- .../DockerService.cs | 48 +-- .../PathUtilityService.cs | 4 +- .../Utilities/StringUtilities.cs | 2 +- .../FileComponentDetector.cs | 120 +++---- .../TypedComponent/CargoComponent.cs | 10 +- .../TypedComponent/ConanComponent.cs | 10 +- .../TypedComponent/DockerImageComponent.cs | 10 +- .../TypedComponent/DotNetComponent.cs | 10 +- .../TypedComponent/GoComponent.cs | 4 +- .../TypedComponent/LinuxComponent.cs | 10 +- .../TypedComponent/NpmComponent.cs | 10 +- .../TypedComponent/NugetComponent.cs | 10 +- .../TypedComponent/OtherComponent.cs | 10 +- .../TypedComponent/PipComponent.cs | 10 +- .../TypedComponent/PodComponent.cs | 10 +- .../TypedComponent/RubyGemsComponent.cs | 10 +- .../TypedComponent/SpdxComponent.cs | 10 +- .../TypedComponent/VcpkgComponent.cs | 10 +- .../dotnet/DotNetComponentDetector.cs | 132 ++++---- .../go/Parsers/GoModParser.cs | 90 ++--- .../linux/LinuxContainerDetector.cs | 18 +- .../maven/MavenCommandService.cs | 2 +- .../maven/MvnCliComponentDetector.cs | 10 +- .../npm/NpmComponentUtilities.cs | 6 +- .../npm/NpmLockfileDetectorBase.cs | 4 +- .../FrameworkPackages/FrameworkPackages.cs | 56 ++-- .../FrameworkPackages.net9.0.cs | 4 +- .../nuget/NuGetComponentDetector.cs | 4 +- ...ectModelProjectCentricComponentDetector.cs | 80 ++--- .../Contracts/PipDependencySpecification.cs | 116 +++---- .../pip/IPyPiClient.cs | 14 +- .../pip/PipCommandService.cs | 88 ++--- .../pip/PipReportUtilities.cs | 8 +- .../pip/PythonCommandService.cs | 32 +- .../pip/PythonResolver.cs | 52 +-- .../pip/PythonResolverBase.cs | 52 +-- .../pip/PythonVersionUtilities.cs | 34 +- .../pip/SimplePypiClient.cs | 26 +- .../pip/SimplePythonResolver.cs | 166 ++++----- .../rust/Parsers/RustCargoLockParser.cs | 4 +- .../rust/Parsers/RustSbomParser.cs | 74 ++--- .../Experiments/DetectorExperiments.cs | 4 +- .../Experiments/ExperimentService.cs | 26 +- .../LoggingEnricher.cs | 6 +- .../DotNetComponentDetectorTests.cs | 314 +++++++++--------- .../GoComponentDetectorTests.cs | 50 +-- .../PipCommandServiceTests.cs | 46 +-- .../PipDependencySpecifierTests.cs | 106 +++--- .../RustCargoLockParserTests.cs | 66 ++-- .../RustCliParserTests.cs | 90 ++--- .../RustMetadataContextBuilderTests.cs | 118 +++---- .../RustSbomParserTests.cs | 114 +++---- .../SimplePypiClientTests.cs | 46 +-- .../ComponentRecorderTestUtilities.cs | 48 +-- .../Experiments/ExperimentServiceTests.cs | 14 +- .../DetectorProcessingServiceTests.cs | 18 +- 57 files changed, 1225 insertions(+), 1223 deletions(-) diff --git a/.editorconfig b/.editorconfig index aa0d438c9..557178076 100644 --- a/.editorconfig +++ b/.editorconfig @@ -481,7 +481,7 @@ dotnet_diagnostic.SA1201.severity = warning # https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1202.md # Elements should be ordered by access -dotnet_diagnostic.SA1202.severity = none +dotnet_diagnostic.SA1202.severity = warning # https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1600.md # Elements should be documented diff --git a/src/Microsoft.ComponentDetection.Common/DockerService.cs b/src/Microsoft.ComponentDetection.Common/DockerService.cs index d9b47e05b..a39d258b3 100644 --- a/src/Microsoft.ComponentDetection.Common/DockerService.cs +++ b/src/Microsoft.ComponentDetection.Common/DockerService.cs @@ -83,13 +83,6 @@ public async Task ImageExistsLocallyAsync(string image, CancellationToken } } - private async Task InspectImageAndSanitizeVarsAsync(string image, CancellationToken cancellationToken = default) - { - var imageInspectResponse = await Client.Images.InspectImageAsync(image, cancellationToken); - this.SanitizeEnvironmentVariables(imageInspectResponse); - return imageInspectResponse; - } - public async Task TryPullImageAsync(string image, CancellationToken cancellationToken = default) { using var record = new DockerServiceTryPullImageTelemetryRecord @@ -118,23 +111,6 @@ public async Task TryPullImageAsync(string image, CancellationToken cancel } } - internal void SanitizeEnvironmentVariables(ImageInspectResponse inspectResponse) - { - var envVariables = inspectResponse?.Config?.Env; - if (envVariables == null || !envVariables.Any()) - { - return; - } - - var sanitizedVarList = new List(); - foreach (var variable in inspectResponse.Config.Env) - { - sanitizedVarList.Add(variable.RemoveSensitiveInformation()); - } - - inspectResponse.Config.Env = sanitizedVarList; - } - public async Task InspectImageAsync(string image, CancellationToken cancellationToken = default) { using var record = new DockerServiceInspectImageTelemetryRecord @@ -201,6 +177,23 @@ public async Task InspectImageAsync(string image, Cancellation return (stdout, stderr); } + internal void SanitizeEnvironmentVariables(ImageInspectResponse inspectResponse) + { + var envVariables = inspectResponse?.Config?.Env; + if (envVariables == null || !envVariables.Any()) + { + return; + } + + var sanitizedVarList = new List(); + foreach (var variable in inspectResponse.Config.Env) + { + sanitizedVarList.Add(variable.RemoveSensitiveInformation()); + } + + inspectResponse.Config.Env = sanitizedVarList; + } + private static async Task CreateContainerAsync( string image, IList command, @@ -262,4 +255,11 @@ private static int GetContainerId() { return Interlocked.Increment(ref incrementingContainerId); } + + private async Task InspectImageAndSanitizeVarsAsync(string image, CancellationToken cancellationToken = default) + { + var imageInspectResponse = await Client.Images.InspectImageAsync(image, cancellationToken); + this.SanitizeEnvironmentVariables(imageInspectResponse); + return imageInspectResponse; + } } diff --git a/src/Microsoft.ComponentDetection.Common/PathUtilityService.cs b/src/Microsoft.ComponentDetection.Common/PathUtilityService.cs index 27df98217..e5405aa93 100644 --- a/src/Microsoft.ComponentDetection.Common/PathUtilityService.cs +++ b/src/Microsoft.ComponentDetection.Common/PathUtilityService.cs @@ -75,8 +75,6 @@ public string ResolvePhysicalPath(string path) return fileInfo.Exists ? this.ResolvePathFromInfo(fileInfo) : null; } - private string ResolvePathFromInfo(FileSystemInfo info) => info.LinkTarget ?? info.FullName; - public string NormalizePath(string path) { if (string.IsNullOrWhiteSpace(path)) @@ -89,4 +87,6 @@ public string NormalizePath(string path) // AltDirectorySeparatorChar is / on Unix and on Windows. return path.Replace('\\', Path.AltDirectorySeparatorChar); } + + private string ResolvePathFromInfo(FileSystemInfo info) => info.LinkTarget ?? info.FullName; } diff --git a/src/Microsoft.ComponentDetection.Common/Utilities/StringUtilities.cs b/src/Microsoft.ComponentDetection.Common/Utilities/StringUtilities.cs index 878e0c140..1c498ba80 100644 --- a/src/Microsoft.ComponentDetection.Common/Utilities/StringUtilities.cs +++ b/src/Microsoft.ComponentDetection.Common/Utilities/StringUtilities.cs @@ -6,8 +6,8 @@ namespace Microsoft.ComponentDetection.Common; public static class StringUtilities { - private static readonly Regex SensitiveInfoRegex = new Regex(@"(?<=https://)(.+)(?=@)", RegexOptions.Compiled | RegexOptions.IgnoreCase, TimeSpan.FromSeconds(5)); public const string SensitivePlaceholder = "******"; + private static readonly Regex SensitiveInfoRegex = new Regex(@"(?<=https://)(.+)(?=@)", RegexOptions.Compiled | RegexOptions.IgnoreCase, TimeSpan.FromSeconds(5)); /// /// Utility method to remove sensitive information from a string, currently focused on removing on the credentials placed within URL which can be part of CLI commands. diff --git a/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs b/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs index 7669726d4..4fe1db91f 100644 --- a/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs @@ -17,18 +17,6 @@ namespace Microsoft.ComponentDetection.Contracts; /// Specialized base class for file based component detection. public abstract class FileComponentDetector : IComponentDetector { - /// - /// Gets or sets the factory for handing back component streams to File detectors. - /// - protected IComponentStreamEnumerableFactory ComponentStreamEnumerableFactory { get; set; } - - protected IObservableDirectoryWalkerFactory Scanner { get; set; } - - /// - /// Gets or sets the logger for writing basic logging message to both console and file. - /// - protected ILogger Logger { get; set; } - public IComponentRecorder ComponentRecorder { get; private set; } /// @@ -46,6 +34,25 @@ public abstract class FileComponentDetector : IComponentDetector /// Gets the version of this component detector. public abstract int Version { get; } + public virtual bool NeedsAutomaticRootDependencyCalculation { get; protected set; } + + /// + /// List of any any additional properties as key-value pairs that we would like to capture for the detector. + /// + public List<(string PropertyKey, string PropertyValue)> AdditionalProperties { get; set; } = []; + + /// + /// Gets or sets the factory for handing back component streams to File detectors. + /// + protected IComponentStreamEnumerableFactory ComponentStreamEnumerableFactory { get; set; } + + protected IObservableDirectoryWalkerFactory Scanner { get; set; } + + /// + /// Gets or sets the logger for writing basic logging message to both console and file. + /// + protected ILogger Logger { get; set; } + /// /// Gets the folder names that will be skipped by the Component Detector. /// @@ -57,15 +64,8 @@ public abstract class FileComponentDetector : IComponentDetector /// protected ScanRequest CurrentScanRequest { get; set; } - public virtual bool NeedsAutomaticRootDependencyCalculation { get; protected set; } - protected ConcurrentDictionary Telemetry { get; set; } = []; - /// - /// List of any any additional properties as key-value pairs that we would like to capture for the detector. - /// - public List<(string PropertyKey, string PropertyValue)> AdditionalProperties { get; set; } = []; - protected IObservable ComponentStreams { get; private set; } protected virtual bool EnableParallelism { get; set; } @@ -78,16 +78,6 @@ public async virtual Task ExecuteDetectorAsync(Sca return await this.ScanDirectoryAsync(request, cancellationToken); } - private Task ScanDirectoryAsync(ScanRequest request, CancellationToken cancellationToken = default) - { - this.CurrentScanRequest = request; - - var filteredObservable = this.Scanner.GetFilteredComponentStreamObservable(request.SourceDirectory, this.SearchPatterns, request.ComponentRecorder); - - this.Logger.LogDebug("Registered {Detector}", this.GetType().FullName); - return this.ProcessAsync(filteredObservable, request.DetectorArgs, request.MaxThreads, request.CleanupCreatedFiles, cancellationToken); - } - /// /// Gets the file streams for the Detector's declared as an . /// @@ -111,37 +101,6 @@ protected Task> GetFileStreamsAsync(DirectoryInfo /// The lockfile version. protected void RecordLockfileVersion(string lockfileVersion) => this.Telemetry["LockfileVersion"] = lockfileVersion; - private async Task ProcessAsync( - IObservable processRequests, IDictionary detectorArgs, int maxThreads, bool cleanupCreatedFiles, CancellationToken cancellationToken = default) - { - var threadsToUse = this.EnableParallelism ? Math.Min(Environment.ProcessorCount, maxThreads) : 1; - this.Telemetry["ThreadsUsed"] = $"{threadsToUse}"; - - var processor = new ActionBlock( - async processRequest => await this.OnFileFoundAsync(processRequest, detectorArgs, cleanupCreatedFiles, cancellationToken), - new ExecutionDataflowBlockOptions - { - // MaxDegreeOfParallelism is the lower of the processor count and the max threads arg that the customer passed in - MaxDegreeOfParallelism = threadsToUse, - }); - - var preprocessedObserbable = await this.OnPrepareDetectionAsync(processRequests, detectorArgs, cancellationToken); - - await preprocessedObserbable.ForEachAsync(processRequest => processor.Post(processRequest)); - - processor.Complete(); - - await processor.Completion; - - await this.OnDetectionFinishedAsync(); - - return new IndividualDetectorScanResult - { - ResultCode = ProcessingResultCode.Success, - AdditionalTelemetryDetails = this.Telemetry.ToDictionary(kvp => kvp.Key, kvp => kvp.Value), - }; - } - /// /// Auxliary method executed before the actual scanning of a given file takes place. /// This method can be used to modify or create new ProcessRequests that later will @@ -174,4 +133,45 @@ protected virtual Task OnDetectionFinishedAsync() // Do not cleanup by default, only if the detector uses the FileComponentWithCleanup abstract class. protected virtual async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, bool cleanupCreatedFiles, CancellationToken cancellationToken = default) => await this.OnFileFoundAsync(processRequest, detectorArgs, cancellationToken); + + private Task ScanDirectoryAsync(ScanRequest request, CancellationToken cancellationToken = default) + { + this.CurrentScanRequest = request; + + var filteredObservable = this.Scanner.GetFilteredComponentStreamObservable(request.SourceDirectory, this.SearchPatterns, request.ComponentRecorder); + + this.Logger.LogDebug("Registered {Detector}", this.GetType().FullName); + return this.ProcessAsync(filteredObservable, request.DetectorArgs, request.MaxThreads, request.CleanupCreatedFiles, cancellationToken); + } + + private async Task ProcessAsync( + IObservable processRequests, IDictionary detectorArgs, int maxThreads, bool cleanupCreatedFiles, CancellationToken cancellationToken = default) + { + var threadsToUse = this.EnableParallelism ? Math.Min(Environment.ProcessorCount, maxThreads) : 1; + this.Telemetry["ThreadsUsed"] = $"{threadsToUse}"; + + var processor = new ActionBlock( + async processRequest => await this.OnFileFoundAsync(processRequest, detectorArgs, cleanupCreatedFiles, cancellationToken), + new ExecutionDataflowBlockOptions + { + // MaxDegreeOfParallelism is the lower of the processor count and the max threads arg that the customer passed in + MaxDegreeOfParallelism = threadsToUse, + }); + + var preprocessedObserbable = await this.OnPrepareDetectionAsync(processRequests, detectorArgs, cancellationToken); + + await preprocessedObserbable.ForEachAsync(processRequest => processor.Post(processRequest)); + + processor.Complete(); + + await processor.Completion; + + await this.OnDetectionFinishedAsync(); + + return new IndividualDetectorScanResult + { + ResultCode = ProcessingResultCode.Success, + AdditionalTelemetryDetails = this.Telemetry.ToDictionary(kvp => kvp.Key, kvp => kvp.Value), + }; + } } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs index 3137601e4..5d7abf08d 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs @@ -6,11 +6,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class CargoComponent : TypedComponent { - private CargoComponent() - { - // reserved for deserialization - } - public CargoComponent(string name, string version, string author = null, string license = null, string source = null) { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Cargo)); @@ -20,6 +15,11 @@ public CargoComponent(string name, string version, string author = null, string this.Source = source; } + private CargoComponent() + { + // reserved for deserialization + } + public string Name { get; set; } public string Version { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ConanComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ConanComponent.cs index a480d3eda..75cc34e77 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ConanComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ConanComponent.cs @@ -5,11 +5,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class ConanComponent : TypedComponent { - private ConanComponent() - { - // reserved for deserialization - } - public ConanComponent(string name, string version, string previous, string packageId) { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Conan)); @@ -18,6 +13,11 @@ public ConanComponent(string name, string version, string previous, string packa this.Sha1Hash = this.ValidateRequiredInput(packageId, nameof(this.Sha1Hash), nameof(ComponentType.Conan)); } + private ConanComponent() + { + // reserved for deserialization + } + public string Name { get; set; } public string Version { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs index e89b45428..b6efb9140 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs @@ -3,11 +3,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class DockerImageComponent : TypedComponent { - private DockerImageComponent() - { - /* Reserved for deserialization */ - } - public DockerImageComponent(string hash, string name = null, string tag = null) { this.Digest = this.ValidateRequiredInput(hash, nameof(this.Digest), nameof(ComponentType.DockerImage)); @@ -15,6 +10,11 @@ public DockerImageComponent(string hash, string name = null, string tag = null) this.Tag = tag; } + private DockerImageComponent() + { + /* Reserved for deserialization */ + } + public string Name { get; set; } public string Digest { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs index 99b49f5af..f1ad3184d 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs @@ -7,11 +7,6 @@ public class DotNetComponent : TypedComponent { private const string UnknownValue = "unknown"; - private DotNetComponent() - { - /* Reserved for deserialization */ - } - public DotNetComponent(string sdkVersion, string targetFramework = null, string projectType = null) { if (string.IsNullOrWhiteSpace(sdkVersion) && string.IsNullOrWhiteSpace(targetFramework)) @@ -24,6 +19,11 @@ public DotNetComponent(string sdkVersion, string targetFramework = null, string this.ProjectType = string.IsNullOrWhiteSpace(projectType) ? UnknownValue : projectType; } + private DotNetComponent() + { + /* Reserved for deserialization */ + } + /// /// SDK Version detected, could be null if no global.json exists and no dotnet is on the path. /// diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs index 749396f87..9291e09da 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs @@ -37,8 +37,6 @@ private GoComponent() public override ComponentType Type => ComponentType.Go; - protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}"; - public override bool Equals(object obj) { return obj is GoComponent otherComponent && this.Equals(otherComponent); @@ -58,4 +56,6 @@ public override int GetHashCode() { return this.Name.GetHashCode() ^ this.Version.GetHashCode() ^ this.Hash.GetHashCode(); } + + protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs index 7ec336364..cfc498d8d 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs @@ -6,11 +6,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class LinuxComponent : TypedComponent { - private LinuxComponent() - { - /* Reserved for deserialization */ - } - public LinuxComponent(string distribution, string release, string name, string version, string license = null, string author = null) { this.Distribution = this.ValidateRequiredInput(distribution, nameof(this.Distribution), nameof(ComponentType.Linux)); @@ -21,6 +16,11 @@ public LinuxComponent(string distribution, string release, string name, string v this.Author = author; } + private LinuxComponent() + { + /* Reserved for deserialization */ + } + public string Distribution { get; set; } public string Release { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs index 6715d6ce4..51edb38ba 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs @@ -6,11 +6,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class NpmComponent : TypedComponent { - private NpmComponent() - { - /* Reserved for deserialization */ - } - public NpmComponent(string name, string version, string hash = null, NpmAuthor author = null) { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Npm)); @@ -19,6 +14,11 @@ public NpmComponent(string name, string version, string hash = null, NpmAuthor a this.Author = author; } + private NpmComponent() + { + /* Reserved for deserialization */ + } + public string Name { get; set; } public string Version { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs index 9e716d391..45eeafb9d 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs @@ -5,11 +5,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class NuGetComponent : TypedComponent { - private NuGetComponent() - { - /* Reserved for deserialization */ - } - public NuGetComponent(string name, string version, string[] authors = null) { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.NuGet)); @@ -17,6 +12,11 @@ public NuGetComponent(string name, string version, string[] authors = null) this.Authors = authors; } + private NuGetComponent() + { + /* Reserved for deserialization */ + } + public string Name { get; set; } public string Version { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs index 68f6483c5..1cdcd29e3 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs @@ -5,11 +5,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class OtherComponent : TypedComponent { - private OtherComponent() - { - /* Reserved for deserialization */ - } - public OtherComponent(string name, string version, Uri downloadUrl, string hash) { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Other)); @@ -18,6 +13,11 @@ public OtherComponent(string name, string version, Uri downloadUrl, string hash) this.Hash = hash; } + private OtherComponent() + { + /* Reserved for deserialization */ + } + public string Name { get; set; } public string Version { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs index 5858fb7c2..7e88ab755 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs @@ -7,11 +7,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class PipComponent : TypedComponent { - private PipComponent() - { - /* Reserved for deserialization */ - } - public PipComponent(string name, string version, string author = null, string license = null) { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Pip)); @@ -20,6 +15,11 @@ public PipComponent(string name, string version, string author = null, string li this.License = license; } + private PipComponent() + { + /* Reserved for deserialization */ + } + public string Name { get; set; } public string Version { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs index 64ae8681e..fe51b572f 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs @@ -6,11 +6,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class PodComponent : TypedComponent { - private PodComponent() - { - /* Reserved for deserialization */ - } - public PodComponent(string name, string version, string specRepo = "") { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Pod)); @@ -18,6 +13,11 @@ public PodComponent(string name, string version, string specRepo = "") this.SpecRepo = specRepo; } + private PodComponent() + { + /* Reserved for deserialization */ + } + public string Name { get; set; } public string Version { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs index 1c6ce94f9..b797451ec 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs @@ -5,11 +5,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class RubyGemsComponent : TypedComponent { - private RubyGemsComponent() - { - /* Reserved for deserialization */ - } - public RubyGemsComponent(string name, string version, string source = "") { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.RubyGems)); @@ -17,6 +12,11 @@ public RubyGemsComponent(string name, string version, string source = "") this.Source = source; } + private RubyGemsComponent() + { + /* Reserved for deserialization */ + } + public string Name { get; set; } public string Version { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs index 79ab31ea1..dcb785afb 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs @@ -5,11 +5,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class SpdxComponent : TypedComponent { - private SpdxComponent() - { - /* Reserved for deserialization */ - } - public SpdxComponent(string spdxVersion, Uri documentNamespace, string name, string checksum, string rootElementId, string path) { this.SpdxVersion = this.ValidateRequiredInput(spdxVersion, nameof(this.SpdxVersion), nameof(ComponentType.Spdx)); @@ -20,6 +15,11 @@ public SpdxComponent(string spdxVersion, Uri documentNamespace, string name, str this.Path = this.ValidateRequiredInput(path, nameof(this.Path), nameof(ComponentType.Spdx)); } + private SpdxComponent() + { + /* Reserved for deserialization */ + } + public override ComponentType Type => ComponentType.Spdx; public string RootElementId { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs index 27c6c7f11..4fc5cc1b1 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs @@ -5,11 +5,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class VcpkgComponent : TypedComponent { - private VcpkgComponent() - { - /* Reserved for deserialization */ - } - public VcpkgComponent(string spdxid, string name, string version, string triplet = null, string portVersion = null, string description = null, string downloadLocation = null) { int.TryParse(portVersion, out var port); @@ -23,6 +18,11 @@ public VcpkgComponent(string spdxid, string name, string version, string triplet this.DownloadLocation = downloadLocation; } + private VcpkgComponent() + { + /* Reserved for deserialization */ + } + public string SPDXID { get; set; } public string Name { get; set; } diff --git a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs index 8770ce3bb..a0a5139e8 100644 --- a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs @@ -59,6 +59,72 @@ public DotNetComponentDetector( public override IEnumerable Categories => ["DotNet"]; + public override Task ExecuteDetectorAsync(ScanRequest request, CancellationToken cancellationToken = default) + { + this.sourceDirectory = this.NormalizeDirectory(request.SourceDirectory.FullName); + this.sourceFileRootDirectory = this.NormalizeDirectory(request.SourceFileRoot?.FullName); + + return base.ExecuteDetectorAsync(request, cancellationToken); + } + + protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) + { + var lockFile = this.lockFileFormat.Read(processRequest.ComponentStream.Stream, processRequest.ComponentStream.Location); + + if (lockFile.PackageSpec is null || lockFile.PackageSpec.RestoreMetadata is null) + { + // The lock file is not valid, or does not contain a PackageSpec. + // This could be due to the lock file being generated by a different version of the SDK. + // We should not fail the detector, but we should log a warning. + this.Logger.LogWarning("Lock file {LockFilePath} does not contain project information.", processRequest.ComponentStream.Location); + return; + } + + var projectAssetsDirectory = this.pathUtilityService.GetParentDirectory(processRequest.ComponentStream.Location); + var projectPath = lockFile.PackageSpec.RestoreMetadata.ProjectPath; + var projectOutputPath = lockFile.PackageSpec.RestoreMetadata.OutputPath; + + // The output path should match the location that the assets file, if it doesn't we could be analyzing paths + // on a different filesystem root than they were created. + // Attempt to rebase paths based on the difference between this file's location and the output path. + var rebasePath = this.GetRootRebasePath(projectAssetsDirectory, projectOutputPath); + + if (rebasePath is not null) + { + projectPath = Path.Combine(this.sourceDirectory!, Path.GetRelativePath(rebasePath, projectPath)); + projectOutputPath = Path.Combine(this.sourceDirectory!, Path.GetRelativePath(rebasePath, projectOutputPath)); + } + + if (!this.fileUtilityService.Exists(projectPath)) + { + // Could be the assets file was not actually from this build + this.Logger.LogWarning("Project path {ProjectPath} specified by {ProjectAssetsPath} does not exist.", projectPath, processRequest.ComponentStream.Location); + } + + var projectDirectory = this.pathUtilityService.GetParentDirectory(projectPath); + var sdkVersion = await this.GetSdkVersionAsync(projectDirectory, cancellationToken); + + var projectName = lockFile.PackageSpec.RestoreMetadata.ProjectName; + + if (!this.directoryUtilityService.Exists(projectOutputPath)) + { + this.Logger.LogWarning("Project output path {ProjectOutputPath} specified by {ProjectAssetsPath} does not exist.", projectOutputPath, processRequest.ComponentStream.Location); + + // default to use the location of the assets file. + projectOutputPath = projectAssetsDirectory; + } + + var targetType = this.GetProjectType(projectOutputPath, projectName, cancellationToken); + + var componentReporter = this.ComponentRecorder.CreateSingleFileComponentRecorder(projectPath); + foreach (var target in lockFile.Targets ?? []) + { + var targetFramework = target.TargetFramework?.GetShortFolderName(); + + componentReporter.RegisterUsage(new DetectedComponent(new DotNetComponent(sdkVersion, targetFramework, targetType))); + } + } + private static string TrimAllEndingDirectorySeparators(string path) { string last; @@ -141,72 +207,6 @@ private static string TrimAllEndingDirectorySeparators(string path) } } - public override Task ExecuteDetectorAsync(ScanRequest request, CancellationToken cancellationToken = default) - { - this.sourceDirectory = this.NormalizeDirectory(request.SourceDirectory.FullName); - this.sourceFileRootDirectory = this.NormalizeDirectory(request.SourceFileRoot?.FullName); - - return base.ExecuteDetectorAsync(request, cancellationToken); - } - - protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) - { - var lockFile = this.lockFileFormat.Read(processRequest.ComponentStream.Stream, processRequest.ComponentStream.Location); - - if (lockFile.PackageSpec is null || lockFile.PackageSpec.RestoreMetadata is null) - { - // The lock file is not valid, or does not contain a PackageSpec. - // This could be due to the lock file being generated by a different version of the SDK. - // We should not fail the detector, but we should log a warning. - this.Logger.LogWarning("Lock file {LockFilePath} does not contain project information.", processRequest.ComponentStream.Location); - return; - } - - var projectAssetsDirectory = this.pathUtilityService.GetParentDirectory(processRequest.ComponentStream.Location); - var projectPath = lockFile.PackageSpec.RestoreMetadata.ProjectPath; - var projectOutputPath = lockFile.PackageSpec.RestoreMetadata.OutputPath; - - // The output path should match the location that the assets file, if it doesn't we could be analyzing paths - // on a different filesystem root than they were created. - // Attempt to rebase paths based on the difference between this file's location and the output path. - var rebasePath = this.GetRootRebasePath(projectAssetsDirectory, projectOutputPath); - - if (rebasePath is not null) - { - projectPath = Path.Combine(this.sourceDirectory!, Path.GetRelativePath(rebasePath, projectPath)); - projectOutputPath = Path.Combine(this.sourceDirectory!, Path.GetRelativePath(rebasePath, projectOutputPath)); - } - - if (!this.fileUtilityService.Exists(projectPath)) - { - // Could be the assets file was not actually from this build - this.Logger.LogWarning("Project path {ProjectPath} specified by {ProjectAssetsPath} does not exist.", projectPath, processRequest.ComponentStream.Location); - } - - var projectDirectory = this.pathUtilityService.GetParentDirectory(projectPath); - var sdkVersion = await this.GetSdkVersionAsync(projectDirectory, cancellationToken); - - var projectName = lockFile.PackageSpec.RestoreMetadata.ProjectName; - - if (!this.directoryUtilityService.Exists(projectOutputPath)) - { - this.Logger.LogWarning("Project output path {ProjectOutputPath} specified by {ProjectAssetsPath} does not exist.", projectOutputPath, processRequest.ComponentStream.Location); - - // default to use the location of the assets file. - projectOutputPath = projectAssetsDirectory; - } - - var targetType = this.GetProjectType(projectOutputPath, projectName, cancellationToken); - - var componentReporter = this.ComponentRecorder.CreateSingleFileComponentRecorder(projectPath); - foreach (var target in lockFile.Targets ?? []) - { - var targetFramework = target.TargetFramework?.GetShortFolderName(); - - componentReporter.RegisterUsage(new DetectedComponent(new DotNetComponent(sdkVersion, targetFramework, targetType))); - } - } - private string? GetProjectType(string projectOutputPath, string projectName, CancellationToken cancellationToken) { if (this.directoryUtilityService.Exists(projectOutputPath) && diff --git a/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoModParser.cs b/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoModParser.cs index 2c3afee0b..55257d186 100644 --- a/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoModParser.cs +++ b/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoModParser.cs @@ -18,6 +18,51 @@ public class GoModParser : IGoParser public GoModParser(ILogger logger) => this.logger = logger; + public async Task ParseAsync( + ISingleFileComponentRecorder singleFileComponentRecorder, + IComponentStream file, + GoGraphTelemetryRecord record) + { + // Collect replace directives + var (replacePathDirectives, moduleReplacements) = await this.GetAllReplaceDirectivesAsync(file); + + // Rewind stream after reading replace directives + file.Stream.Seek(0, SeekOrigin.Begin); + + using var reader = new StreamReader(file.Stream); + + // There can be multiple require( ) sections in go 1.17+. loop over all of them. + while (!reader.EndOfStream) + { + var line = await reader.ReadLineAsync(); + + while (line != null && !line.StartsWith("require (")) + { + if (line.StartsWith("go ")) + { + record.GoModVersion = line[3..].Trim(); + } + + // In go >= 1.17, direct dependencies are listed as "require x/y v1.2.3", and transitive dependencies + // are listed in the require () section + if (line.StartsWith(StartString)) + { + this.TryRegisterDependencyFromModLine(file, line[StartString.Length..], singleFileComponentRecorder, replacePathDirectives, moduleReplacements); + } + + line = await reader.ReadLineAsync(); + } + + // Stopping at the first ) restrict the detection to only the require section. + while ((line = await reader.ReadLineAsync()) != null && !line.EndsWith(')')) + { + this.TryRegisterDependencyFromModLine(file, line, singleFileComponentRecorder, replacePathDirectives, moduleReplacements); + } + } + + return true; + } + /// /// Checks whether the input path is a potential local file system path /// 1. '.' checks whether the path is relative to current directory. @@ -68,51 +113,6 @@ private static void HandleReplaceDirective( } } - public async Task ParseAsync( - ISingleFileComponentRecorder singleFileComponentRecorder, - IComponentStream file, - GoGraphTelemetryRecord record) - { - // Collect replace directives - var (replacePathDirectives, moduleReplacements) = await this.GetAllReplaceDirectivesAsync(file); - - // Rewind stream after reading replace directives - file.Stream.Seek(0, SeekOrigin.Begin); - - using var reader = new StreamReader(file.Stream); - - // There can be multiple require( ) sections in go 1.17+. loop over all of them. - while (!reader.EndOfStream) - { - var line = await reader.ReadLineAsync(); - - while (line != null && !line.StartsWith("require (")) - { - if (line.StartsWith("go ")) - { - record.GoModVersion = line[3..].Trim(); - } - - // In go >= 1.17, direct dependencies are listed as "require x/y v1.2.3", and transitive dependencies - // are listed in the require () section - if (line.StartsWith(StartString)) - { - this.TryRegisterDependencyFromModLine(file, line[StartString.Length..], singleFileComponentRecorder, replacePathDirectives, moduleReplacements); - } - - line = await reader.ReadLineAsync(); - } - - // Stopping at the first ) restrict the detection to only the require section. - while ((line = await reader.ReadLineAsync()) != null && !line.EndsWith(')')) - { - this.TryRegisterDependencyFromModLine(file, line, singleFileComponentRecorder, replacePathDirectives, moduleReplacements); - } - } - - return true; - } - private void TryRegisterDependencyFromModLine(IComponentStream file, string line, ISingleFileComponentRecorder singleFileComponentRecorder, HashSet replacePathDirectives, Dictionary moduleReplacements) { if (line.Trim().StartsWith("//")) diff --git a/src/Microsoft.ComponentDetection.Detectors/linux/LinuxContainerDetector.cs b/src/Microsoft.ComponentDetection.Detectors/linux/LinuxContainerDetector.cs index fdf19a4bf..228f476be 100644 --- a/src/Microsoft.ComponentDetection.Detectors/linux/LinuxContainerDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/linux/LinuxContainerDetector.cs @@ -49,15 +49,6 @@ ILogger logger /// public bool NeedsAutomaticRootDependencyCalculation => false; - /// - /// Gets the component types that should be detected by this detector. - /// By default, only Linux system packages are detected. - /// Override this method in derived classes to enable detection of additional component types. - /// - /// A set of component types to include in scan results. - protected virtual ISet GetEnabledComponentTypes() => - new HashSet { ComponentType.Linux }; - /// public async Task ExecuteDetectorAsync( ScanRequest request, @@ -118,6 +109,15 @@ public async Task ExecuteDetectorAsync( }; } + /// + /// Gets the component types that should be detected by this detector. + /// By default, only Linux system packages are detected. + /// Override this method in derived classes to enable detection of additional component types. + /// + /// A set of component types to include in scan results. + protected virtual ISet GetEnabledComponentTypes() => + new HashSet { ComponentType.Linux }; + /// /// Extracts and returns the timeout defined by the user, or a default value if one is not provided. /// diff --git a/src/Microsoft.ComponentDetection.Detectors/maven/MavenCommandService.cs b/src/Microsoft.ComponentDetection.Detectors/maven/MavenCommandService.cs index 2e75bd43f..c3a633b97 100644 --- a/src/Microsoft.ComponentDetection.Detectors/maven/MavenCommandService.cs +++ b/src/Microsoft.ComponentDetection.Detectors/maven/MavenCommandService.cs @@ -11,7 +11,6 @@ namespace Microsoft.ComponentDetection.Detectors.Maven; public class MavenCommandService : IMavenCommandService { - private const string DetectorLogPrefix = "MvnCli detector"; internal const string MvnCLIFileLevelTimeoutSecondsEnvVar = "MvnCLIFileLevelTimeoutSeconds"; internal const string PrimaryCommand = "mvn"; @@ -19,6 +18,7 @@ public class MavenCommandService : IMavenCommandService internal static readonly string[] AdditionalValidCommands = ["mvn.cmd"]; + private const string DetectorLogPrefix = "MvnCli detector"; private readonly ICommandLineInvocationService commandLineInvocationService; private readonly IMavenStyleDependencyGraphParserService parserService; private readonly IEnvironmentVariableService envVarService; diff --git a/src/Microsoft.ComponentDetection.Detectors/maven/MvnCliComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/maven/MvnCliComponentDetector.cs index a93f5c6b2..6e15a38c9 100644 --- a/src/Microsoft.ComponentDetection.Detectors/maven/MvnCliComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/maven/MvnCliComponentDetector.cs @@ -45,11 +45,6 @@ public MvnCliComponentDetector( public override IEnumerable Categories => [Enum.GetName(typeof(DetectorClass), DetectorClass.Maven)]; - private void LogDebugWithId(string message) - { - this.Logger.LogDebug("{DetectorId} detector: {Message}", this.Id, message); - } - protected override async Task> OnPrepareDetectionAsync(IObservable processRequests, IDictionary detectorArgs, CancellationToken cancellationToken = default) { if (!await this.mavenCommandService.MavenCLIExistsAsync()) @@ -102,6 +97,11 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID await Task.CompletedTask; } + private void LogDebugWithId(string message) + { + this.Logger.LogDebug("{DetectorId} detector: {Message}", this.Id, message); + } + private IObservable RemoveNestedPomXmls(IObservable componentStreams) { var directoryItemFacades = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); diff --git a/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentUtilities.cs b/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentUtilities.cs index 12effe1f3..85a89b32c 100644 --- a/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentUtilities.cs +++ b/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentUtilities.cs @@ -15,13 +15,13 @@ namespace Microsoft.ComponentDetection.Detectors.Npm; public static class NpmComponentUtilities { + public static readonly string NodeModules = "node_modules"; + public static readonly string LockFile3EnvFlag = "CD_LOCKFILE_V3_ENABLED"; + private static readonly Regex UnsafeCharactersRegex = new Regex( @"[?<>#%{}|`'^\\~\[\]""\s\x7f]|[\x00-\x1f]|[\x80-\xff]", RegexOptions.Compiled); - public static readonly string NodeModules = "node_modules"; - public static readonly string LockFile3EnvFlag = "CD_LOCKFILE_V3_ENABLED"; - public static void TraverseAndRecordComponents(JProperty currentDependency, ISingleFileComponentRecorder singleFileComponentRecorder, TypedComponent component, TypedComponent explicitReferencedDependency, string parentComponentId = null) { var isDevDependency = currentDependency.Value["dev"] is JValue devJValue && (bool)devJValue; diff --git a/src/Microsoft.ComponentDetection.Detectors/npm/NpmLockfileDetectorBase.cs b/src/Microsoft.ComponentDetection.Detectors/npm/NpmLockfileDetectorBase.cs index 690b13172..ef8609e70 100644 --- a/src/Microsoft.ComponentDetection.Detectors/npm/NpmLockfileDetectorBase.cs +++ b/src/Microsoft.ComponentDetection.Detectors/npm/NpmLockfileDetectorBase.cs @@ -54,11 +54,11 @@ protected NpmLockfileDetectorBase( public override IEnumerable SupportedComponentTypes { get; } = [ComponentType.Npm]; - private List LernaFiles { get; } = []; - /// protected override IList SkippedFolders => ["node_modules", "pnpm-store"]; + private List LernaFiles { get; } = []; + protected abstract bool IsSupportedLockfileVersion(int lockfileVersion); protected abstract JToken ResolveDependencyObject(JToken packageLockJToken); diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/FrameworkPackages/FrameworkPackages.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/FrameworkPackages/FrameworkPackages.cs index 6dd5443df..7f600fe32 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/FrameworkPackages/FrameworkPackages.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/FrameworkPackages/FrameworkPackages.cs @@ -54,28 +54,6 @@ public FrameworkPackages(NuGetFramework framework, string frameworkName, Framewo public Dictionary Packages { get; } = new Dictionary(StringComparer.OrdinalIgnoreCase); - private static string GetFrameworkKey(string frameworkName) => - frameworkName switch - { - FrameworkNames.NetStandardLibrary => DefaultFrameworkKey, - FrameworkNames.NetCoreApp => DefaultFrameworkKey, - _ => frameworkName, - }; - - internal static void Register(params FrameworkPackages[] toRegister) - { - foreach (var frameworkPackages in toRegister) - { - if (!FrameworkPackagesByFramework.TryGetValue(frameworkPackages.Framework, out var frameworkPackagesForVersion)) - { - FrameworkPackagesByFramework[frameworkPackages.Framework] = frameworkPackagesForVersion = []; - } - - var frameworkKey = GetFrameworkKey(frameworkPackages.FrameworkName); - frameworkPackagesForVersion[frameworkKey] = frameworkPackages; - } - } - public static FrameworkPackages[] GetFrameworkPackages(NuGetFramework framework, string[] frameworkReferences, LockFileTarget lockFileTarget) { var frameworkPackages = new List(); @@ -110,6 +88,34 @@ public static FrameworkPackages[] GetFrameworkPackages(NuGetFramework framework, return frameworkPackages.ToArray(); } + public bool IsAFrameworkComponent(string id, NuGetVersion version) => this.Packages.TryGetValue(id, out var frameworkPackageVersion) && frameworkPackageVersion >= version; + + IEnumerator> IEnumerable>.GetEnumerator() => this.Packages.GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => throw new NotImplementedException(); + + internal static void Register(params FrameworkPackages[] toRegister) + { + foreach (var frameworkPackages in toRegister) + { + if (!FrameworkPackagesByFramework.TryGetValue(frameworkPackages.Framework, out var frameworkPackagesForVersion)) + { + FrameworkPackagesByFramework[frameworkPackages.Framework] = frameworkPackagesForVersion = []; + } + + var frameworkKey = GetFrameworkKey(frameworkPackages.FrameworkName); + frameworkPackagesForVersion[frameworkKey] = frameworkPackages; + } + } + + private static string GetFrameworkKey(string frameworkName) => + frameworkName switch + { + FrameworkNames.NetStandardLibrary => DefaultFrameworkKey, + FrameworkNames.NetCoreApp => DefaultFrameworkKey, + _ => frameworkName, + }; + private static IEnumerable GetLegacyFrameworkPackagesFromPlatformPackages(NuGetFramework framework, LockFileTarget lockFileTarget) { if (framework.Framework == FrameworkConstants.FrameworkIdentifiers.NetCoreApp && framework.Version.Major < 3) @@ -229,12 +235,6 @@ private void Add(string id, string version) } } - public bool IsAFrameworkComponent(string id, NuGetVersion version) => this.Packages.TryGetValue(id, out var frameworkPackageVersion) && frameworkPackageVersion >= version; - - IEnumerator> IEnumerable>.GetEnumerator() => this.Packages.GetEnumerator(); - - IEnumerator IEnumerable.GetEnumerator() => throw new NotImplementedException(); - internal static class FrameworkNames { public const string AspNetCoreApp = "Microsoft.AspNetCore.App"; diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/FrameworkPackages/FrameworkPackages.net9.0.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/FrameworkPackages/FrameworkPackages.net9.0.cs index 15c0a7c15..330001de7 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/FrameworkPackages/FrameworkPackages.net9.0.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/FrameworkPackages/FrameworkPackages.net9.0.cs @@ -11,8 +11,6 @@ internal partial class FrameworkPackages { internal static class NETCoreApp90 { - private static NuGetFramework Net90 { get; } = new NuGetFramework(FrameworkConstants.FrameworkIdentifiers.NetCoreApp, new Version(9, 0)); - internal static FrameworkPackages Instance { get; } = new(Net90, FrameworkNames.NetCoreApp, NETCoreApp80.Instance) { { "Microsoft.VisualBasic", "10.4.0" }, @@ -195,6 +193,8 @@ internal static class NETCoreApp90 { "System.Security.Cryptography.Xml", "8.0.2" }, }; + private static NuGetFramework Net90 { get; } = new NuGetFramework(FrameworkConstants.FrameworkIdentifiers.NetCoreApp, new Version(9, 0)); + internal static void Register() => FrameworkPackages.Register(Instance, AspNetCore, WindowsDesktop); } } diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetComponentDetector.cs index c4ccac918..87a335b4b 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetComponentDetector.cs @@ -19,10 +19,10 @@ namespace Microsoft.ComponentDetection.Detectors.NuGet; public class NuGetComponentDetector : FileComponentDetector { - private static readonly IEnumerable LowConfidencePackages = ["Newtonsoft.Json"]; - public const string NugetConfigFileName = "nuget.config"; + private static readonly IEnumerable LowConfidencePackages = ["Newtonsoft.Json"]; + private readonly IList repositoryPathKeyNames = ["repositorypath", "globalpackagesfolder"]; public NuGetComponentDetector( diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetProjectModelProjectCentricComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetProjectModelProjectCentricComponentDetector.cs index 068283dd4..20252538f 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetProjectModelProjectCentricComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetProjectModelProjectCentricComponentDetector.cs @@ -43,46 +43,6 @@ public NuGetProjectModelProjectCentricComponentDetector( public override int Version { get; } = 2; - private static string[] GetFrameworkReferences(LockFile lockFile, LockFileTarget target) - { - var frameworkInformation = lockFile.PackageSpec.TargetFrameworks.FirstOrDefault(x => x.FrameworkName.Equals(target.TargetFramework)); - - if (frameworkInformation == null) - { - return []; - } - - // add directly referenced frameworks - var results = frameworkInformation.FrameworkReferences.Select(x => x.Name); - - // add transitive framework references - results = results.Concat(target.Libraries.SelectMany(l => l.FrameworkReferences)); - - return results.Distinct().ToArray(); - } - - private static bool IsADevelopmentDependency(LockFileTargetLibrary library, LockFile lockFile) - { - // a placeholder item is an empty file that doesn't exist with name _._ meant to indicate an empty folder in a nuget package, but also used by NuGet when a package's assets are excluded. - static bool IsAPlaceholderItem(LockFileItem item) => Path.GetFileName(item.Path).Equals(PackagingCoreConstants.EmptyFolder, StringComparison.OrdinalIgnoreCase); - - // All(IsAPlaceholderItem) checks if the collection is empty or all items are placeholders. - return library.RuntimeAssemblies.All(IsAPlaceholderItem) && - library.RuntimeTargets.All(IsAPlaceholderItem) && - library.ResourceAssemblies.All(IsAPlaceholderItem) && - library.NativeLibraries.All(IsAPlaceholderItem) && - library.ContentFiles.All(IsAPlaceholderItem) && - library.Build.All(IsAPlaceholderItem) && - library.BuildMultiTargeting.All(IsAPlaceholderItem) && - - // The SDK looks at the library for analyzers using the following hueristic: - // https://github.com/dotnet/sdk/blob/d7fe6e66d8f67dc93c5c294a75f42a2924889196/src/Tasks/Microsoft.NET.Build.Tasks/NuGetUtils.NuGet.cs#L43 - (!lockFile.GetLibrary(library.Name, library.Version)?.Files - .Any(file => file.StartsWith("analyzers", StringComparison.Ordinal) - && file.EndsWith(".dll", StringComparison.OrdinalIgnoreCase) - && !file.EndsWith(".resources.dll", StringComparison.OrdinalIgnoreCase)) ?? false); - } - protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) { try @@ -135,6 +95,46 @@ bool IsFrameworkOrDevelopmentDependency(LockFileTargetLibrary library) => return Task.CompletedTask; } + private static string[] GetFrameworkReferences(LockFile lockFile, LockFileTarget target) + { + var frameworkInformation = lockFile.PackageSpec.TargetFrameworks.FirstOrDefault(x => x.FrameworkName.Equals(target.TargetFramework)); + + if (frameworkInformation == null) + { + return []; + } + + // add directly referenced frameworks + var results = frameworkInformation.FrameworkReferences.Select(x => x.Name); + + // add transitive framework references + results = results.Concat(target.Libraries.SelectMany(l => l.FrameworkReferences)); + + return results.Distinct().ToArray(); + } + + private static bool IsADevelopmentDependency(LockFileTargetLibrary library, LockFile lockFile) + { + // a placeholder item is an empty file that doesn't exist with name _._ meant to indicate an empty folder in a nuget package, but also used by NuGet when a package's assets are excluded. + static bool IsAPlaceholderItem(LockFileItem item) => Path.GetFileName(item.Path).Equals(PackagingCoreConstants.EmptyFolder, StringComparison.OrdinalIgnoreCase); + + // All(IsAPlaceholderItem) checks if the collection is empty or all items are placeholders. + return library.RuntimeAssemblies.All(IsAPlaceholderItem) && + library.RuntimeTargets.All(IsAPlaceholderItem) && + library.ResourceAssemblies.All(IsAPlaceholderItem) && + library.NativeLibraries.All(IsAPlaceholderItem) && + library.ContentFiles.All(IsAPlaceholderItem) && + library.Build.All(IsAPlaceholderItem) && + library.BuildMultiTargeting.All(IsAPlaceholderItem) && + + // The SDK looks at the library for analyzers using the following hueristic: + // https://github.com/dotnet/sdk/blob/d7fe6e66d8f67dc93c5c294a75f42a2924889196/src/Tasks/Microsoft.NET.Build.Tasks/NuGetUtils.NuGet.cs#L43 + (!lockFile.GetLibrary(library.Name, library.Version)?.Files + .Any(file => file.StartsWith("analyzers", StringComparison.Ordinal) + && file.EndsWith(".dll", StringComparison.OrdinalIgnoreCase) + && !file.EndsWith(".resources.dll", StringComparison.OrdinalIgnoreCase)) ?? false); + } + private void NavigateAndRegister( LockFileTarget target, HashSet explicitlyReferencedComponentIds, diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/Contracts/PipDependencySpecification.cs b/src/Microsoft.ComponentDetection.Detectors/pip/Contracts/PipDependencySpecification.cs index 0fd941d75..7c111bce6 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/Contracts/PipDependencySpecification.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/Contracts/PipDependencySpecification.cs @@ -91,64 +91,6 @@ public PipDependencySpecification(ILogger logger, string packageString, bool req private ILogger Logger { get; set; } - private void Initialize(string packageString, bool requiresDist) - { - if (requiresDist) - { - var distMatch = RequiresDistRegex.Match(packageString); - - for (var i = 1; i < distMatch.Groups.Count; i++) - { - if (string.IsNullOrWhiteSpace(distMatch.Groups[i].Value)) - { - continue; - } - - if (string.IsNullOrWhiteSpace(this.Name)) - { - this.Name = distMatch.Groups[i].Value.Trim(); - } - else - { - this.DependencySpecifiers = distMatch.Groups[i].Value.Split(','); - } - } - - var conditionalDependenciesMatches = RequiresDistConditionalDependenciesMatch.Matches(packageString); - - for (var i = 0; i < conditionalDependenciesMatches.Count; i++) - { - if (!string.IsNullOrWhiteSpace(conditionalDependenciesMatches[i].Value)) - { - this.ConditionalDependencySpecifiers.Add(conditionalDependenciesMatches[i].Value); - } - } - } - else - { - var nameMatches = PipNameExtractionRegex.Match(packageString); - var versionMatches = PipVersionExtractionRegex.Match(packageString); - - if (nameMatches.Captures.Count > 0) - { - this.Name = nameMatches.Captures[0].Value; - } - else - { - this.Name = packageString; - } - - if (versionMatches.Captures.Count > 0) - { - this.DependencySpecifiers = versionMatches.Captures[0].Value.Split(','); - } - } - - this.DependencySpecifiers = this.DependencySpecifiers.Where(x => !x.Contains("python_version")) - .Select(x => x.Trim()) - .ToList(); - } - /// /// Whether or not the package is safe to resolve based on the packagesToIgnore. /// @@ -272,4 +214,62 @@ public bool IsValidParentPackage(Dictionary pythonEnvironmentVar !this.PackageIsUnsafe() && this.PackageConditionsMet(pythonEnvironmentVariables) && !this.ConditionalDependencySpecifiers.Any(s => s.Contains("extra ==", StringComparison.OrdinalIgnoreCase)); + + private void Initialize(string packageString, bool requiresDist) + { + if (requiresDist) + { + var distMatch = RequiresDistRegex.Match(packageString); + + for (var i = 1; i < distMatch.Groups.Count; i++) + { + if (string.IsNullOrWhiteSpace(distMatch.Groups[i].Value)) + { + continue; + } + + if (string.IsNullOrWhiteSpace(this.Name)) + { + this.Name = distMatch.Groups[i].Value.Trim(); + } + else + { + this.DependencySpecifiers = distMatch.Groups[i].Value.Split(','); + } + } + + var conditionalDependenciesMatches = RequiresDistConditionalDependenciesMatch.Matches(packageString); + + for (var i = 0; i < conditionalDependenciesMatches.Count; i++) + { + if (!string.IsNullOrWhiteSpace(conditionalDependenciesMatches[i].Value)) + { + this.ConditionalDependencySpecifiers.Add(conditionalDependenciesMatches[i].Value); + } + } + } + else + { + var nameMatches = PipNameExtractionRegex.Match(packageString); + var versionMatches = PipVersionExtractionRegex.Match(packageString); + + if (nameMatches.Captures.Count > 0) + { + this.Name = nameMatches.Captures[0].Value; + } + else + { + this.Name = packageString; + } + + if (versionMatches.Captures.Count > 0) + { + this.DependencySpecifiers = versionMatches.Captures[0].Value.Split(','); + } + } + + this.DependencySpecifiers = this.DependencySpecifiers.Where(x => !x.Contains("python_version")) + .Select(x => x.Trim()) + .ToList(); + } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/IPyPiClient.cs b/src/Microsoft.ComponentDetection.Detectors/pip/IPyPiClient.cs index 29b89122a..dc4829f72 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/IPyPiClient.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/IPyPiClient.cs @@ -235,6 +235,13 @@ public async Task GetProjectAsync(PipDependencySpecification spec return versions; } + public void Dispose() + { + this.cacheTelemetry.FinalCacheSize = this.cachedResponses.Count; + this.cacheTelemetry.Dispose(); + this.cachedResponses.Dispose(); + } + /// /// Returns a cached response if it exists, otherwise returns the response from PyPi REST call. /// The response from PyPi is automatically added to the cache. @@ -285,11 +292,4 @@ private void InitializeNonDefaultMemoryCache() this.checkedMaxEntriesVariable = true; } - - public void Dispose() - { - this.cacheTelemetry.FinalCacheSize = this.cachedResponses.Count; - this.cacheTelemetry.Dispose(); - this.cachedResponses.Dispose(); - } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PipCommandService.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PipCommandService.cs index 8d96944b5..2bb874921 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/PipCommandService.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/PipCommandService.cs @@ -76,50 +76,6 @@ public async Task GetPipVersionAsync(string pipPath = null, string pyth } } - private async Task<(string PipExectuable, string PythonExecutable)> ResolvePipAsync(string pipPath = null, string pythonPath = null) - { - var pipCommand = string.IsNullOrEmpty(pipPath) ? "pip" : pipPath; - var pythonCommand = string.IsNullOrEmpty(pythonPath) ? "python" : pythonPath; - - if (await this.CanCommandBeLocatedAsync(pipCommand)) - { - return (pipCommand, null); - } - else if (await this.commandLineInvocationService.CanCommandBeLocatedAsync(pythonCommand, null, "-m", "pip", "--version")) - { - return (null, pythonCommand); - } - - return (null, null); - } - - private async Task CanCommandBeLocatedAsync(string pipPath) - { - return await this.commandLineInvocationService.CanCommandBeLocatedAsync(pipPath, ["pip3"], "--version"); - } - - private async Task ExecuteCommandAsync( - string pipExecutable = null, - string pythonExecutable = null, - IEnumerable additionalCandidateCommands = null, - DirectoryInfo workingDirectory = null, - CancellationToken cancellationToken = default, - params string[] parameters) - { - if (!string.IsNullOrEmpty(pipExecutable)) - { - return await this.commandLineInvocationService.ExecuteCommandAsync( - pipExecutable, additionalCandidateCommands, workingDirectory, cancellationToken, parameters); - } - else - { - var pythonPipParams = new[] { "-m", "pip" }; - var parametersFull = pythonPipParams.Concat(parameters).ToArray(); - return await this.commandLineInvocationService.ExecuteCommandAsync( - pythonExecutable, additionalCandidateCommands, workingDirectory, cancellationToken, parametersFull); - } - } - public async Task<(PipInstallationReport Report, FileInfo ReportFile)> GenerateInstallationReportAsync( string path, string pipExePath = null, string pythonExePath = null, CancellationToken cancellationToken = default) { @@ -225,4 +181,48 @@ private async Task ExecuteCommandAsync( } } } + + private async Task<(string PipExectuable, string PythonExecutable)> ResolvePipAsync(string pipPath = null, string pythonPath = null) + { + var pipCommand = string.IsNullOrEmpty(pipPath) ? "pip" : pipPath; + var pythonCommand = string.IsNullOrEmpty(pythonPath) ? "python" : pythonPath; + + if (await this.CanCommandBeLocatedAsync(pipCommand)) + { + return (pipCommand, null); + } + else if (await this.commandLineInvocationService.CanCommandBeLocatedAsync(pythonCommand, null, "-m", "pip", "--version")) + { + return (null, pythonCommand); + } + + return (null, null); + } + + private async Task CanCommandBeLocatedAsync(string pipPath) + { + return await this.commandLineInvocationService.CanCommandBeLocatedAsync(pipPath, ["pip3"], "--version"); + } + + private async Task ExecuteCommandAsync( + string pipExecutable = null, + string pythonExecutable = null, + IEnumerable additionalCandidateCommands = null, + DirectoryInfo workingDirectory = null, + CancellationToken cancellationToken = default, + params string[] parameters) + { + if (!string.IsNullOrEmpty(pipExecutable)) + { + return await this.commandLineInvocationService.ExecuteCommandAsync( + pipExecutable, additionalCandidateCommands, workingDirectory, cancellationToken, parameters); + } + else + { + var pythonPipParams = new[] { "-m", "pip" }; + var parametersFull = pythonPipParams.Concat(parameters).ToArray(); + return await this.commandLineInvocationService.ExecuteCommandAsync( + pythonExecutable, additionalCandidateCommands, workingDirectory, cancellationToken, parametersFull); + } + } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PipReportUtilities.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PipReportUtilities.cs index 1495f752f..a3802e9fa 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/PipReportUtilities.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/PipReportUtilities.cs @@ -6,16 +6,16 @@ namespace Microsoft.ComponentDetection.Detectors.Pip; internal class PipReportUtilities { - private const int MaxLicenseFieldLength = 100; - private const string ClassifierFieldSeparator = " :: "; - private const string ClassifierFieldLicensePrefix = "License"; - // Python regular expression for version schema: // https://www.python.org/dev/peps/pep-0440/#appendix-b-parsing-version-strings-with-regular-expressions public static readonly Regex CanonicalVersionPatternMatch = new Regex( @"^([1-9]\d*!)?(0|[1-9]\d*)(\.(0|[1-9]\d*))*((a|b|rc)(0|[1-9]\d*))?(\.post(0|[1-9]\d*))?(\.dev(0|[1-9]\d*))?(\+(?:(?[a-z0-9]+(?:[.][a-z0-9]+)*))?)?$", RegexOptions.Compiled); + private const int MaxLicenseFieldLength = 100; + private const string ClassifierFieldSeparator = " :: "; + private const string ClassifierFieldLicensePrefix = "License"; + /// /// Normalize the package name format to the standard Python Packaging format. /// See https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization. diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PythonCommandService.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PythonCommandService.cs index 962003f9c..d824f1ea9 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/PythonCommandService.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/PythonCommandService.cs @@ -59,6 +59,22 @@ public async Task PythonExistsAsync(string pythonPath = null) } } + public async Task GetPythonVersionAsync(string pythonPath) + { + var pythonCommand = await this.ResolvePythonAsync(pythonPath); + var versionResult = await this.commandLineInvocationService.ExecuteCommandAsync(pythonCommand, ["python3", "python2"], "--version"); + var version = new Regex("Python ([\\d.]+)"); + var match = version.Match(versionResult.StdOut); + return match.Success ? match.Groups[1].Value : null; + } + + public async Task GetOsTypeAsync(string pythonPath) + { + var pythonCommand = await this.ResolvePythonAsync(pythonPath); + var versionResult = await this.commandLineInvocationService.ExecuteCommandAsync(pythonCommand, ["python3", "python2"], "-c", "\"import sys; print(sys.platform);\""); + return versionResult.ExitCode == 0 && string.IsNullOrEmpty(versionResult.StdErr) ? versionResult.StdOut.Trim() : null; + } + private async Task> ParseSetupPyFileAsync(string filePath, string pythonExePath = null) { var pythonExecutable = await this.ResolvePythonAsync(pythonExePath); @@ -173,20 +189,4 @@ private async Task CanCommandBeLocatedAsync(string pythonPath) { return await this.commandLineInvocationService.CanCommandBeLocatedAsync(pythonPath, ["python3", "python2"], "--version"); } - - public async Task GetPythonVersionAsync(string pythonPath) - { - var pythonCommand = await this.ResolvePythonAsync(pythonPath); - var versionResult = await this.commandLineInvocationService.ExecuteCommandAsync(pythonCommand, ["python3", "python2"], "--version"); - var version = new Regex("Python ([\\d.]+)"); - var match = version.Match(versionResult.StdOut); - return match.Success ? match.Groups[1].Value : null; - } - - public async Task GetOsTypeAsync(string pythonPath) - { - var pythonCommand = await this.ResolvePythonAsync(pythonPath); - var versionResult = await this.commandLineInvocationService.ExecuteCommandAsync(pythonCommand, ["python3", "python2"], "-c", "\"import sys; print(sys.platform);\""); - return versionResult.ExitCode == 0 && string.IsNullOrEmpty(versionResult.StdErr) ? versionResult.StdOut.Trim() : null; - } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs index 1dd7c34f2..124279cee 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs @@ -79,6 +79,32 @@ public async Task> ResolveRootsAsync(ISingleFileComponentRec return await this.ProcessQueueAsync(singleFileComponentRecorder, state) ?? []; } + public void SetPythonEnvironmentVariable(string key, string value) + { + this.pythonEnvironmentVariables[key] = value; + } + + public Dictionary GetPythonEnvironmentVariables() + { + return this.pythonEnvironmentVariables; + } + + protected override async Task> FetchPackageDependenciesAsync( + PythonResolverState state, + PipDependencySpecification spec) + { + var candidateVersion = state.NodeReferences[spec.Name].Value.Version; + + var packageToFetch = state.ValidVersionMap[spec.Name][candidateVersion].FirstOrDefault(x => string.Equals("bdist_wheel", x.PackageType, StringComparison.OrdinalIgnoreCase)) ?? + state.ValidVersionMap[spec.Name][candidateVersion].FirstOrDefault(x => string.Equals("bdist_egg", x.PackageType, StringComparison.OrdinalIgnoreCase)); + if (packageToFetch == null) + { + return []; + } + + return await this.pypiClient.FetchPackageDependenciesAsync(spec.Name, candidateVersion, packageToFetch); + } + private async Task> ProcessQueueAsync(ISingleFileComponentRecorder singleFileComponentRecorder, PythonResolverState state) { while (state.ProcessingQueue.Count > 0) @@ -154,22 +180,6 @@ private async Task> ProcessQueueAsync(ISingleFileComponentRe return state.Roots; } - protected override async Task> FetchPackageDependenciesAsync( - PythonResolverState state, - PipDependencySpecification spec) - { - var candidateVersion = state.NodeReferences[spec.Name].Value.Version; - - var packageToFetch = state.ValidVersionMap[spec.Name][candidateVersion].FirstOrDefault(x => string.Equals("bdist_wheel", x.PackageType, StringComparison.OrdinalIgnoreCase)) ?? - state.ValidVersionMap[spec.Name][candidateVersion].FirstOrDefault(x => string.Equals("bdist_egg", x.PackageType, StringComparison.OrdinalIgnoreCase)); - if (packageToFetch == null) - { - return []; - } - - return await this.pypiClient.FetchPackageDependenciesAsync(spec.Name, candidateVersion, packageToFetch); - } - private void AddGraphNode(PythonResolverState state, PipGraphNode parent, string name, string version, string license = null, string author = null) { if (state.NodeReferences.TryGetValue(name, out var value)) @@ -232,14 +242,4 @@ private string GetLicenseFromProject(PythonProject project) return null; } - - public void SetPythonEnvironmentVariable(string key, string value) - { - this.pythonEnvironmentVariables[key] = value; - } - - public Dictionary GetPythonEnvironmentVariables() - { - return this.pythonEnvironmentVariables; - } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolverBase.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolverBase.cs index fe44081c0..3ef861601 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolverBase.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolverBase.cs @@ -15,6 +15,32 @@ public abstract class PythonResolverBase internal PythonResolverBase(ILogger logger) => this.logger = logger; + /// + /// Multiple dependency specification versions can be given for a single package name. + /// Until a better method is devised, choose the latest entry. + /// See https://github.com/microsoft/component-detection/issues/963. + /// + /// Dictionary of package names to dependency version specifiers. + public Dictionary ResolveDependencySpecifications(PipComponent component, IList fetchedDependences) + { + var dependencies = new Dictionary(); + fetchedDependences.ForEach(d => + { + if (!dependencies.TryAdd(d.Name, d)) + { + this.logger.LogWarning( + "Duplicate package dependencies entry for component:{ComponentName} with dependency:{DependencyName}. Existing dependency specifiers: {ExistingSpecifiers}. New dependency specifiers: {NewSpecifiers}.", + component.Name, + d.Name, + JsonConvert.SerializeObject(dependencies[d.Name].DependencySpecifiers), + JsonConvert.SerializeObject(d.DependencySpecifiers)); + dependencies[d.Name] = d; + } + }); + + return dependencies; + } + /// /// Given a state, node, and new spec, will reprocess a new valid version for the node. /// @@ -76,32 +102,6 @@ protected async Task InvalidateAndReprocessAsync( return true; } - /// - /// Multiple dependency specification versions can be given for a single package name. - /// Until a better method is devised, choose the latest entry. - /// See https://github.com/microsoft/component-detection/issues/963. - /// - /// Dictionary of package names to dependency version specifiers. - public Dictionary ResolveDependencySpecifications(PipComponent component, IList fetchedDependences) - { - var dependencies = new Dictionary(); - fetchedDependences.ForEach(d => - { - if (!dependencies.TryAdd(d.Name, d)) - { - this.logger.LogWarning( - "Duplicate package dependencies entry for component:{ComponentName} with dependency:{DependencyName}. Existing dependency specifiers: {ExistingSpecifiers}. New dependency specifiers: {NewSpecifiers}.", - component.Name, - d.Name, - JsonConvert.SerializeObject(dependencies[d.Name].DependencySpecifiers), - JsonConvert.SerializeObject(d.DependencySpecifiers)); - dependencies[d.Name] = d; - } - }); - - return dependencies; - } - protected abstract Task> FetchPackageDependenciesAsync( PythonResolverState state, PipDependencySpecification spec); diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PythonVersionUtilities.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PythonVersionUtilities.cs index ede50da9b..d933113d7 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/PythonVersionUtilities.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/PythonVersionUtilities.cs @@ -84,6 +84,23 @@ public static bool CheckEquality(string version, string specVer, bool fuzzy = fa } } + public static (string Operator, string Version) ParseSpec(string spec) + { + var opChars = new char[] { '=', '<', '>', '~', '!' }; + var specArray = spec.ToCharArray(); + + var i = 0; + while (i < spec.Length && i < 3 && opChars.Contains(specArray[i])) + { + i++; + } + + var op = spec[..i]; + var specVerSection = spec[i..].Trim(); + + return (op, specVerSection); + } + private static bool VersionValidForSpec(string version, string spec) { (var op, var specVersion) = ParseSpec(spec); @@ -114,21 +131,4 @@ private static bool VersionValidForSpec(string version, string spec) _ => false, }; } - - public static (string Operator, string Version) ParseSpec(string spec) - { - var opChars = new char[] { '=', '<', '>', '~', '!' }; - var specArray = spec.ToCharArray(); - - var i = 0; - while (i < spec.Length && i < 3 && opChars.Contains(specArray[i])) - { - i++; - } - - var op = spec[..i]; - var specVerSection = spec[i..].Trim(); - - return (op, specVerSection); - } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/SimplePypiClient.cs b/src/Microsoft.ComponentDetection.Detectors/pip/SimplePypiClient.cs index 52be242f9..9407d9df9 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/SimplePypiClient.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/SimplePypiClient.cs @@ -18,13 +18,13 @@ namespace Microsoft.ComponentDetection.Detectors.Pip; public sealed class SimplePyPiClient : ISimplePyPiClient, IDisposable { + // max number of retries allowed, to cap the total delay period + public const long MAXRETRIES = 15; + // Values used for cache creation private const long CACHEINTERVALSECONDS = 180; private const long DEFAULTCACHEENTRIES = 4096; - // max number of retries allowed, to cap the total delay period - public const long MAXRETRIES = 15; - private static readonly ProductInfoHeaderValue ProductValue = new( "ComponentDetection", Assembly.GetEntryAssembly().GetCustomAttribute()?.InformationalVersion); @@ -85,6 +85,16 @@ public async Task FetchPackageFileStreamAsync(Uri releaseUrl) return projectStream; } + public void Dispose() + { + this.cacheTelemetry.FinalSimpleProjectCacheSize = this.cachedSimplePyPiProjects.Count; + this.cacheTelemetry.FinalProjectFileCacheSize = this.cachedProjectWheelFiles.Count; + this.cacheTelemetry.Dispose(); + this.cachedProjectWheelFiles.Dispose(); + this.cachedSimplePyPiProjects.Dispose(); + HttpClient.Dispose(); + } + /// /// Returns a cached response if it exists, otherwise returns the response from PyPi REST call. /// The response from PyPi is automatically added to the cache. @@ -274,14 +284,4 @@ private async Task GetPypiResponseAsync(Uri uri) var response = await HttpClient.SendAsync(request); return response; } - - public void Dispose() - { - this.cacheTelemetry.FinalSimpleProjectCacheSize = this.cachedSimplePyPiProjects.Count; - this.cacheTelemetry.FinalProjectFileCacheSize = this.cachedProjectWheelFiles.Count; - this.cacheTelemetry.Dispose(); - this.cachedProjectWheelFiles.Dispose(); - this.cachedSimplePyPiProjects.Dispose(); - HttpClient.Dispose(); - } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/SimplePythonResolver.cs b/src/Microsoft.ComponentDetection.Detectors/pip/SimplePythonResolver.cs index 6a6f6c5ed..e60e6a5e6 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/SimplePythonResolver.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/SimplePythonResolver.cs @@ -33,60 +33,6 @@ public SimplePythonResolver(ISimplePyPiClient simplePypiClient, ILogger - /// Uses regex to extract the version from the file name. - /// - /// the name of the file from simple pypi. - /// returns a string representing the release version. - private static string GetVersionFromFileName(string fileName) - { - var version = VersionRegex.Match(fileName).Groups[1]; - return version.Value; - } - - /// - /// Returns the package type based on the file name. - /// - /// the name of the file from simple pypi. - /// a string representing the package type. - private static string GetPackageType(string fileName) - { - if (fileName.EndsWith(".whl")) - { - return "bdist_wheel"; - } - - if (fileName.EndsWith(".tar.gz")) - { - return "sdist"; - } - - return fileName.EndsWith(".egg") ? "bdist_egg" : string.Empty; - } - - /// - /// Adds a node to the graph. - /// - /// The PythonResolverState. - /// The parent node. - /// The package name. - /// The package version. - private static void AddGraphNode(PythonResolverState state, PipGraphNode parent, string name, string version) - { - if (state.NodeReferences.TryGetValue(name, out var value)) - { - parent.Children.Add(value); - value.Parents.Add(parent); - } - else - { - var node = new PipGraphNode(new PipComponent(name, version)); - state.NodeReferences[name] = node; - parent.Children.Add(node); - node.Parents.Add(parent); - } - } - /// public async Task> ResolveRootsAsync(ISingleFileComponentRecorder singleFileComponentRecorder, IList initialPackages) { @@ -151,6 +97,89 @@ await Parallel.ForEachAsync(initialPackages, async (rootPackage, ct) => return await this.ProcessQueueAsync(singleFileComponentRecorder, state) ?? []; } + /// + /// Fetches the dependencies for a package. + /// + /// The PythonResolverState. + /// The PipDependencySpecification. + /// Returns a list of PipDependencySpecification. + protected override async Task> FetchPackageDependenciesAsync( + PythonResolverState state, + PipDependencySpecification spec) + { + var candidateVersion = state.NodeReferences[spec.Name].Value.Version; + + var packageToFetch = state.ValidVersionMap[spec.Name][candidateVersion].FirstOrDefault(x => string.Equals("bdist_wheel", x.PackageType, StringComparison.OrdinalIgnoreCase)) ?? + state.ValidVersionMap[spec.Name][candidateVersion].FirstOrDefault(x => string.Equals("bdist_egg", x.PackageType, StringComparison.OrdinalIgnoreCase)); + if (packageToFetch == null) + { + return []; + } + + var packageFileStream = await this.simplePypiClient.FetchPackageFileStreamAsync(packageToFetch.Url); + + if (packageFileStream.Length == 0) + { + return []; + } + + return await this.FetchDependenciesFromPackageStreamAsync(spec.Name, candidateVersion, packageFileStream); + } + + /// + /// Uses regex to extract the version from the file name. + /// + /// the name of the file from simple pypi. + /// returns a string representing the release version. + private static string GetVersionFromFileName(string fileName) + { + var version = VersionRegex.Match(fileName).Groups[1]; + return version.Value; + } + + /// + /// Returns the package type based on the file name. + /// + /// the name of the file from simple pypi. + /// a string representing the package type. + private static string GetPackageType(string fileName) + { + if (fileName.EndsWith(".whl")) + { + return "bdist_wheel"; + } + + if (fileName.EndsWith(".tar.gz")) + { + return "sdist"; + } + + return fileName.EndsWith(".egg") ? "bdist_egg" : string.Empty; + } + + /// + /// Adds a node to the graph. + /// + /// The PythonResolverState. + /// The parent node. + /// The package name. + /// The package version. + private static void AddGraphNode(PythonResolverState state, PipGraphNode parent, string name, string version) + { + if (state.NodeReferences.TryGetValue(name, out var value)) + { + parent.Children.Add(value); + value.Parents.Add(parent); + } + else + { + var node = new PipGraphNode(new PipComponent(name, version)); + state.NodeReferences[name] = node; + parent.Children.Add(node); + node.Parents.Add(parent); + } + } + private async Task> ProcessQueueAsync(ISingleFileComponentRecorder singleFileComponentRecorder, PythonResolverState state) { while (state.ProcessingQueue.Count > 0) @@ -280,35 +309,6 @@ private SortedDictionary> ConvertSimplePypiP return sortedProjectVersions; } - /// - /// Fetches the dependencies for a package. - /// - /// The PythonResolverState. - /// The PipDependencySpecification. - /// Returns a list of PipDependencySpecification. - protected override async Task> FetchPackageDependenciesAsync( - PythonResolverState state, - PipDependencySpecification spec) - { - var candidateVersion = state.NodeReferences[spec.Name].Value.Version; - - var packageToFetch = state.ValidVersionMap[spec.Name][candidateVersion].FirstOrDefault(x => string.Equals("bdist_wheel", x.PackageType, StringComparison.OrdinalIgnoreCase)) ?? - state.ValidVersionMap[spec.Name][candidateVersion].FirstOrDefault(x => string.Equals("bdist_egg", x.PackageType, StringComparison.OrdinalIgnoreCase)); - if (packageToFetch == null) - { - return []; - } - - var packageFileStream = await this.simplePypiClient.FetchPackageFileStreamAsync(packageToFetch.Url); - - if (packageFileStream.Length == 0) - { - return []; - } - - return await this.FetchDependenciesFromPackageStreamAsync(spec.Name, candidateVersion, packageFileStream); - } - /// /// Given a package stream will unzip and return the dependencies in the metadata file. /// diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/Parsers/RustCargoLockParser.cs b/src/Microsoft.ComponentDetection.Detectors/rust/Parsers/RustCargoLockParser.cs index a7efc2270..a2a1407af 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/Parsers/RustCargoLockParser.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/Parsers/RustCargoLockParser.cs @@ -38,8 +38,6 @@ public class RustCargoLockParser : IRustCargoLockParser /// The logger. public RustCargoLockParser(ILogger logger) => this.logger = logger; - private static bool IsLocalPackage(CargoPackage package) => package.Source == null; - /// /// Parses a Cargo.lock file and records components. /// @@ -67,6 +65,8 @@ public class RustCargoLockParser : IRustCargoLockParser } } + private static bool IsLocalPackage(CargoPackage package) => package.Source == null; + private void ProcessCargoLock(CargoLock cargoLock, ISingleFileComponentRecorder singleFileComponentRecorder, IComponentStream cargoLockFile) { try diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/Parsers/RustSbomParser.cs b/src/Microsoft.ComponentDetection.Detectors/rust/Parsers/RustSbomParser.cs index b377ef664..5868d7852 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/Parsers/RustSbomParser.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/Parsers/RustSbomParser.cs @@ -61,6 +61,43 @@ public class RustSbomParser : IRustSbomParser } } + /// + /// Parses a Cargo SBOM file and registers each discovered component against all owning Cargo.toml recorders + /// using the provided ownership map (cargo metadata package id -> set of manifest paths). + /// Falls back to the supplied sbomRecorder when ownership info is absent. + /// + /// SBOM stream. + /// Recorder tied to the SBOM file (fallback target). + /// Root component recorder used to create (or reuse) per-manifest recorders. + /// Package ownership map from RustMetadataContextBuilder (may be null). + /// Cancellation token. + /// SBOM version or null on failure. + public async Task ParseWithOwnershipAsync( + IComponentStream componentStream, + ISingleFileComponentRecorder sbomRecorder, + IComponentRecorder parentComponentRecorder, + IReadOnlyDictionary> ownershipMap, + CancellationToken cancellationToken = default) + { + try + { + using var reader = new StreamReader(componentStream.Stream); + var cargoSbom = CargoSbom.FromJson(await reader.ReadToEndAsync(cancellationToken)); + this.ProcessCargoSbomWithOwnership( + cargoSbom, + componentStream, + sbomRecorder, + parentComponentRecorder, + ownershipMap); + return cargoSbom.Version; + } + catch (Exception e) + { + this.logger.LogError(e, "Failed to parse Cargo SBOM (ownership mode) '{FileLocation}'", componentStream.Location); + return null; + } + } + private void ProcessCargoSbom(CargoSbom sbom, ISingleFileComponentRecorder recorder, IComponentStream components) { try @@ -113,43 +150,6 @@ private void ProcessDependency( } } - /// - /// Parses a Cargo SBOM file and registers each discovered component against all owning Cargo.toml recorders - /// using the provided ownership map (cargo metadata package id -> set of manifest paths). - /// Falls back to the supplied sbomRecorder when ownership info is absent. - /// - /// SBOM stream. - /// Recorder tied to the SBOM file (fallback target). - /// Root component recorder used to create (or reuse) per-manifest recorders. - /// Package ownership map from RustMetadataContextBuilder (may be null). - /// Cancellation token. - /// SBOM version or null on failure. - public async Task ParseWithOwnershipAsync( - IComponentStream componentStream, - ISingleFileComponentRecorder sbomRecorder, - IComponentRecorder parentComponentRecorder, - IReadOnlyDictionary> ownershipMap, - CancellationToken cancellationToken = default) - { - try - { - using var reader = new StreamReader(componentStream.Stream); - var cargoSbom = CargoSbom.FromJson(await reader.ReadToEndAsync(cancellationToken)); - this.ProcessCargoSbomWithOwnership( - cargoSbom, - componentStream, - sbomRecorder, - parentComponentRecorder, - ownershipMap); - return cargoSbom.Version; - } - catch (Exception e) - { - this.logger.LogError(e, "Failed to parse Cargo SBOM (ownership mode) '{FileLocation}'", componentStream.Location); - return null; - } - } - private void ProcessCargoSbomWithOwnership( CargoSbom sbom, IComponentStream sbomStream, diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs index 10561cd28..b75d8f802 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs @@ -17,8 +17,8 @@ public static class DetectorExperiments /// public static bool Enable { get; set; } + internal static bool AreExperimentsEnabled => Enable || EnvironmentEnabled; + private static bool EnvironmentEnabled => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("CD_DETECTOR_EXPERIMENTS")); - - internal static bool AreExperimentsEnabled => Enable || EnvironmentEnabled; } diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs index 4468199f0..b8b2c0dde 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs @@ -125,19 +125,6 @@ public void RecordDetectorRun( } } - private void FilterExperiments(IComponentDetector detector, int count) - { - var experimentsToRemove = this.experiments - .Where(x => !x.Key.ShouldRecord(detector, count)) - .Select(x => x.Key) - .ToList(); - - foreach (var config in experimentsToRemove.Where(config => this.experiments.TryRemove(config, out _))) - { - this.logger.LogDebug("Removing {Experiment} from active experiments", config.Name); - } - } - public void RemoveUnwantedExperimentsbyDetectors(IEnumerable detectors) { if (detectors == null) @@ -201,4 +188,17 @@ public async Task FinishAsync() } } } + + private void FilterExperiments(IComponentDetector detector, int count) + { + var experimentsToRemove = this.experiments + .Where(x => !x.Key.ShouldRecord(detector, count)) + .Select(x => x.Key) + .ToList(); + + foreach (var config in experimentsToRemove.Where(config => this.experiments.TryRemove(config, out _))) + { + this.logger.LogDebug("Removing {Experiment} from active experiments", config.Name); + } + } } diff --git a/src/Microsoft.ComponentDetection.Orchestrator/LoggingEnricher.cs b/src/Microsoft.ComponentDetection.Orchestrator/LoggingEnricher.cs index 3b1d153e0..2f8a8e17c 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/LoggingEnricher.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/LoggingEnricher.cs @@ -13,13 +13,15 @@ public class LoggingEnricher : ILogEventEnricher /// The name of the log file path property. /// public const string LogFilePathPropertyName = "LogFilePath"; - private string cachedLogFilePath; - private LogEventProperty cachedLogFilePathProperty; /// /// The name of the print stderr property. /// public const string PrintStderrPropertyName = "PrintStderr"; + + private string cachedLogFilePath; + private LogEventProperty cachedLogFilePathProperty; + private bool? cachedPrintStderr; private LogEventProperty cachedPrintStderrProperty; diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs index d4941512a..a704e59e9 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs @@ -73,106 +73,6 @@ public DotNetComponentDetectorTests() .Returns((string c, IEnumerable ac, DirectoryInfo d, CancellationToken ct, string[] args) => Task.FromResult(this.CommandResult(c, d))); } - private bool FileExists(string path) - { - var fileName = Path.GetFileName(path); - var directory = Path.GetDirectoryName(path); - - return this.files.TryGetValue(directory, out var fileNames) && - fileNames.TryGetValue(fileName, out _); - } - - private Stream OpenFile(string path) - { - var fileName = Path.GetFileName(path); - var directory = Path.GetDirectoryName(path); - - return this.files.TryGetValue(directory, out var fileNames) && - fileNames.TryGetValue(fileName, out var stream) ? stream : null; - } - - private bool DirectoryExists(string directory) => this.files.ContainsKey(directory); - - private IEnumerable EnumerateFilesRecursive(string directory, string pattern) - { - if (this.files.TryGetValue(directory, out var fileNames)) - { - // a basic approximation of globbing - var patternRegex = new Regex(pattern.Replace(".", "\\.").Replace("*", ".*")); - - foreach (var fileName in fileNames.Keys) - { - var filePath = Path.Combine(directory, fileName); - - if (fileName.EndsWith(Path.DirectorySeparatorChar)) - { - foreach (var subFile in this.EnumerateFilesRecursive(Path.TrimEndingDirectorySeparator(filePath), pattern)) - { - yield return subFile; - } - } - else - { - if (patternRegex.IsMatch(fileName)) - { - yield return filePath; - } - } - } - } - } - - private void AddFile(string path, Stream content) - { - var fileName = Path.GetFileName(path); - var directory = Path.GetDirectoryName(path); - this.AddDirectory(directory); - this.files[directory][fileName] = content; - } - - private void AddDirectory(string path, string subDirectory = null) - { - if (string.IsNullOrEmpty(path)) - { - return; - } - - if (subDirectory is not null) - { - // use a trailing slash to indicate a sub directory in the files collection - subDirectory += Path.DirectorySeparatorChar; - } - - if (this.files.TryGetValue(path, out var directoryFiles)) - { - if (subDirectory is not null) - { - directoryFiles.Add(subDirectory, null); - } - } - else - { - this.files.Add(path, subDirectory is null ? [] : new() { { subDirectory, null } }); - this.AddDirectory(Path.GetDirectoryName(path), Path.GetFileName(path)); - } - } - - private void SetCommandResult(int exitCode, string stdOut = null, string stdErr = null) - { - this.commandLineCallback = null; - this.commandLineExecutionResult.ExitCode = exitCode; - this.commandLineExecutionResult.StdOut = stdOut; - this.commandLineExecutionResult.StdErr = stdErr; - } - - private void SetCommandResult(Func callback) - { - this.commandLineCallback = callback; - } - - private CommandLineExecutionResult CommandResult(string command, DirectoryInfo directory) => - (this.commandLineCallback != null) ? this.commandLineCallback(command, directory) : this.commandLineExecutionResult; - [TestCleanup] public void ClearMocks() { @@ -180,63 +80,6 @@ public void ClearMocks() this.SetCommandResult(-1); } - private static string ProjectAssets(string projectName, string outputPath, string projectPath, params string[] targetFrameworks) - { - LockFileFormat format = new(); - LockFile lockFile = new(); - using var textWriter = new StringWriter(); - - // assets file always includes a trailing separator - if (!Path.EndsInDirectorySeparator(outputPath)) - { - outputPath += Path.DirectorySeparatorChar; - } - - lockFile.Targets = targetFrameworks.Select(tfm => new LockFileTarget() { TargetFramework = NuGetFramework.Parse(tfm) }).ToList(); - lockFile.PackageSpec = new() - { - RestoreMetadata = new() - { - ProjectName = projectName, - OutputPath = outputPath, - ProjectPath = projectPath, - }, - }; - - format.Write(textWriter, lockFile); - return textWriter.ToString(); - } - - private static Stream GlobalJson(string sdkVersion) - { - var stream = new MemoryStream(); - using (var writer = new Utf8JsonWriter(stream, new() { Indented = true })) - { - writer.WriteStartObject(); - writer.WritePropertyName("sdk"); - writer.WriteStartObject(); - writer.WriteString("version", sdkVersion); - writer.WriteEndObject(); - writer.WriteEndObject(); - } - - stream.Position = 0; - return stream; - } - - private static Stream StreamFromString(string content) - { - var stream = new MemoryStream(); - using (var writer = new StreamWriter(stream, leaveOpen: true)) - { - writer.Write(content); - writer.Flush(); - stream.Position = 0; - } - - return stream; - } - [TestMethod] public async Task TestDotNetDetectorWithNoFiles_ReturnsSuccessfullyAsync() { @@ -745,4 +588,161 @@ public async Task TestDotNetDetectorRebasePaths(string additionalPathSegment) discoveredComponents.Where(component => component.Component.Id == "4.5.6 net6.0 library - DotNet").Should().ContainSingle(); discoveredComponents.Where(component => component.Component.Id == "4.5.6 netstandard2.0 library - DotNet").Should().ContainSingle(); } + + private static string ProjectAssets(string projectName, string outputPath, string projectPath, params string[] targetFrameworks) + { + LockFileFormat format = new(); + LockFile lockFile = new(); + using var textWriter = new StringWriter(); + + // assets file always includes a trailing separator + if (!Path.EndsInDirectorySeparator(outputPath)) + { + outputPath += Path.DirectorySeparatorChar; + } + + lockFile.Targets = targetFrameworks.Select(tfm => new LockFileTarget() { TargetFramework = NuGetFramework.Parse(tfm) }).ToList(); + lockFile.PackageSpec = new() + { + RestoreMetadata = new() + { + ProjectName = projectName, + OutputPath = outputPath, + ProjectPath = projectPath, + }, + }; + + format.Write(textWriter, lockFile); + return textWriter.ToString(); + } + + private static Stream GlobalJson(string sdkVersion) + { + var stream = new MemoryStream(); + using (var writer = new Utf8JsonWriter(stream, new() { Indented = true })) + { + writer.WriteStartObject(); + writer.WritePropertyName("sdk"); + writer.WriteStartObject(); + writer.WriteString("version", sdkVersion); + writer.WriteEndObject(); + writer.WriteEndObject(); + } + + stream.Position = 0; + return stream; + } + + private static Stream StreamFromString(string content) + { + var stream = new MemoryStream(); + using (var writer = new StreamWriter(stream, leaveOpen: true)) + { + writer.Write(content); + writer.Flush(); + stream.Position = 0; + } + + return stream; + } + + private bool FileExists(string path) + { + var fileName = Path.GetFileName(path); + var directory = Path.GetDirectoryName(path); + + return this.files.TryGetValue(directory, out var fileNames) && + fileNames.TryGetValue(fileName, out _); + } + + private Stream OpenFile(string path) + { + var fileName = Path.GetFileName(path); + var directory = Path.GetDirectoryName(path); + + return this.files.TryGetValue(directory, out var fileNames) && + fileNames.TryGetValue(fileName, out var stream) ? stream : null; + } + + private bool DirectoryExists(string directory) => this.files.ContainsKey(directory); + + private IEnumerable EnumerateFilesRecursive(string directory, string pattern) + { + if (this.files.TryGetValue(directory, out var fileNames)) + { + // a basic approximation of globbing + var patternRegex = new Regex(pattern.Replace(".", "\\.").Replace("*", ".*")); + + foreach (var fileName in fileNames.Keys) + { + var filePath = Path.Combine(directory, fileName); + + if (fileName.EndsWith(Path.DirectorySeparatorChar)) + { + foreach (var subFile in this.EnumerateFilesRecursive(Path.TrimEndingDirectorySeparator(filePath), pattern)) + { + yield return subFile; + } + } + else + { + if (patternRegex.IsMatch(fileName)) + { + yield return filePath; + } + } + } + } + } + + private void AddFile(string path, Stream content) + { + var fileName = Path.GetFileName(path); + var directory = Path.GetDirectoryName(path); + this.AddDirectory(directory); + this.files[directory][fileName] = content; + } + + private void AddDirectory(string path, string subDirectory = null) + { + if (string.IsNullOrEmpty(path)) + { + return; + } + + if (subDirectory is not null) + { + // use a trailing slash to indicate a sub directory in the files collection + subDirectory += Path.DirectorySeparatorChar; + } + + if (this.files.TryGetValue(path, out var directoryFiles)) + { + if (subDirectory is not null) + { + directoryFiles.Add(subDirectory, null); + } + } + else + { + this.files.Add(path, subDirectory is null ? [] : new() { { subDirectory, null } }); + this.AddDirectory(Path.GetDirectoryName(path), Path.GetFileName(path)); + } + } + + private void SetCommandResult(int exitCode, string stdOut = null, string stdErr = null) + { + this.commandLineCallback = null; + this.commandLineExecutionResult.ExitCode = exitCode; + this.commandLineExecutionResult.StdOut = stdOut; + this.commandLineExecutionResult.StdErr = stdErr; + } + + private void SetCommandResult(Func callback) + { + this.commandLineCallback = callback; + } + + private CommandLineExecutionResult CommandResult(string command, DirectoryInfo directory) => + (this.commandLineCallback != null) ? this.commandLineCallback(command, directory) : this.commandLineExecutionResult; } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/GoComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/GoComponentDetectorTests.cs index 97d06c3e3..75e6b98a1 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/GoComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/GoComponentDetectorTests.cs @@ -53,31 +53,6 @@ public GoComponentDetectorTests() this.DetectorTestUtility.AddServiceMock(this.mockParserFactory); } - private void SetupMockGoModParser() - { - this.mockParserFactory.Setup(f => f.CreateParser(GoParserType.GoMod, It.IsAny())).Returns(this.mockGoModParser.Object); - } - - private void SetupMockGoSumParser() - { - this.mockParserFactory.Setup(f => f.CreateParser(GoParserType.GoSum, It.IsAny())).Returns(this.mockGoSumParser.Object); - } - - private void SetupMockGoCLIParser() - { - this.mockParserFactory.Setup(f => f.CreateParser(GoParserType.GoCLI, It.IsAny())).Returns(this.mockGoCliParser.Object); - } - - private void SetupActualGoModParser() - { - this.mockParserFactory.Setup(f => f.CreateParser(GoParserType.GoMod, It.IsAny())).Returns(new GoModParser(this.mockLogger.Object)); - } - - private void SetupActualGoSumParser() - { - this.mockParserFactory.Setup(f => f.CreateParser(GoParserType.GoSum, It.IsAny())).Returns(new GoSumParser(this.mockLogger.Object)); - } - [TestMethod] public async Task TestGoModDetectorWithValidFile_ReturnsSuccessfullyAsync() { @@ -1208,4 +1183,29 @@ public async Task GoDetector_GoSum_VerifyNestedRootsAreNotSkippedIfParentParseFa Path.Combine(root, "a", "a", "go.mod"), Path.Combine(root, "a", "b", "go.mod")); } + + private void SetupMockGoModParser() + { + this.mockParserFactory.Setup(f => f.CreateParser(GoParserType.GoMod, It.IsAny())).Returns(this.mockGoModParser.Object); + } + + private void SetupMockGoSumParser() + { + this.mockParserFactory.Setup(f => f.CreateParser(GoParserType.GoSum, It.IsAny())).Returns(this.mockGoSumParser.Object); + } + + private void SetupMockGoCLIParser() + { + this.mockParserFactory.Setup(f => f.CreateParser(GoParserType.GoCLI, It.IsAny())).Returns(this.mockGoCliParser.Object); + } + + private void SetupActualGoModParser() + { + this.mockParserFactory.Setup(f => f.CreateParser(GoParserType.GoMod, It.IsAny())).Returns(new GoModParser(this.mockLogger.Object)); + } + + private void SetupActualGoSumParser() + { + this.mockParserFactory.Setup(f => f.CreateParser(GoParserType.GoSum, It.IsAny())).Returns(new GoSumParser(this.mockLogger.Object)); + } } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/PipCommandServiceTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/PipCommandServiceTests.cs index a8d173801..b13341ab2 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/PipCommandServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/PipCommandServiceTests.cs @@ -359,29 +359,6 @@ public async Task PythonPipCommandService_GeneratesReport_RequirementsTxt_Correc this.commandLineInvokationService.Verify(); } - private static void ValidateRequirementsTxtReportFile(PipInstallationReport report, FileInfo reportFile) - { - // the file shouldn't exist since we're not writing to it in the test - reportFile.Should().NotBeNull(); - reportFile.Exists.Should().Be(false); - - // validate report parameters - report.Should().NotBeNull(); - report.Version.Should().Be("1"); - report.InstallItems.Should().NotBeNull(); - report.InstallItems.Should().ContainSingle(); - - // validate packages - report.InstallItems[0].Requested.Should().BeTrue(); - report.InstallItems[0].Metadata.Name.Should().Be("six"); - report.InstallItems[0].Metadata.Version.Should().Be("1.16.0"); - report.InstallItems[0].Metadata.License.Should().Be("MIT"); - report.InstallItems[0].Metadata.Author.Should().Be("Benjamin Peterson"); - report.InstallItems[0].Metadata.AuthorEmail.Should().Be("benjamin@python.org"); - report.InstallItems[0].Metadata.Maintainer.Should().BeNullOrEmpty(); - report.InstallItems[0].Metadata.MaintainerEmail.Should().BeNullOrEmpty(); - } - [TestMethod] public async Task PipCommandService_GeneratesReport_SetupPy_CorrectlyAsync() { @@ -703,4 +680,27 @@ public async Task PipCommandService_CancelledAsync() var action = async () => await service.GenerateInstallationReportAsync(testPath, cancellationToken: cts.Token); await action.Should().ThrowAsync().WithMessage("PipReport: Cancelled*"); } + + private static void ValidateRequirementsTxtReportFile(PipInstallationReport report, FileInfo reportFile) + { + // the file shouldn't exist since we're not writing to it in the test + reportFile.Should().NotBeNull(); + reportFile.Exists.Should().Be(false); + + // validate report parameters + report.Should().NotBeNull(); + report.Version.Should().Be("1"); + report.InstallItems.Should().NotBeNull(); + report.InstallItems.Should().ContainSingle(); + + // validate packages + report.InstallItems[0].Requested.Should().BeTrue(); + report.InstallItems[0].Metadata.Name.Should().Be("six"); + report.InstallItems[0].Metadata.Version.Should().Be("1.16.0"); + report.InstallItems[0].Metadata.License.Should().Be("MIT"); + report.InstallItems[0].Metadata.Author.Should().Be("Benjamin Peterson"); + report.InstallItems[0].Metadata.AuthorEmail.Should().Be("benjamin@python.org"); + report.InstallItems[0].Metadata.Maintainer.Should().BeNullOrEmpty(); + report.InstallItems[0].Metadata.MaintainerEmail.Should().BeNullOrEmpty(); + } } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/PipDependencySpecifierTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/PipDependencySpecifierTests.cs index 29faeef0b..3b6e37b18 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/PipDependencySpecifierTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/PipDependencySpecifierTests.cs @@ -9,59 +9,6 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; [TestClass] public class PipDependencySpecifierTests { - private static void VerifyPipDependencyParsing( - List<(string SpecString, PipDependencySpecification ReferenceDependencySpecification)> testCases, - bool requiresDist = false) - { - foreach (var (specString, referenceDependencySpecification) in testCases) - { - var dependencySpecifier = new PipDependencySpecification(specString, requiresDist); - - dependencySpecifier.Name.Should().Be(referenceDependencySpecification.Name); - dependencySpecifier.DependencySpecifiers.Should().HaveCount(referenceDependencySpecification.DependencySpecifiers.Count); - for (var i = 0; i < referenceDependencySpecification.DependencySpecifiers.Count; i++) - { - dependencySpecifier.DependencySpecifiers.Should().HaveElementAt( - i, referenceDependencySpecification.DependencySpecifiers[i]); - } - - dependencySpecifier.ConditionalDependencySpecifiers.Should().HaveCount(referenceDependencySpecification.ConditionalDependencySpecifiers.Count); - for (var i = 0; i < referenceDependencySpecification.ConditionalDependencySpecifiers.Count; i++) - { - dependencySpecifier.ConditionalDependencySpecifiers.Should().HaveElementAt( - i, referenceDependencySpecification.ConditionalDependencySpecifiers[i]); - } - } - } - - private static void VerifyPipConditionalDependencyParsing( - List<(string SpecString, bool ShouldBeIncluded, PipDependencySpecification ReferenceDependencySpecification)> testCases, - Dictionary pythonEnvironmentVariables, - bool requiresDist = false) - { - foreach (var (specString, shouldBeIncluded, referenceDependencySpecification) in testCases) - { - var dependencySpecifier = new PipDependencySpecification(specString, requiresDist); - - dependencySpecifier.Name.Should().Be(referenceDependencySpecification.Name); - dependencySpecifier.DependencySpecifiers.Should().HaveCount(referenceDependencySpecification.DependencySpecifiers.Count); - for (var i = 0; i < referenceDependencySpecification.DependencySpecifiers.Count; i++) - { - dependencySpecifier.DependencySpecifiers.Should().HaveElementAt( - i, referenceDependencySpecification.DependencySpecifiers[i]); - } - - dependencySpecifier.ConditionalDependencySpecifiers.Should().HaveCount(referenceDependencySpecification.ConditionalDependencySpecifiers.Count); - for (var i = 0; i < referenceDependencySpecification.ConditionalDependencySpecifiers.Count; i++) - { - dependencySpecifier.ConditionalDependencySpecifiers.Should().HaveElementAt( - i, referenceDependencySpecification.ConditionalDependencySpecifiers[i]); - } - - dependencySpecifier.PackageConditionsMet(pythonEnvironmentVariables).Should().Be(shouldBeIncluded, string.Join(',', dependencySpecifier.ConditionalDependencySpecifiers)); - } - } - [TestMethod] public void TestPipDependencySpecifierConstruction() { @@ -201,4 +148,57 @@ public void TestPipDependencyGetHighestExplicitPackageVersion_AllInvalidSpec() var highestVersion = spec.GetHighestExplicitPackageVersion(); highestVersion.Should().BeNull(); } + + private static void VerifyPipDependencyParsing( + List<(string SpecString, PipDependencySpecification ReferenceDependencySpecification)> testCases, + bool requiresDist = false) + { + foreach (var (specString, referenceDependencySpecification) in testCases) + { + var dependencySpecifier = new PipDependencySpecification(specString, requiresDist); + + dependencySpecifier.Name.Should().Be(referenceDependencySpecification.Name); + dependencySpecifier.DependencySpecifiers.Should().HaveCount(referenceDependencySpecification.DependencySpecifiers.Count); + for (var i = 0; i < referenceDependencySpecification.DependencySpecifiers.Count; i++) + { + dependencySpecifier.DependencySpecifiers.Should().HaveElementAt( + i, referenceDependencySpecification.DependencySpecifiers[i]); + } + + dependencySpecifier.ConditionalDependencySpecifiers.Should().HaveCount(referenceDependencySpecification.ConditionalDependencySpecifiers.Count); + for (var i = 0; i < referenceDependencySpecification.ConditionalDependencySpecifiers.Count; i++) + { + dependencySpecifier.ConditionalDependencySpecifiers.Should().HaveElementAt( + i, referenceDependencySpecification.ConditionalDependencySpecifiers[i]); + } + } + } + + private static void VerifyPipConditionalDependencyParsing( + List<(string SpecString, bool ShouldBeIncluded, PipDependencySpecification ReferenceDependencySpecification)> testCases, + Dictionary pythonEnvironmentVariables, + bool requiresDist = false) + { + foreach (var (specString, shouldBeIncluded, referenceDependencySpecification) in testCases) + { + var dependencySpecifier = new PipDependencySpecification(specString, requiresDist); + + dependencySpecifier.Name.Should().Be(referenceDependencySpecification.Name); + dependencySpecifier.DependencySpecifiers.Should().HaveCount(referenceDependencySpecification.DependencySpecifiers.Count); + for (var i = 0; i < referenceDependencySpecification.DependencySpecifiers.Count; i++) + { + dependencySpecifier.DependencySpecifiers.Should().HaveElementAt( + i, referenceDependencySpecification.DependencySpecifiers[i]); + } + + dependencySpecifier.ConditionalDependencySpecifiers.Should().HaveCount(referenceDependencySpecification.ConditionalDependencySpecifiers.Count); + for (var i = 0; i < referenceDependencySpecification.ConditionalDependencySpecifiers.Count; i++) + { + dependencySpecifier.ConditionalDependencySpecifiers.Should().HaveElementAt( + i, referenceDependencySpecification.ConditionalDependencySpecifiers[i]); + } + + dependencySpecifier.PackageConditionsMet(pythonEnvironmentVariables).Should().Be(shouldBeIncluded, string.Join(',', dependencySpecifier.ConditionalDependencySpecifiers)); + } + } } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/RustCargoLockParserTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/RustCargoLockParserTests.cs index 159e9dcab..c37919188 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/RustCargoLockParserTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/RustCargoLockParserTests.cs @@ -31,39 +31,6 @@ public void Init() this.parser = new RustCargoLockParser(this.logger.Object); } - private static IComponentStream MakeStream(string name, string toml) - { - return new ComponentStream - { - Location = name, - Pattern = "Cargo.lock", - Stream = new MemoryStream(Encoding.UTF8.GetBytes(toml)), - }; - } - - private static (int Usages, int ExplicitRoots, int Edges, int Failures) Analyze(Mock recorder) - { - var usageInvocations = recorder.Invocations.Where(i => i.Method.Name == "RegisterUsage").ToList(); - var explicitRoots = 0; - var edges = 0; - foreach (var inv in usageInvocations) - { - // Signature: RegisterUsage(DetectedComponent dc, bool isExplicitReferencedDependency = false, string parentComponentId = null, bool isDevelopmentDependency = false) - if (inv.Arguments.Count >= 2 && inv.Arguments[1] is bool explicitFlag && explicitFlag) - { - explicitRoots++; - } - - if (inv.Arguments.Count >= 3 && inv.Arguments[2] is string parentId) - { - edges++; - } - } - - var failures = recorder.Invocations.Count(i => i.Method.Name == "RegisterPackageParseFailure"); - return (usageInvocations.Count, explicitRoots, edges, failures); - } - [TestMethod] public async Task ParseAsync_NoPackages_ReturnsVersion_NoUsage() { @@ -731,4 +698,37 @@ public async Task DifferentVersionsSamePackage_BothRegistered() edges.Should().Be(2); failures.Should().Be(0); } + + private static IComponentStream MakeStream(string name, string toml) + { + return new ComponentStream + { + Location = name, + Pattern = "Cargo.lock", + Stream = new MemoryStream(Encoding.UTF8.GetBytes(toml)), + }; + } + + private static (int Usages, int ExplicitRoots, int Edges, int Failures) Analyze(Mock recorder) + { + var usageInvocations = recorder.Invocations.Where(i => i.Method.Name == "RegisterUsage").ToList(); + var explicitRoots = 0; + var edges = 0; + foreach (var inv in usageInvocations) + { + // Signature: RegisterUsage(DetectedComponent dc, bool isExplicitReferencedDependency = false, string parentComponentId = null, bool isDevelopmentDependency = false) + if (inv.Arguments.Count >= 2 && inv.Arguments[1] is bool explicitFlag && explicitFlag) + { + explicitRoots++; + } + + if (inv.Arguments.Count >= 3 && inv.Arguments[2] is string parentId) + { + edges++; + } + } + + var failures = recorder.Invocations.Count(i => i.Method.Name == "RegisterPackageParseFailure"); + return (usageInvocations.Count, explicitRoots, edges, failures); + } } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/RustCliParserTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/RustCliParserTests.cs index 1fb1b7b54..e91d57243 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/RustCliParserTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/RustCliParserTests.cs @@ -41,51 +41,6 @@ public void Init() this.parser = new RustCliParser(this.cli.Object, this.env.Object, new PathUtilityService(new Mock>().Object), this.logger.Object); } - private static IComponentStream MakeTomlStream(string path) => - new ComponentStream { Location = path, Pattern = "Cargo.toml", Stream = new MemoryStream(Encoding.UTF8.GetBytes("[package]\nname=\"x\"")) }; - - // kind: build (non-dev), kind: dev (development), or absent/null. - private static string BuildNormalRootMetadataJson() => """ - { - "packages": [ - { "name":"rootpkg", "version":"1.0.0", "id":"rootpkg 1.0.0", "authors":[""], "license":"", "source":null, "manifest_path":"C:/repo/root/Cargo.toml" }, - { "name":"childA", "version":"2.0.0", "id":"childA 2.0.0", "authors":["Alice"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/root/childA/Cargo.toml" }, - { "name":"childDev", "version":"3.0.0", "id":"childDev 3.0.0", "authors":["Bob"], "license":"Apache-2.0", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/root/childDev/Cargo.toml" } - ], - "resolve": { - "root":"rootpkg 1.0.0", - "nodes":[ - { "id":"rootpkg 1.0.0", - "deps":[ - { "pkg":"childA 2.0.0", "dep_kinds":[{"kind":"build"}] }, - { "pkg":"childDev 3.0.0", "dep_kinds":[{"kind":"dev"}] } - ] - }, - { "id":"childA 2.0.0", "deps":[] }, - { "id":"childDev 3.0.0", "deps":[] } - ] - } - } - """; - - private static string BuildVirtualManifestMetadataJson() => """ - { - "packages": [ - { "name":"virtA", "version":"0.2.0", "id":"virtA 0.2.0", "authors":["Ann"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/virtA/Cargo.toml" }, - { "name":"virtB", "version":"0.3.0", "id":"virtB 0.3.0", "authors":["Ben"], "license":"Apache-2.0", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/virtB/Cargo.toml" } - ], - "resolve": { - "root": null, - "nodes":[ - { "id":"virtA 0.2.0", "deps":[ { "pkg":"virtB 0.3.0", "dep_kinds":[{"kind":"build"}] } ] }, - { "id":"virtB 0.3.0", "deps":[] } - ] - } - } - """; - - private static CargoMetadata ParseMetadata(string json) => CargoMetadata.FromJson(json); - [TestMethod] public async Task ParseAsync_ManuallyDisabled_ReturnsFailure() { @@ -1136,6 +1091,51 @@ public async Task VirtualManifest_MultipleRootNodes_AllProcessed() distinctNames.Should().Contain("pkgB"); } + private static IComponentStream MakeTomlStream(string path) => + new ComponentStream { Location = path, Pattern = "Cargo.toml", Stream = new MemoryStream(Encoding.UTF8.GetBytes("[package]\nname=\"x\"")) }; + + // kind: build (non-dev), kind: dev (development), or absent/null. + private static string BuildNormalRootMetadataJson() => """ + { + "packages": [ + { "name":"rootpkg", "version":"1.0.0", "id":"rootpkg 1.0.0", "authors":[""], "license":"", "source":null, "manifest_path":"C:/repo/root/Cargo.toml" }, + { "name":"childA", "version":"2.0.0", "id":"childA 2.0.0", "authors":["Alice"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/root/childA/Cargo.toml" }, + { "name":"childDev", "version":"3.0.0", "id":"childDev 3.0.0", "authors":["Bob"], "license":"Apache-2.0", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/root/childDev/Cargo.toml" } + ], + "resolve": { + "root":"rootpkg 1.0.0", + "nodes":[ + { "id":"rootpkg 1.0.0", + "deps":[ + { "pkg":"childA 2.0.0", "dep_kinds":[{"kind":"build"}] }, + { "pkg":"childDev 3.0.0", "dep_kinds":[{"kind":"dev"}] } + ] + }, + { "id":"childA 2.0.0", "deps":[] }, + { "id":"childDev 3.0.0", "deps":[] } + ] + } + } + """; + + private static string BuildVirtualManifestMetadataJson() => """ + { + "packages": [ + { "name":"virtA", "version":"0.2.0", "id":"virtA 0.2.0", "authors":["Ann"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/virtA/Cargo.toml" }, + { "name":"virtB", "version":"0.3.0", "id":"virtB 0.3.0", "authors":["Ben"], "license":"Apache-2.0", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/virtB/Cargo.toml" } + ], + "resolve": { + "root": null, + "nodes":[ + { "id":"virtA 0.2.0", "deps":[ { "pkg":"virtB 0.3.0", "dep_kinds":[{"kind":"build"}] } ] }, + { "id":"virtB 0.3.0", "deps":[] } + ] + } + } + """; + + private static CargoMetadata ParseMetadata(string json) => CargoMetadata.FromJson(json); + private async Task InvokeProcessMetadataAsync(string manifestLocation, ISingleFileComponentRecorder fallbackRecorder, CargoMetadata metadata) => await this.parser.ParseFromMetadataAsync( new ComponentStream { Location = manifestLocation, Pattern = "Cargo.toml", Stream = new MemoryStream([]) }, diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/RustMetadataContextBuilderTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/RustMetadataContextBuilderTests.cs index 5e70e9b80..fb3dc7eac 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/RustMetadataContextBuilderTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/RustMetadataContextBuilderTests.cs @@ -35,65 +35,6 @@ public void Init() this.envVarService.Object); } - private static string BuildSimpleMetadataJson(string rootManifest, string rootId) => $$""" - { - "packages": [ - { "name":"rootpkg", "version":"1.0.0", "id":"{{rootId}}", "authors":[""], "license":"", "source":null, "manifest_path":"{{rootManifest}}" }, - { "name":"dep1", "version":"2.0.0", "id":"dep1 2.0.0", "authors":["A"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/dep1/Cargo.toml" } - ], - "resolve": { - "root":"{{rootId}}", - "nodes":[ - { "id":"{{rootId}}", "deps":[ { "pkg":"dep1 2.0.0", "dep_kinds":[{"kind":"build"}] } ] }, - { "id":"dep1 2.0.0", "deps":[] } - ] - } - } - """; - - private static string BuildWorkspaceMetadataJson() => """ - { - "packages": [ - { "name":"workspace", "version":"0.1.0", "id":"workspace 0.1.0", "authors":[""], "license":"", "source":null, "manifest_path":"C:/repo/Cargo.toml" }, - { "name":"member1", "version":"0.2.0", "id":"member1 0.2.0", "authors":[""], "license":"", "source":null, "manifest_path":"C:/repo/member1/Cargo.toml" }, - { "name":"member2", "version":"0.3.0", "id":"member2 0.3.0", "authors":[""], "license":"", "source":null, "manifest_path":"C:/repo/member2/Cargo.toml" }, - { "name":"shared", "version":"1.0.0", "id":"shared 1.0.0", "authors":["S"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/shared/Cargo.toml" } - ], - "resolve": { - "root":"workspace 0.1.0", - "nodes":[ - { "id":"workspace 0.1.0", "deps":[] }, - { "id":"member1 0.2.0", "deps":[ { "pkg":"shared 1.0.0", "dep_kinds":[{"kind":"build"}] } ] }, - { "id":"member2 0.3.0", "deps":[ { "pkg":"shared 1.0.0", "dep_kinds":[{"kind":"build"}] } ] }, - { "id":"shared 1.0.0", "deps":[] } - ] - } - } - """; - - private static string BuildDiamondDependencyJson() => """ - { - "packages": [ - { "name":"root", "version":"1.0.0", "id":"root 1.0.0", "authors":[""], "license":"", "source":null, "manifest_path":"C:/repo/Cargo.toml" }, - { "name":"depA", "version":"1.0.0", "id":"depA 1.0.0", "authors":["A"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/depA/Cargo.toml" }, - { "name":"depB", "version":"1.0.0", "id":"depB 1.0.0", "authors":["B"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/depB/Cargo.toml" }, - { "name":"shared", "version":"1.0.0", "id":"shared 1.0.0", "authors":["S"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/shared/Cargo.toml" } - ], - "resolve": { - "root":"root 1.0.0", - "nodes":[ - { "id":"root 1.0.0", "deps":[ - { "pkg":"depA 1.0.0", "dep_kinds":[{"kind":"build"}] }, - { "pkg":"depB 1.0.0", "dep_kinds":[{"kind":"build"}] } - ] }, - { "id":"depA 1.0.0", "deps":[ { "pkg":"shared 1.0.0", "dep_kinds":[{"kind":"build"}] } ] }, - { "id":"depB 1.0.0", "deps":[ { "pkg":"shared 1.0.0", "dep_kinds":[{"kind":"build"}] } ] }, - { "id":"shared 1.0.0", "deps":[] } - ] - } - } - """; - [TestMethod] public async Task BuildPackageOwnershipMapAsync_ManuallyDisabled_ReturnsEmptyResult() { @@ -474,4 +415,63 @@ public async Task BuildPackageOwnershipMapAsync_PathNormalization_UsesNormalized result.LocalPackageManifests.Should().Contain(normalizedPath); result.ManifestToMetadata.Should().ContainKey(normalizedPath); } + + private static string BuildSimpleMetadataJson(string rootManifest, string rootId) => $$""" + { + "packages": [ + { "name":"rootpkg", "version":"1.0.0", "id":"{{rootId}}", "authors":[""], "license":"", "source":null, "manifest_path":"{{rootManifest}}" }, + { "name":"dep1", "version":"2.0.0", "id":"dep1 2.0.0", "authors":["A"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/dep1/Cargo.toml" } + ], + "resolve": { + "root":"{{rootId}}", + "nodes":[ + { "id":"{{rootId}}", "deps":[ { "pkg":"dep1 2.0.0", "dep_kinds":[{"kind":"build"}] } ] }, + { "id":"dep1 2.0.0", "deps":[] } + ] + } + } + """; + + private static string BuildWorkspaceMetadataJson() => """ + { + "packages": [ + { "name":"workspace", "version":"0.1.0", "id":"workspace 0.1.0", "authors":[""], "license":"", "source":null, "manifest_path":"C:/repo/Cargo.toml" }, + { "name":"member1", "version":"0.2.0", "id":"member1 0.2.0", "authors":[""], "license":"", "source":null, "manifest_path":"C:/repo/member1/Cargo.toml" }, + { "name":"member2", "version":"0.3.0", "id":"member2 0.3.0", "authors":[""], "license":"", "source":null, "manifest_path":"C:/repo/member2/Cargo.toml" }, + { "name":"shared", "version":"1.0.0", "id":"shared 1.0.0", "authors":["S"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/shared/Cargo.toml" } + ], + "resolve": { + "root":"workspace 0.1.0", + "nodes":[ + { "id":"workspace 0.1.0", "deps":[] }, + { "id":"member1 0.2.0", "deps":[ { "pkg":"shared 1.0.0", "dep_kinds":[{"kind":"build"}] } ] }, + { "id":"member2 0.3.0", "deps":[ { "pkg":"shared 1.0.0", "dep_kinds":[{"kind":"build"}] } ] }, + { "id":"shared 1.0.0", "deps":[] } + ] + } + } + """; + + private static string BuildDiamondDependencyJson() => """ + { + "packages": [ + { "name":"root", "version":"1.0.0", "id":"root 1.0.0", "authors":[""], "license":"", "source":null, "manifest_path":"C:/repo/Cargo.toml" }, + { "name":"depA", "version":"1.0.0", "id":"depA 1.0.0", "authors":["A"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/depA/Cargo.toml" }, + { "name":"depB", "version":"1.0.0", "id":"depB 1.0.0", "authors":["B"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/depB/Cargo.toml" }, + { "name":"shared", "version":"1.0.0", "id":"shared 1.0.0", "authors":["S"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/shared/Cargo.toml" } + ], + "resolve": { + "root":"root 1.0.0", + "nodes":[ + { "id":"root 1.0.0", "deps":[ + { "pkg":"depA 1.0.0", "dep_kinds":[{"kind":"build"}] }, + { "pkg":"depB 1.0.0", "dep_kinds":[{"kind":"build"}] } + ] }, + { "id":"depA 1.0.0", "deps":[ { "pkg":"shared 1.0.0", "dep_kinds":[{"kind":"build"}] } ] }, + { "id":"depB 1.0.0", "deps":[ { "pkg":"shared 1.0.0", "dep_kinds":[{"kind":"build"}] } ] }, + { "id":"shared 1.0.0", "deps":[] } + ] + } + } + """; } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/RustSbomParserTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/RustSbomParserTests.cs index 04ac761ea..93cffc708 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/RustSbomParserTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/RustSbomParserTests.cs @@ -32,63 +32,6 @@ public void Init() this.parser = new RustSbomParser(this.logger.Object); } - private static IComponentStream MakeSbomStream(string location, string json) => - new ComponentStream - { - Location = location, - Pattern = "*.cargo-sbom.json", - Stream = new MemoryStream(Encoding.UTF8.GetBytes(json)), - }; - - private static string BuildSimpleSbomJson() => $$""" - { - "version": 1, - "root": 0, - "crates": [ - { - "id": "path+file:///repo/root#0.1.0", - "features": [], - "dependencies": [ - { "index": 1, "kind": "normal" } - ] - }, - { - "id": "{{CratesIo}}#dep1@1.0.0", - "features": [], - "dependencies": [] - } - ] - } - """; - - private static string BuildNestedSbomJson() => $$""" - { - "version": 1, - "root": 0, - "crates": [ - { - "id": "path+file:///repo/root#0.1.0", - "features": [], - "dependencies": [ - { "index": 1, "kind": "normal" } - ] - }, - { - "id": "{{CratesIo}}#parent@2.0.0", - "features": [], - "dependencies": [ - { "index": 2, "kind": "normal" } - ] - }, - { - "id": "{{CratesIo}}#child@3.0.0", - "features": [], - "dependencies": [] - } - ] - } - """; - [TestMethod] public async Task ParseAsync_ValidSimpleSbom_RegistersComponents() { @@ -992,4 +935,61 @@ await this.parser.ParseWithOwnershipAsync( // Verify fallback happened and parentId was checked in graph sbomRecorder.Invocations.Count(i => i.Method.Name == "RegisterUsage").Should().BePositive(); } + + private static IComponentStream MakeSbomStream(string location, string json) => + new ComponentStream + { + Location = location, + Pattern = "*.cargo-sbom.json", + Stream = new MemoryStream(Encoding.UTF8.GetBytes(json)), + }; + + private static string BuildSimpleSbomJson() => $$""" + { + "version": 1, + "root": 0, + "crates": [ + { + "id": "path+file:///repo/root#0.1.0", + "features": [], + "dependencies": [ + { "index": 1, "kind": "normal" } + ] + }, + { + "id": "{{CratesIo}}#dep1@1.0.0", + "features": [], + "dependencies": [] + } + ] + } + """; + + private static string BuildNestedSbomJson() => $$""" + { + "version": 1, + "root": 0, + "crates": [ + { + "id": "path+file:///repo/root#0.1.0", + "features": [], + "dependencies": [ + { "index": 1, "kind": "normal" } + ] + }, + { + "id": "{{CratesIo}}#parent@2.0.0", + "features": [], + "dependencies": [ + { "index": 2, "kind": "normal" } + ] + }, + { + "id": "{{CratesIo}}#child@3.0.0", + "features": [], + "dependencies": [] + } + ] + } + """; } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/SimplePypiClientTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/SimplePypiClientTests.cs index b41880bb4..9a2612b4d 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/SimplePypiClientTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/SimplePypiClientTests.cs @@ -20,29 +20,6 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; [TestClass] public class SimplePyPiClientTests { - private Mock MockHttpMessageHandler(string content, HttpStatusCode statusCode) - { - var handlerMock = new Mock(); - handlerMock.Protected() - .Setup>( - "SendAsync", - ItExpr.IsAny(), - ItExpr.IsAny()) - .ReturnsAsync(new HttpResponseMessage() - { - StatusCode = statusCode, - Content = new StringContent(content), - }); - - return handlerMock; - } - - private ISimplePyPiClient CreateSimplePypiClient(HttpMessageHandler messageHandler, IEnvironmentVariableService evs, ILogger logger) - { - SimplePyPiClient.HttpClient = new HttpClient(messageHandler); - return new SimplePyPiClient(evs, logger); - } - [TestMethod] public async Task GetSimplePypiProject_DuplicateEntries_CallsGetAsync_OnceAsync() { @@ -355,4 +332,27 @@ public string SampleValidApiJsonResponse(string packageName, string version) return packageJsonTemplate; } + + private Mock MockHttpMessageHandler(string content, HttpStatusCode statusCode) + { + var handlerMock = new Mock(); + handlerMock.Protected() + .Setup>( + "SendAsync", + ItExpr.IsAny(), + ItExpr.IsAny()) + .ReturnsAsync(new HttpResponseMessage() + { + StatusCode = statusCode, + Content = new StringContent(content), + }); + + return handlerMock; + } + + private ISimplePyPiClient CreateSimplePypiClient(HttpMessageHandler messageHandler, IEnvironmentVariableService evs, ILogger logger) + { + SimplePyPiClient.HttpClient = new HttpClient(messageHandler); + return new SimplePyPiClient(evs, logger); + } } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Utilities/ComponentRecorderTestUtilities.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/Utilities/ComponentRecorderTestUtilities.cs index ae290cc85..404c358cf 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/Utilities/ComponentRecorderTestUtilities.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/Utilities/ComponentRecorderTestUtilities.cs @@ -113,30 +113,6 @@ public static void AssertAllExplicitlyReferencedComponents( }); } - private static ComponentOrientedGrouping TupleToObject(IEnumerable<(string Location, IDependencyGraph Graph, string ComponentId)> x) - { - var additionalRelatedFiles = new List(x.SelectMany(y => y.Graph.GetAdditionalRelatedFiles())); - additionalRelatedFiles.AddRange(x.Select(y => y.Location)); - - return new ComponentOrientedGrouping - { - ComponentId = x.First().ComponentId, - FoundInGraphs = x.Select(y => (y.Location, y.Graph)).ToList(), - AllFileLocations = additionalRelatedFiles.Distinct().ToList(), - ParentComponentIdsThatAreExplicitReferences = x.SelectMany(y => y.Graph.GetExplicitReferencedDependencyIds(x.First().ComponentId)).Distinct().ToList(), - }; - } - - private static List> GroupByComponentId(IReadOnlyDictionary graphs) - { - return graphs - .Select(x => (Location: x.Key, Graph: x.Value)) - .SelectMany(x => x.Graph.GetComponents() - .Select(componentId => (x.Location, x.Graph, ComponentId: componentId))) - .GroupBy(x => x.ComponentId) - .ToList(); - } - public static void CheckGraphStructure(IDependencyGraph graph, Dictionary graphComponentsWithDeps) { var graphComponents = graph.GetComponents().ToArray(); @@ -172,6 +148,30 @@ public static void CheckChild(IComponentRecorder recorder, string childId, st parentIds.Select(parentId => new Func(x => x.Id == parentId)).ToArray()); } + private static ComponentOrientedGrouping TupleToObject(IEnumerable<(string Location, IDependencyGraph Graph, string ComponentId)> x) + { + var additionalRelatedFiles = new List(x.SelectMany(y => y.Graph.GetAdditionalRelatedFiles())); + additionalRelatedFiles.AddRange(x.Select(y => y.Location)); + + return new ComponentOrientedGrouping + { + ComponentId = x.First().ComponentId, + FoundInGraphs = x.Select(y => (y.Location, y.Graph)).ToList(), + AllFileLocations = additionalRelatedFiles.Distinct().ToList(), + ParentComponentIdsThatAreExplicitReferences = x.SelectMany(y => y.Graph.GetExplicitReferencedDependencyIds(x.First().ComponentId)).Distinct().ToList(), + }; + } + + private static List> GroupByComponentId(IReadOnlyDictionary graphs) + { + return graphs + .Select(x => (Location: x.Key, Graph: x.Value)) + .SelectMany(x => x.Graph.GetComponents() + .Select(componentId => (x.Location, x.Graph, ComponentId: componentId))) + .GroupBy(x => x.ComponentId) + .ToList(); + } + public class ComponentOrientedGrouping { public IEnumerable<(string ManifestFile, IDependencyGraph Graph)> FoundInGraphs { get; set; } diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs index ee369f242..00b8ced9d 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs @@ -50,13 +50,6 @@ public ExperimentServiceTests() this.experimentConfigMock.Setup(x => x.ShouldRecord(this.detectorMock.Object, It.IsAny())).Returns(true); } - private void SetupGraphMock(IEnumerable components) - { - this.graphTranslationServiceMock - .Setup(x => x.GenerateScanResultFromProcessingResult(It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(new ScanResult() { ComponentsFound = components }); - } - [TestInitialize] public void TestInitialize() { @@ -358,4 +351,11 @@ public async Task InitializeAsync_SwallowsExceptionsAsync() await action.Should().NotThrowAsync(); this.experimentConfigMock.Verify(x => x.InitAsync(), Times.Once()); } + + private void SetupGraphMock(IEnumerable components) + { + this.graphTranslationServiceMock + .Setup(x => x.GenerateScanResultFromProcessingResult(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(new ScanResult() { ComponentsFound = components }); + } } diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs index 0d9c87c4d..f33b57914 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs @@ -72,15 +72,6 @@ public DetectorProcessingServiceTests() this.isWin = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); } - private IndividualDetectorScanResult ExpectedResultForDetector(string detectorId) - { - return new IndividualDetectorScanResult - { - AdditionalTelemetryDetails = new Dictionary { { "detectorId", detectorId } }, - ResultCode = ProcessingResultCode.Success, - }; - } - [TestMethod] public async Task ProcessDetectorsAsync_HappyPathReturnsDetectedComponentsAsync() { @@ -588,6 +579,15 @@ public async Task ProcessDetectorsAsync_InitializesExperimentsAsync() this.experimentServiceMock.Verify(x => x.InitializeAsync(), Times.Once); } + private IndividualDetectorScanResult ExpectedResultForDetector(string detectorId) + { + return new IndividualDetectorScanResult + { + AdditionalTelemetryDetails = new Dictionary { { "detectorId", detectorId } }, + ResultCode = ProcessingResultCode.Success, + }; + } + private Mock SetupFileDetectorMock(string id, DirectoryInfo sourceDirectory = null) { var mockFileDetector = new Mock(); From e928e0ce20f03cbd1e51c2025e59c7ef5c29101d Mon Sep 17 00:00:00 2001 From: Adam Plaskitt Date: Mon, 24 Nov 2025 18:15:09 +0000 Subject: [PATCH 4/5] Revert "Ref: Address SA1202 warning" This reverts commit 4204e694cbf41017b6c51c4a330023f02dde90c2. --- .editorconfig | 2 +- .../DockerService.cs | 48 +-- .../PathUtilityService.cs | 4 +- .../Utilities/StringUtilities.cs | 2 +- .../FileComponentDetector.cs | 120 +++---- .../TypedComponent/CargoComponent.cs | 10 +- .../TypedComponent/ConanComponent.cs | 10 +- .../TypedComponent/DockerImageComponent.cs | 10 +- .../TypedComponent/DotNetComponent.cs | 10 +- .../TypedComponent/GoComponent.cs | 4 +- .../TypedComponent/LinuxComponent.cs | 10 +- .../TypedComponent/NpmComponent.cs | 10 +- .../TypedComponent/NugetComponent.cs | 10 +- .../TypedComponent/OtherComponent.cs | 10 +- .../TypedComponent/PipComponent.cs | 10 +- .../TypedComponent/PodComponent.cs | 10 +- .../TypedComponent/RubyGemsComponent.cs | 10 +- .../TypedComponent/SpdxComponent.cs | 10 +- .../TypedComponent/VcpkgComponent.cs | 10 +- .../dotnet/DotNetComponentDetector.cs | 132 ++++---- .../go/Parsers/GoModParser.cs | 90 ++--- .../linux/LinuxContainerDetector.cs | 18 +- .../maven/MavenCommandService.cs | 2 +- .../maven/MvnCliComponentDetector.cs | 10 +- .../npm/NpmComponentUtilities.cs | 6 +- .../npm/NpmLockfileDetectorBase.cs | 4 +- .../FrameworkPackages/FrameworkPackages.cs | 56 ++-- .../FrameworkPackages.net9.0.cs | 4 +- .../nuget/NuGetComponentDetector.cs | 4 +- ...ectModelProjectCentricComponentDetector.cs | 80 ++--- .../Contracts/PipDependencySpecification.cs | 116 +++---- .../pip/IPyPiClient.cs | 14 +- .../pip/PipCommandService.cs | 88 ++--- .../pip/PipReportUtilities.cs | 8 +- .../pip/PythonCommandService.cs | 32 +- .../pip/PythonResolver.cs | 52 +-- .../pip/PythonResolverBase.cs | 52 +-- .../pip/PythonVersionUtilities.cs | 34 +- .../pip/SimplePypiClient.cs | 26 +- .../pip/SimplePythonResolver.cs | 166 ++++----- .../rust/Parsers/RustCargoLockParser.cs | 4 +- .../rust/Parsers/RustSbomParser.cs | 74 ++--- .../Experiments/DetectorExperiments.cs | 4 +- .../Experiments/ExperimentService.cs | 26 +- .../LoggingEnricher.cs | 6 +- .../DotNetComponentDetectorTests.cs | 314 +++++++++--------- .../GoComponentDetectorTests.cs | 50 +-- .../PipCommandServiceTests.cs | 46 +-- .../PipDependencySpecifierTests.cs | 106 +++--- .../RustCargoLockParserTests.cs | 66 ++-- .../RustCliParserTests.cs | 90 ++--- .../RustMetadataContextBuilderTests.cs | 118 +++---- .../RustSbomParserTests.cs | 114 +++---- .../SimplePypiClientTests.cs | 46 +-- .../ComponentRecorderTestUtilities.cs | 48 +-- .../Experiments/ExperimentServiceTests.cs | 14 +- .../DetectorProcessingServiceTests.cs | 18 +- 57 files changed, 1223 insertions(+), 1225 deletions(-) diff --git a/.editorconfig b/.editorconfig index 557178076..aa0d438c9 100644 --- a/.editorconfig +++ b/.editorconfig @@ -481,7 +481,7 @@ dotnet_diagnostic.SA1201.severity = warning # https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1202.md # Elements should be ordered by access -dotnet_diagnostic.SA1202.severity = warning +dotnet_diagnostic.SA1202.severity = none # https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1600.md # Elements should be documented diff --git a/src/Microsoft.ComponentDetection.Common/DockerService.cs b/src/Microsoft.ComponentDetection.Common/DockerService.cs index a39d258b3..d9b47e05b 100644 --- a/src/Microsoft.ComponentDetection.Common/DockerService.cs +++ b/src/Microsoft.ComponentDetection.Common/DockerService.cs @@ -83,6 +83,13 @@ public async Task ImageExistsLocallyAsync(string image, CancellationToken } } + private async Task InspectImageAndSanitizeVarsAsync(string image, CancellationToken cancellationToken = default) + { + var imageInspectResponse = await Client.Images.InspectImageAsync(image, cancellationToken); + this.SanitizeEnvironmentVariables(imageInspectResponse); + return imageInspectResponse; + } + public async Task TryPullImageAsync(string image, CancellationToken cancellationToken = default) { using var record = new DockerServiceTryPullImageTelemetryRecord @@ -111,6 +118,23 @@ public async Task TryPullImageAsync(string image, CancellationToken cancel } } + internal void SanitizeEnvironmentVariables(ImageInspectResponse inspectResponse) + { + var envVariables = inspectResponse?.Config?.Env; + if (envVariables == null || !envVariables.Any()) + { + return; + } + + var sanitizedVarList = new List(); + foreach (var variable in inspectResponse.Config.Env) + { + sanitizedVarList.Add(variable.RemoveSensitiveInformation()); + } + + inspectResponse.Config.Env = sanitizedVarList; + } + public async Task InspectImageAsync(string image, CancellationToken cancellationToken = default) { using var record = new DockerServiceInspectImageTelemetryRecord @@ -177,23 +201,6 @@ public async Task InspectImageAsync(string image, Cancellation return (stdout, stderr); } - internal void SanitizeEnvironmentVariables(ImageInspectResponse inspectResponse) - { - var envVariables = inspectResponse?.Config?.Env; - if (envVariables == null || !envVariables.Any()) - { - return; - } - - var sanitizedVarList = new List(); - foreach (var variable in inspectResponse.Config.Env) - { - sanitizedVarList.Add(variable.RemoveSensitiveInformation()); - } - - inspectResponse.Config.Env = sanitizedVarList; - } - private static async Task CreateContainerAsync( string image, IList command, @@ -255,11 +262,4 @@ private static int GetContainerId() { return Interlocked.Increment(ref incrementingContainerId); } - - private async Task InspectImageAndSanitizeVarsAsync(string image, CancellationToken cancellationToken = default) - { - var imageInspectResponse = await Client.Images.InspectImageAsync(image, cancellationToken); - this.SanitizeEnvironmentVariables(imageInspectResponse); - return imageInspectResponse; - } } diff --git a/src/Microsoft.ComponentDetection.Common/PathUtilityService.cs b/src/Microsoft.ComponentDetection.Common/PathUtilityService.cs index e5405aa93..27df98217 100644 --- a/src/Microsoft.ComponentDetection.Common/PathUtilityService.cs +++ b/src/Microsoft.ComponentDetection.Common/PathUtilityService.cs @@ -75,6 +75,8 @@ public string ResolvePhysicalPath(string path) return fileInfo.Exists ? this.ResolvePathFromInfo(fileInfo) : null; } + private string ResolvePathFromInfo(FileSystemInfo info) => info.LinkTarget ?? info.FullName; + public string NormalizePath(string path) { if (string.IsNullOrWhiteSpace(path)) @@ -87,6 +89,4 @@ public string NormalizePath(string path) // AltDirectorySeparatorChar is / on Unix and on Windows. return path.Replace('\\', Path.AltDirectorySeparatorChar); } - - private string ResolvePathFromInfo(FileSystemInfo info) => info.LinkTarget ?? info.FullName; } diff --git a/src/Microsoft.ComponentDetection.Common/Utilities/StringUtilities.cs b/src/Microsoft.ComponentDetection.Common/Utilities/StringUtilities.cs index 1c498ba80..878e0c140 100644 --- a/src/Microsoft.ComponentDetection.Common/Utilities/StringUtilities.cs +++ b/src/Microsoft.ComponentDetection.Common/Utilities/StringUtilities.cs @@ -6,8 +6,8 @@ namespace Microsoft.ComponentDetection.Common; public static class StringUtilities { - public const string SensitivePlaceholder = "******"; private static readonly Regex SensitiveInfoRegex = new Regex(@"(?<=https://)(.+)(?=@)", RegexOptions.Compiled | RegexOptions.IgnoreCase, TimeSpan.FromSeconds(5)); + public const string SensitivePlaceholder = "******"; /// /// Utility method to remove sensitive information from a string, currently focused on removing on the credentials placed within URL which can be part of CLI commands. diff --git a/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs b/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs index 4fe1db91f..7669726d4 100644 --- a/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs @@ -17,6 +17,18 @@ namespace Microsoft.ComponentDetection.Contracts; /// Specialized base class for file based component detection. public abstract class FileComponentDetector : IComponentDetector { + /// + /// Gets or sets the factory for handing back component streams to File detectors. + /// + protected IComponentStreamEnumerableFactory ComponentStreamEnumerableFactory { get; set; } + + protected IObservableDirectoryWalkerFactory Scanner { get; set; } + + /// + /// Gets or sets the logger for writing basic logging message to both console and file. + /// + protected ILogger Logger { get; set; } + public IComponentRecorder ComponentRecorder { get; private set; } /// @@ -34,25 +46,6 @@ public abstract class FileComponentDetector : IComponentDetector /// Gets the version of this component detector. public abstract int Version { get; } - public virtual bool NeedsAutomaticRootDependencyCalculation { get; protected set; } - - /// - /// List of any any additional properties as key-value pairs that we would like to capture for the detector. - /// - public List<(string PropertyKey, string PropertyValue)> AdditionalProperties { get; set; } = []; - - /// - /// Gets or sets the factory for handing back component streams to File detectors. - /// - protected IComponentStreamEnumerableFactory ComponentStreamEnumerableFactory { get; set; } - - protected IObservableDirectoryWalkerFactory Scanner { get; set; } - - /// - /// Gets or sets the logger for writing basic logging message to both console and file. - /// - protected ILogger Logger { get; set; } - /// /// Gets the folder names that will be skipped by the Component Detector. /// @@ -64,8 +57,15 @@ public abstract class FileComponentDetector : IComponentDetector /// protected ScanRequest CurrentScanRequest { get; set; } + public virtual bool NeedsAutomaticRootDependencyCalculation { get; protected set; } + protected ConcurrentDictionary Telemetry { get; set; } = []; + /// + /// List of any any additional properties as key-value pairs that we would like to capture for the detector. + /// + public List<(string PropertyKey, string PropertyValue)> AdditionalProperties { get; set; } = []; + protected IObservable ComponentStreams { get; private set; } protected virtual bool EnableParallelism { get; set; } @@ -78,6 +78,16 @@ public async virtual Task ExecuteDetectorAsync(Sca return await this.ScanDirectoryAsync(request, cancellationToken); } + private Task ScanDirectoryAsync(ScanRequest request, CancellationToken cancellationToken = default) + { + this.CurrentScanRequest = request; + + var filteredObservable = this.Scanner.GetFilteredComponentStreamObservable(request.SourceDirectory, this.SearchPatterns, request.ComponentRecorder); + + this.Logger.LogDebug("Registered {Detector}", this.GetType().FullName); + return this.ProcessAsync(filteredObservable, request.DetectorArgs, request.MaxThreads, request.CleanupCreatedFiles, cancellationToken); + } + /// /// Gets the file streams for the Detector's declared as an . /// @@ -101,6 +111,37 @@ protected Task> GetFileStreamsAsync(DirectoryInfo /// The lockfile version. protected void RecordLockfileVersion(string lockfileVersion) => this.Telemetry["LockfileVersion"] = lockfileVersion; + private async Task ProcessAsync( + IObservable processRequests, IDictionary detectorArgs, int maxThreads, bool cleanupCreatedFiles, CancellationToken cancellationToken = default) + { + var threadsToUse = this.EnableParallelism ? Math.Min(Environment.ProcessorCount, maxThreads) : 1; + this.Telemetry["ThreadsUsed"] = $"{threadsToUse}"; + + var processor = new ActionBlock( + async processRequest => await this.OnFileFoundAsync(processRequest, detectorArgs, cleanupCreatedFiles, cancellationToken), + new ExecutionDataflowBlockOptions + { + // MaxDegreeOfParallelism is the lower of the processor count and the max threads arg that the customer passed in + MaxDegreeOfParallelism = threadsToUse, + }); + + var preprocessedObserbable = await this.OnPrepareDetectionAsync(processRequests, detectorArgs, cancellationToken); + + await preprocessedObserbable.ForEachAsync(processRequest => processor.Post(processRequest)); + + processor.Complete(); + + await processor.Completion; + + await this.OnDetectionFinishedAsync(); + + return new IndividualDetectorScanResult + { + ResultCode = ProcessingResultCode.Success, + AdditionalTelemetryDetails = this.Telemetry.ToDictionary(kvp => kvp.Key, kvp => kvp.Value), + }; + } + /// /// Auxliary method executed before the actual scanning of a given file takes place. /// This method can be used to modify or create new ProcessRequests that later will @@ -133,45 +174,4 @@ protected virtual Task OnDetectionFinishedAsync() // Do not cleanup by default, only if the detector uses the FileComponentWithCleanup abstract class. protected virtual async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, bool cleanupCreatedFiles, CancellationToken cancellationToken = default) => await this.OnFileFoundAsync(processRequest, detectorArgs, cancellationToken); - - private Task ScanDirectoryAsync(ScanRequest request, CancellationToken cancellationToken = default) - { - this.CurrentScanRequest = request; - - var filteredObservable = this.Scanner.GetFilteredComponentStreamObservable(request.SourceDirectory, this.SearchPatterns, request.ComponentRecorder); - - this.Logger.LogDebug("Registered {Detector}", this.GetType().FullName); - return this.ProcessAsync(filteredObservable, request.DetectorArgs, request.MaxThreads, request.CleanupCreatedFiles, cancellationToken); - } - - private async Task ProcessAsync( - IObservable processRequests, IDictionary detectorArgs, int maxThreads, bool cleanupCreatedFiles, CancellationToken cancellationToken = default) - { - var threadsToUse = this.EnableParallelism ? Math.Min(Environment.ProcessorCount, maxThreads) : 1; - this.Telemetry["ThreadsUsed"] = $"{threadsToUse}"; - - var processor = new ActionBlock( - async processRequest => await this.OnFileFoundAsync(processRequest, detectorArgs, cleanupCreatedFiles, cancellationToken), - new ExecutionDataflowBlockOptions - { - // MaxDegreeOfParallelism is the lower of the processor count and the max threads arg that the customer passed in - MaxDegreeOfParallelism = threadsToUse, - }); - - var preprocessedObserbable = await this.OnPrepareDetectionAsync(processRequests, detectorArgs, cancellationToken); - - await preprocessedObserbable.ForEachAsync(processRequest => processor.Post(processRequest)); - - processor.Complete(); - - await processor.Completion; - - await this.OnDetectionFinishedAsync(); - - return new IndividualDetectorScanResult - { - ResultCode = ProcessingResultCode.Success, - AdditionalTelemetryDetails = this.Telemetry.ToDictionary(kvp => kvp.Key, kvp => kvp.Value), - }; - } } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs index 5d7abf08d..3137601e4 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs @@ -6,6 +6,11 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class CargoComponent : TypedComponent { + private CargoComponent() + { + // reserved for deserialization + } + public CargoComponent(string name, string version, string author = null, string license = null, string source = null) { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Cargo)); @@ -15,11 +20,6 @@ public CargoComponent(string name, string version, string author = null, string this.Source = source; } - private CargoComponent() - { - // reserved for deserialization - } - public string Name { get; set; } public string Version { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ConanComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ConanComponent.cs index 75cc34e77..a480d3eda 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ConanComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ConanComponent.cs @@ -5,6 +5,11 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class ConanComponent : TypedComponent { + private ConanComponent() + { + // reserved for deserialization + } + public ConanComponent(string name, string version, string previous, string packageId) { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Conan)); @@ -13,11 +18,6 @@ public ConanComponent(string name, string version, string previous, string packa this.Sha1Hash = this.ValidateRequiredInput(packageId, nameof(this.Sha1Hash), nameof(ComponentType.Conan)); } - private ConanComponent() - { - // reserved for deserialization - } - public string Name { get; set; } public string Version { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs index b6efb9140..e89b45428 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs @@ -3,6 +3,11 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class DockerImageComponent : TypedComponent { + private DockerImageComponent() + { + /* Reserved for deserialization */ + } + public DockerImageComponent(string hash, string name = null, string tag = null) { this.Digest = this.ValidateRequiredInput(hash, nameof(this.Digest), nameof(ComponentType.DockerImage)); @@ -10,11 +15,6 @@ public DockerImageComponent(string hash, string name = null, string tag = null) this.Tag = tag; } - private DockerImageComponent() - { - /* Reserved for deserialization */ - } - public string Name { get; set; } public string Digest { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs index f1ad3184d..99b49f5af 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs @@ -7,6 +7,11 @@ public class DotNetComponent : TypedComponent { private const string UnknownValue = "unknown"; + private DotNetComponent() + { + /* Reserved for deserialization */ + } + public DotNetComponent(string sdkVersion, string targetFramework = null, string projectType = null) { if (string.IsNullOrWhiteSpace(sdkVersion) && string.IsNullOrWhiteSpace(targetFramework)) @@ -19,11 +24,6 @@ public DotNetComponent(string sdkVersion, string targetFramework = null, string this.ProjectType = string.IsNullOrWhiteSpace(projectType) ? UnknownValue : projectType; } - private DotNetComponent() - { - /* Reserved for deserialization */ - } - /// /// SDK Version detected, could be null if no global.json exists and no dotnet is on the path. /// diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs index 9291e09da..749396f87 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs @@ -37,6 +37,8 @@ private GoComponent() public override ComponentType Type => ComponentType.Go; + protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}"; + public override bool Equals(object obj) { return obj is GoComponent otherComponent && this.Equals(otherComponent); @@ -56,6 +58,4 @@ public override int GetHashCode() { return this.Name.GetHashCode() ^ this.Version.GetHashCode() ^ this.Hash.GetHashCode(); } - - protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs index cfc498d8d..7ec336364 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs @@ -6,6 +6,11 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class LinuxComponent : TypedComponent { + private LinuxComponent() + { + /* Reserved for deserialization */ + } + public LinuxComponent(string distribution, string release, string name, string version, string license = null, string author = null) { this.Distribution = this.ValidateRequiredInput(distribution, nameof(this.Distribution), nameof(ComponentType.Linux)); @@ -16,11 +21,6 @@ public LinuxComponent(string distribution, string release, string name, string v this.Author = author; } - private LinuxComponent() - { - /* Reserved for deserialization */ - } - public string Distribution { get; set; } public string Release { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs index 51edb38ba..6715d6ce4 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs @@ -6,6 +6,11 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class NpmComponent : TypedComponent { + private NpmComponent() + { + /* Reserved for deserialization */ + } + public NpmComponent(string name, string version, string hash = null, NpmAuthor author = null) { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Npm)); @@ -14,11 +19,6 @@ public NpmComponent(string name, string version, string hash = null, NpmAuthor a this.Author = author; } - private NpmComponent() - { - /* Reserved for deserialization */ - } - public string Name { get; set; } public string Version { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs index 45eeafb9d..9e716d391 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs @@ -5,6 +5,11 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class NuGetComponent : TypedComponent { + private NuGetComponent() + { + /* Reserved for deserialization */ + } + public NuGetComponent(string name, string version, string[] authors = null) { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.NuGet)); @@ -12,11 +17,6 @@ public NuGetComponent(string name, string version, string[] authors = null) this.Authors = authors; } - private NuGetComponent() - { - /* Reserved for deserialization */ - } - public string Name { get; set; } public string Version { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs index 1cdcd29e3..68f6483c5 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs @@ -5,6 +5,11 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class OtherComponent : TypedComponent { + private OtherComponent() + { + /* Reserved for deserialization */ + } + public OtherComponent(string name, string version, Uri downloadUrl, string hash) { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Other)); @@ -13,11 +18,6 @@ public OtherComponent(string name, string version, Uri downloadUrl, string hash) this.Hash = hash; } - private OtherComponent() - { - /* Reserved for deserialization */ - } - public string Name { get; set; } public string Version { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs index 7e88ab755..5858fb7c2 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs @@ -7,6 +7,11 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class PipComponent : TypedComponent { + private PipComponent() + { + /* Reserved for deserialization */ + } + public PipComponent(string name, string version, string author = null, string license = null) { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Pip)); @@ -15,11 +20,6 @@ public PipComponent(string name, string version, string author = null, string li this.License = license; } - private PipComponent() - { - /* Reserved for deserialization */ - } - public string Name { get; set; } public string Version { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs index fe51b572f..64ae8681e 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs @@ -6,6 +6,11 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class PodComponent : TypedComponent { + private PodComponent() + { + /* Reserved for deserialization */ + } + public PodComponent(string name, string version, string specRepo = "") { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Pod)); @@ -13,11 +18,6 @@ public PodComponent(string name, string version, string specRepo = "") this.SpecRepo = specRepo; } - private PodComponent() - { - /* Reserved for deserialization */ - } - public string Name { get; set; } public string Version { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs index b797451ec..1c6ce94f9 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs @@ -5,6 +5,11 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class RubyGemsComponent : TypedComponent { + private RubyGemsComponent() + { + /* Reserved for deserialization */ + } + public RubyGemsComponent(string name, string version, string source = "") { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.RubyGems)); @@ -12,11 +17,6 @@ public RubyGemsComponent(string name, string version, string source = "") this.Source = source; } - private RubyGemsComponent() - { - /* Reserved for deserialization */ - } - public string Name { get; set; } public string Version { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs index dcb785afb..79ab31ea1 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs @@ -5,6 +5,11 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class SpdxComponent : TypedComponent { + private SpdxComponent() + { + /* Reserved for deserialization */ + } + public SpdxComponent(string spdxVersion, Uri documentNamespace, string name, string checksum, string rootElementId, string path) { this.SpdxVersion = this.ValidateRequiredInput(spdxVersion, nameof(this.SpdxVersion), nameof(ComponentType.Spdx)); @@ -15,11 +20,6 @@ public SpdxComponent(string spdxVersion, Uri documentNamespace, string name, str this.Path = this.ValidateRequiredInput(path, nameof(this.Path), nameof(ComponentType.Spdx)); } - private SpdxComponent() - { - /* Reserved for deserialization */ - } - public override ComponentType Type => ComponentType.Spdx; public string RootElementId { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs index 4fc5cc1b1..27c6c7f11 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs @@ -5,6 +5,11 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class VcpkgComponent : TypedComponent { + private VcpkgComponent() + { + /* Reserved for deserialization */ + } + public VcpkgComponent(string spdxid, string name, string version, string triplet = null, string portVersion = null, string description = null, string downloadLocation = null) { int.TryParse(portVersion, out var port); @@ -18,11 +23,6 @@ public VcpkgComponent(string spdxid, string name, string version, string triplet this.DownloadLocation = downloadLocation; } - private VcpkgComponent() - { - /* Reserved for deserialization */ - } - public string SPDXID { get; set; } public string Name { get; set; } diff --git a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs index a0a5139e8..8770ce3bb 100644 --- a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs @@ -59,72 +59,6 @@ public DotNetComponentDetector( public override IEnumerable Categories => ["DotNet"]; - public override Task ExecuteDetectorAsync(ScanRequest request, CancellationToken cancellationToken = default) - { - this.sourceDirectory = this.NormalizeDirectory(request.SourceDirectory.FullName); - this.sourceFileRootDirectory = this.NormalizeDirectory(request.SourceFileRoot?.FullName); - - return base.ExecuteDetectorAsync(request, cancellationToken); - } - - protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) - { - var lockFile = this.lockFileFormat.Read(processRequest.ComponentStream.Stream, processRequest.ComponentStream.Location); - - if (lockFile.PackageSpec is null || lockFile.PackageSpec.RestoreMetadata is null) - { - // The lock file is not valid, or does not contain a PackageSpec. - // This could be due to the lock file being generated by a different version of the SDK. - // We should not fail the detector, but we should log a warning. - this.Logger.LogWarning("Lock file {LockFilePath} does not contain project information.", processRequest.ComponentStream.Location); - return; - } - - var projectAssetsDirectory = this.pathUtilityService.GetParentDirectory(processRequest.ComponentStream.Location); - var projectPath = lockFile.PackageSpec.RestoreMetadata.ProjectPath; - var projectOutputPath = lockFile.PackageSpec.RestoreMetadata.OutputPath; - - // The output path should match the location that the assets file, if it doesn't we could be analyzing paths - // on a different filesystem root than they were created. - // Attempt to rebase paths based on the difference between this file's location and the output path. - var rebasePath = this.GetRootRebasePath(projectAssetsDirectory, projectOutputPath); - - if (rebasePath is not null) - { - projectPath = Path.Combine(this.sourceDirectory!, Path.GetRelativePath(rebasePath, projectPath)); - projectOutputPath = Path.Combine(this.sourceDirectory!, Path.GetRelativePath(rebasePath, projectOutputPath)); - } - - if (!this.fileUtilityService.Exists(projectPath)) - { - // Could be the assets file was not actually from this build - this.Logger.LogWarning("Project path {ProjectPath} specified by {ProjectAssetsPath} does not exist.", projectPath, processRequest.ComponentStream.Location); - } - - var projectDirectory = this.pathUtilityService.GetParentDirectory(projectPath); - var sdkVersion = await this.GetSdkVersionAsync(projectDirectory, cancellationToken); - - var projectName = lockFile.PackageSpec.RestoreMetadata.ProjectName; - - if (!this.directoryUtilityService.Exists(projectOutputPath)) - { - this.Logger.LogWarning("Project output path {ProjectOutputPath} specified by {ProjectAssetsPath} does not exist.", projectOutputPath, processRequest.ComponentStream.Location); - - // default to use the location of the assets file. - projectOutputPath = projectAssetsDirectory; - } - - var targetType = this.GetProjectType(projectOutputPath, projectName, cancellationToken); - - var componentReporter = this.ComponentRecorder.CreateSingleFileComponentRecorder(projectPath); - foreach (var target in lockFile.Targets ?? []) - { - var targetFramework = target.TargetFramework?.GetShortFolderName(); - - componentReporter.RegisterUsage(new DetectedComponent(new DotNetComponent(sdkVersion, targetFramework, targetType))); - } - } - private static string TrimAllEndingDirectorySeparators(string path) { string last; @@ -207,6 +141,72 @@ private static string TrimAllEndingDirectorySeparators(string path) } } + public override Task ExecuteDetectorAsync(ScanRequest request, CancellationToken cancellationToken = default) + { + this.sourceDirectory = this.NormalizeDirectory(request.SourceDirectory.FullName); + this.sourceFileRootDirectory = this.NormalizeDirectory(request.SourceFileRoot?.FullName); + + return base.ExecuteDetectorAsync(request, cancellationToken); + } + + protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) + { + var lockFile = this.lockFileFormat.Read(processRequest.ComponentStream.Stream, processRequest.ComponentStream.Location); + + if (lockFile.PackageSpec is null || lockFile.PackageSpec.RestoreMetadata is null) + { + // The lock file is not valid, or does not contain a PackageSpec. + // This could be due to the lock file being generated by a different version of the SDK. + // We should not fail the detector, but we should log a warning. + this.Logger.LogWarning("Lock file {LockFilePath} does not contain project information.", processRequest.ComponentStream.Location); + return; + } + + var projectAssetsDirectory = this.pathUtilityService.GetParentDirectory(processRequest.ComponentStream.Location); + var projectPath = lockFile.PackageSpec.RestoreMetadata.ProjectPath; + var projectOutputPath = lockFile.PackageSpec.RestoreMetadata.OutputPath; + + // The output path should match the location that the assets file, if it doesn't we could be analyzing paths + // on a different filesystem root than they were created. + // Attempt to rebase paths based on the difference between this file's location and the output path. + var rebasePath = this.GetRootRebasePath(projectAssetsDirectory, projectOutputPath); + + if (rebasePath is not null) + { + projectPath = Path.Combine(this.sourceDirectory!, Path.GetRelativePath(rebasePath, projectPath)); + projectOutputPath = Path.Combine(this.sourceDirectory!, Path.GetRelativePath(rebasePath, projectOutputPath)); + } + + if (!this.fileUtilityService.Exists(projectPath)) + { + // Could be the assets file was not actually from this build + this.Logger.LogWarning("Project path {ProjectPath} specified by {ProjectAssetsPath} does not exist.", projectPath, processRequest.ComponentStream.Location); + } + + var projectDirectory = this.pathUtilityService.GetParentDirectory(projectPath); + var sdkVersion = await this.GetSdkVersionAsync(projectDirectory, cancellationToken); + + var projectName = lockFile.PackageSpec.RestoreMetadata.ProjectName; + + if (!this.directoryUtilityService.Exists(projectOutputPath)) + { + this.Logger.LogWarning("Project output path {ProjectOutputPath} specified by {ProjectAssetsPath} does not exist.", projectOutputPath, processRequest.ComponentStream.Location); + + // default to use the location of the assets file. + projectOutputPath = projectAssetsDirectory; + } + + var targetType = this.GetProjectType(projectOutputPath, projectName, cancellationToken); + + var componentReporter = this.ComponentRecorder.CreateSingleFileComponentRecorder(projectPath); + foreach (var target in lockFile.Targets ?? []) + { + var targetFramework = target.TargetFramework?.GetShortFolderName(); + + componentReporter.RegisterUsage(new DetectedComponent(new DotNetComponent(sdkVersion, targetFramework, targetType))); + } + } + private string? GetProjectType(string projectOutputPath, string projectName, CancellationToken cancellationToken) { if (this.directoryUtilityService.Exists(projectOutputPath) && diff --git a/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoModParser.cs b/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoModParser.cs index 55257d186..2c3afee0b 100644 --- a/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoModParser.cs +++ b/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoModParser.cs @@ -18,51 +18,6 @@ public class GoModParser : IGoParser public GoModParser(ILogger logger) => this.logger = logger; - public async Task ParseAsync( - ISingleFileComponentRecorder singleFileComponentRecorder, - IComponentStream file, - GoGraphTelemetryRecord record) - { - // Collect replace directives - var (replacePathDirectives, moduleReplacements) = await this.GetAllReplaceDirectivesAsync(file); - - // Rewind stream after reading replace directives - file.Stream.Seek(0, SeekOrigin.Begin); - - using var reader = new StreamReader(file.Stream); - - // There can be multiple require( ) sections in go 1.17+. loop over all of them. - while (!reader.EndOfStream) - { - var line = await reader.ReadLineAsync(); - - while (line != null && !line.StartsWith("require (")) - { - if (line.StartsWith("go ")) - { - record.GoModVersion = line[3..].Trim(); - } - - // In go >= 1.17, direct dependencies are listed as "require x/y v1.2.3", and transitive dependencies - // are listed in the require () section - if (line.StartsWith(StartString)) - { - this.TryRegisterDependencyFromModLine(file, line[StartString.Length..], singleFileComponentRecorder, replacePathDirectives, moduleReplacements); - } - - line = await reader.ReadLineAsync(); - } - - // Stopping at the first ) restrict the detection to only the require section. - while ((line = await reader.ReadLineAsync()) != null && !line.EndsWith(')')) - { - this.TryRegisterDependencyFromModLine(file, line, singleFileComponentRecorder, replacePathDirectives, moduleReplacements); - } - } - - return true; - } - /// /// Checks whether the input path is a potential local file system path /// 1. '.' checks whether the path is relative to current directory. @@ -113,6 +68,51 @@ private static void HandleReplaceDirective( } } + public async Task ParseAsync( + ISingleFileComponentRecorder singleFileComponentRecorder, + IComponentStream file, + GoGraphTelemetryRecord record) + { + // Collect replace directives + var (replacePathDirectives, moduleReplacements) = await this.GetAllReplaceDirectivesAsync(file); + + // Rewind stream after reading replace directives + file.Stream.Seek(0, SeekOrigin.Begin); + + using var reader = new StreamReader(file.Stream); + + // There can be multiple require( ) sections in go 1.17+. loop over all of them. + while (!reader.EndOfStream) + { + var line = await reader.ReadLineAsync(); + + while (line != null && !line.StartsWith("require (")) + { + if (line.StartsWith("go ")) + { + record.GoModVersion = line[3..].Trim(); + } + + // In go >= 1.17, direct dependencies are listed as "require x/y v1.2.3", and transitive dependencies + // are listed in the require () section + if (line.StartsWith(StartString)) + { + this.TryRegisterDependencyFromModLine(file, line[StartString.Length..], singleFileComponentRecorder, replacePathDirectives, moduleReplacements); + } + + line = await reader.ReadLineAsync(); + } + + // Stopping at the first ) restrict the detection to only the require section. + while ((line = await reader.ReadLineAsync()) != null && !line.EndsWith(')')) + { + this.TryRegisterDependencyFromModLine(file, line, singleFileComponentRecorder, replacePathDirectives, moduleReplacements); + } + } + + return true; + } + private void TryRegisterDependencyFromModLine(IComponentStream file, string line, ISingleFileComponentRecorder singleFileComponentRecorder, HashSet replacePathDirectives, Dictionary moduleReplacements) { if (line.Trim().StartsWith("//")) diff --git a/src/Microsoft.ComponentDetection.Detectors/linux/LinuxContainerDetector.cs b/src/Microsoft.ComponentDetection.Detectors/linux/LinuxContainerDetector.cs index 228f476be..fdf19a4bf 100644 --- a/src/Microsoft.ComponentDetection.Detectors/linux/LinuxContainerDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/linux/LinuxContainerDetector.cs @@ -49,6 +49,15 @@ ILogger logger /// public bool NeedsAutomaticRootDependencyCalculation => false; + /// + /// Gets the component types that should be detected by this detector. + /// By default, only Linux system packages are detected. + /// Override this method in derived classes to enable detection of additional component types. + /// + /// A set of component types to include in scan results. + protected virtual ISet GetEnabledComponentTypes() => + new HashSet { ComponentType.Linux }; + /// public async Task ExecuteDetectorAsync( ScanRequest request, @@ -109,15 +118,6 @@ public async Task ExecuteDetectorAsync( }; } - /// - /// Gets the component types that should be detected by this detector. - /// By default, only Linux system packages are detected. - /// Override this method in derived classes to enable detection of additional component types. - /// - /// A set of component types to include in scan results. - protected virtual ISet GetEnabledComponentTypes() => - new HashSet { ComponentType.Linux }; - /// /// Extracts and returns the timeout defined by the user, or a default value if one is not provided. /// diff --git a/src/Microsoft.ComponentDetection.Detectors/maven/MavenCommandService.cs b/src/Microsoft.ComponentDetection.Detectors/maven/MavenCommandService.cs index c3a633b97..2e75bd43f 100644 --- a/src/Microsoft.ComponentDetection.Detectors/maven/MavenCommandService.cs +++ b/src/Microsoft.ComponentDetection.Detectors/maven/MavenCommandService.cs @@ -11,6 +11,7 @@ namespace Microsoft.ComponentDetection.Detectors.Maven; public class MavenCommandService : IMavenCommandService { + private const string DetectorLogPrefix = "MvnCli detector"; internal const string MvnCLIFileLevelTimeoutSecondsEnvVar = "MvnCLIFileLevelTimeoutSeconds"; internal const string PrimaryCommand = "mvn"; @@ -18,7 +19,6 @@ public class MavenCommandService : IMavenCommandService internal static readonly string[] AdditionalValidCommands = ["mvn.cmd"]; - private const string DetectorLogPrefix = "MvnCli detector"; private readonly ICommandLineInvocationService commandLineInvocationService; private readonly IMavenStyleDependencyGraphParserService parserService; private readonly IEnvironmentVariableService envVarService; diff --git a/src/Microsoft.ComponentDetection.Detectors/maven/MvnCliComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/maven/MvnCliComponentDetector.cs index 6e15a38c9..a93f5c6b2 100644 --- a/src/Microsoft.ComponentDetection.Detectors/maven/MvnCliComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/maven/MvnCliComponentDetector.cs @@ -45,6 +45,11 @@ public MvnCliComponentDetector( public override IEnumerable Categories => [Enum.GetName(typeof(DetectorClass), DetectorClass.Maven)]; + private void LogDebugWithId(string message) + { + this.Logger.LogDebug("{DetectorId} detector: {Message}", this.Id, message); + } + protected override async Task> OnPrepareDetectionAsync(IObservable processRequests, IDictionary detectorArgs, CancellationToken cancellationToken = default) { if (!await this.mavenCommandService.MavenCLIExistsAsync()) @@ -97,11 +102,6 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID await Task.CompletedTask; } - private void LogDebugWithId(string message) - { - this.Logger.LogDebug("{DetectorId} detector: {Message}", this.Id, message); - } - private IObservable RemoveNestedPomXmls(IObservable componentStreams) { var directoryItemFacades = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); diff --git a/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentUtilities.cs b/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentUtilities.cs index 85a89b32c..12effe1f3 100644 --- a/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentUtilities.cs +++ b/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentUtilities.cs @@ -15,13 +15,13 @@ namespace Microsoft.ComponentDetection.Detectors.Npm; public static class NpmComponentUtilities { - public static readonly string NodeModules = "node_modules"; - public static readonly string LockFile3EnvFlag = "CD_LOCKFILE_V3_ENABLED"; - private static readonly Regex UnsafeCharactersRegex = new Regex( @"[?<>#%{}|`'^\\~\[\]""\s\x7f]|[\x00-\x1f]|[\x80-\xff]", RegexOptions.Compiled); + public static readonly string NodeModules = "node_modules"; + public static readonly string LockFile3EnvFlag = "CD_LOCKFILE_V3_ENABLED"; + public static void TraverseAndRecordComponents(JProperty currentDependency, ISingleFileComponentRecorder singleFileComponentRecorder, TypedComponent component, TypedComponent explicitReferencedDependency, string parentComponentId = null) { var isDevDependency = currentDependency.Value["dev"] is JValue devJValue && (bool)devJValue; diff --git a/src/Microsoft.ComponentDetection.Detectors/npm/NpmLockfileDetectorBase.cs b/src/Microsoft.ComponentDetection.Detectors/npm/NpmLockfileDetectorBase.cs index ef8609e70..690b13172 100644 --- a/src/Microsoft.ComponentDetection.Detectors/npm/NpmLockfileDetectorBase.cs +++ b/src/Microsoft.ComponentDetection.Detectors/npm/NpmLockfileDetectorBase.cs @@ -54,11 +54,11 @@ protected NpmLockfileDetectorBase( public override IEnumerable SupportedComponentTypes { get; } = [ComponentType.Npm]; + private List LernaFiles { get; } = []; + /// protected override IList SkippedFolders => ["node_modules", "pnpm-store"]; - private List LernaFiles { get; } = []; - protected abstract bool IsSupportedLockfileVersion(int lockfileVersion); protected abstract JToken ResolveDependencyObject(JToken packageLockJToken); diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/FrameworkPackages/FrameworkPackages.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/FrameworkPackages/FrameworkPackages.cs index 7f600fe32..6dd5443df 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/FrameworkPackages/FrameworkPackages.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/FrameworkPackages/FrameworkPackages.cs @@ -54,6 +54,28 @@ public FrameworkPackages(NuGetFramework framework, string frameworkName, Framewo public Dictionary Packages { get; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + private static string GetFrameworkKey(string frameworkName) => + frameworkName switch + { + FrameworkNames.NetStandardLibrary => DefaultFrameworkKey, + FrameworkNames.NetCoreApp => DefaultFrameworkKey, + _ => frameworkName, + }; + + internal static void Register(params FrameworkPackages[] toRegister) + { + foreach (var frameworkPackages in toRegister) + { + if (!FrameworkPackagesByFramework.TryGetValue(frameworkPackages.Framework, out var frameworkPackagesForVersion)) + { + FrameworkPackagesByFramework[frameworkPackages.Framework] = frameworkPackagesForVersion = []; + } + + var frameworkKey = GetFrameworkKey(frameworkPackages.FrameworkName); + frameworkPackagesForVersion[frameworkKey] = frameworkPackages; + } + } + public static FrameworkPackages[] GetFrameworkPackages(NuGetFramework framework, string[] frameworkReferences, LockFileTarget lockFileTarget) { var frameworkPackages = new List(); @@ -88,34 +110,6 @@ public static FrameworkPackages[] GetFrameworkPackages(NuGetFramework framework, return frameworkPackages.ToArray(); } - public bool IsAFrameworkComponent(string id, NuGetVersion version) => this.Packages.TryGetValue(id, out var frameworkPackageVersion) && frameworkPackageVersion >= version; - - IEnumerator> IEnumerable>.GetEnumerator() => this.Packages.GetEnumerator(); - - IEnumerator IEnumerable.GetEnumerator() => throw new NotImplementedException(); - - internal static void Register(params FrameworkPackages[] toRegister) - { - foreach (var frameworkPackages in toRegister) - { - if (!FrameworkPackagesByFramework.TryGetValue(frameworkPackages.Framework, out var frameworkPackagesForVersion)) - { - FrameworkPackagesByFramework[frameworkPackages.Framework] = frameworkPackagesForVersion = []; - } - - var frameworkKey = GetFrameworkKey(frameworkPackages.FrameworkName); - frameworkPackagesForVersion[frameworkKey] = frameworkPackages; - } - } - - private static string GetFrameworkKey(string frameworkName) => - frameworkName switch - { - FrameworkNames.NetStandardLibrary => DefaultFrameworkKey, - FrameworkNames.NetCoreApp => DefaultFrameworkKey, - _ => frameworkName, - }; - private static IEnumerable GetLegacyFrameworkPackagesFromPlatformPackages(NuGetFramework framework, LockFileTarget lockFileTarget) { if (framework.Framework == FrameworkConstants.FrameworkIdentifiers.NetCoreApp && framework.Version.Major < 3) @@ -235,6 +229,12 @@ private void Add(string id, string version) } } + public bool IsAFrameworkComponent(string id, NuGetVersion version) => this.Packages.TryGetValue(id, out var frameworkPackageVersion) && frameworkPackageVersion >= version; + + IEnumerator> IEnumerable>.GetEnumerator() => this.Packages.GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => throw new NotImplementedException(); + internal static class FrameworkNames { public const string AspNetCoreApp = "Microsoft.AspNetCore.App"; diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/FrameworkPackages/FrameworkPackages.net9.0.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/FrameworkPackages/FrameworkPackages.net9.0.cs index 330001de7..15c0a7c15 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/FrameworkPackages/FrameworkPackages.net9.0.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/FrameworkPackages/FrameworkPackages.net9.0.cs @@ -11,6 +11,8 @@ internal partial class FrameworkPackages { internal static class NETCoreApp90 { + private static NuGetFramework Net90 { get; } = new NuGetFramework(FrameworkConstants.FrameworkIdentifiers.NetCoreApp, new Version(9, 0)); + internal static FrameworkPackages Instance { get; } = new(Net90, FrameworkNames.NetCoreApp, NETCoreApp80.Instance) { { "Microsoft.VisualBasic", "10.4.0" }, @@ -193,8 +195,6 @@ internal static class NETCoreApp90 { "System.Security.Cryptography.Xml", "8.0.2" }, }; - private static NuGetFramework Net90 { get; } = new NuGetFramework(FrameworkConstants.FrameworkIdentifiers.NetCoreApp, new Version(9, 0)); - internal static void Register() => FrameworkPackages.Register(Instance, AspNetCore, WindowsDesktop); } } diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetComponentDetector.cs index 87a335b4b..c4ccac918 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetComponentDetector.cs @@ -19,10 +19,10 @@ namespace Microsoft.ComponentDetection.Detectors.NuGet; public class NuGetComponentDetector : FileComponentDetector { - public const string NugetConfigFileName = "nuget.config"; - private static readonly IEnumerable LowConfidencePackages = ["Newtonsoft.Json"]; + public const string NugetConfigFileName = "nuget.config"; + private readonly IList repositoryPathKeyNames = ["repositorypath", "globalpackagesfolder"]; public NuGetComponentDetector( diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetProjectModelProjectCentricComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetProjectModelProjectCentricComponentDetector.cs index 20252538f..068283dd4 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetProjectModelProjectCentricComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetProjectModelProjectCentricComponentDetector.cs @@ -43,6 +43,46 @@ public NuGetProjectModelProjectCentricComponentDetector( public override int Version { get; } = 2; + private static string[] GetFrameworkReferences(LockFile lockFile, LockFileTarget target) + { + var frameworkInformation = lockFile.PackageSpec.TargetFrameworks.FirstOrDefault(x => x.FrameworkName.Equals(target.TargetFramework)); + + if (frameworkInformation == null) + { + return []; + } + + // add directly referenced frameworks + var results = frameworkInformation.FrameworkReferences.Select(x => x.Name); + + // add transitive framework references + results = results.Concat(target.Libraries.SelectMany(l => l.FrameworkReferences)); + + return results.Distinct().ToArray(); + } + + private static bool IsADevelopmentDependency(LockFileTargetLibrary library, LockFile lockFile) + { + // a placeholder item is an empty file that doesn't exist with name _._ meant to indicate an empty folder in a nuget package, but also used by NuGet when a package's assets are excluded. + static bool IsAPlaceholderItem(LockFileItem item) => Path.GetFileName(item.Path).Equals(PackagingCoreConstants.EmptyFolder, StringComparison.OrdinalIgnoreCase); + + // All(IsAPlaceholderItem) checks if the collection is empty or all items are placeholders. + return library.RuntimeAssemblies.All(IsAPlaceholderItem) && + library.RuntimeTargets.All(IsAPlaceholderItem) && + library.ResourceAssemblies.All(IsAPlaceholderItem) && + library.NativeLibraries.All(IsAPlaceholderItem) && + library.ContentFiles.All(IsAPlaceholderItem) && + library.Build.All(IsAPlaceholderItem) && + library.BuildMultiTargeting.All(IsAPlaceholderItem) && + + // The SDK looks at the library for analyzers using the following hueristic: + // https://github.com/dotnet/sdk/blob/d7fe6e66d8f67dc93c5c294a75f42a2924889196/src/Tasks/Microsoft.NET.Build.Tasks/NuGetUtils.NuGet.cs#L43 + (!lockFile.GetLibrary(library.Name, library.Version)?.Files + .Any(file => file.StartsWith("analyzers", StringComparison.Ordinal) + && file.EndsWith(".dll", StringComparison.OrdinalIgnoreCase) + && !file.EndsWith(".resources.dll", StringComparison.OrdinalIgnoreCase)) ?? false); + } + protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) { try @@ -95,46 +135,6 @@ bool IsFrameworkOrDevelopmentDependency(LockFileTargetLibrary library) => return Task.CompletedTask; } - private static string[] GetFrameworkReferences(LockFile lockFile, LockFileTarget target) - { - var frameworkInformation = lockFile.PackageSpec.TargetFrameworks.FirstOrDefault(x => x.FrameworkName.Equals(target.TargetFramework)); - - if (frameworkInformation == null) - { - return []; - } - - // add directly referenced frameworks - var results = frameworkInformation.FrameworkReferences.Select(x => x.Name); - - // add transitive framework references - results = results.Concat(target.Libraries.SelectMany(l => l.FrameworkReferences)); - - return results.Distinct().ToArray(); - } - - private static bool IsADevelopmentDependency(LockFileTargetLibrary library, LockFile lockFile) - { - // a placeholder item is an empty file that doesn't exist with name _._ meant to indicate an empty folder in a nuget package, but also used by NuGet when a package's assets are excluded. - static bool IsAPlaceholderItem(LockFileItem item) => Path.GetFileName(item.Path).Equals(PackagingCoreConstants.EmptyFolder, StringComparison.OrdinalIgnoreCase); - - // All(IsAPlaceholderItem) checks if the collection is empty or all items are placeholders. - return library.RuntimeAssemblies.All(IsAPlaceholderItem) && - library.RuntimeTargets.All(IsAPlaceholderItem) && - library.ResourceAssemblies.All(IsAPlaceholderItem) && - library.NativeLibraries.All(IsAPlaceholderItem) && - library.ContentFiles.All(IsAPlaceholderItem) && - library.Build.All(IsAPlaceholderItem) && - library.BuildMultiTargeting.All(IsAPlaceholderItem) && - - // The SDK looks at the library for analyzers using the following hueristic: - // https://github.com/dotnet/sdk/blob/d7fe6e66d8f67dc93c5c294a75f42a2924889196/src/Tasks/Microsoft.NET.Build.Tasks/NuGetUtils.NuGet.cs#L43 - (!lockFile.GetLibrary(library.Name, library.Version)?.Files - .Any(file => file.StartsWith("analyzers", StringComparison.Ordinal) - && file.EndsWith(".dll", StringComparison.OrdinalIgnoreCase) - && !file.EndsWith(".resources.dll", StringComparison.OrdinalIgnoreCase)) ?? false); - } - private void NavigateAndRegister( LockFileTarget target, HashSet explicitlyReferencedComponentIds, diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/Contracts/PipDependencySpecification.cs b/src/Microsoft.ComponentDetection.Detectors/pip/Contracts/PipDependencySpecification.cs index 7c111bce6..0fd941d75 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/Contracts/PipDependencySpecification.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/Contracts/PipDependencySpecification.cs @@ -91,6 +91,64 @@ public PipDependencySpecification(ILogger logger, string packageString, bool req private ILogger Logger { get; set; } + private void Initialize(string packageString, bool requiresDist) + { + if (requiresDist) + { + var distMatch = RequiresDistRegex.Match(packageString); + + for (var i = 1; i < distMatch.Groups.Count; i++) + { + if (string.IsNullOrWhiteSpace(distMatch.Groups[i].Value)) + { + continue; + } + + if (string.IsNullOrWhiteSpace(this.Name)) + { + this.Name = distMatch.Groups[i].Value.Trim(); + } + else + { + this.DependencySpecifiers = distMatch.Groups[i].Value.Split(','); + } + } + + var conditionalDependenciesMatches = RequiresDistConditionalDependenciesMatch.Matches(packageString); + + for (var i = 0; i < conditionalDependenciesMatches.Count; i++) + { + if (!string.IsNullOrWhiteSpace(conditionalDependenciesMatches[i].Value)) + { + this.ConditionalDependencySpecifiers.Add(conditionalDependenciesMatches[i].Value); + } + } + } + else + { + var nameMatches = PipNameExtractionRegex.Match(packageString); + var versionMatches = PipVersionExtractionRegex.Match(packageString); + + if (nameMatches.Captures.Count > 0) + { + this.Name = nameMatches.Captures[0].Value; + } + else + { + this.Name = packageString; + } + + if (versionMatches.Captures.Count > 0) + { + this.DependencySpecifiers = versionMatches.Captures[0].Value.Split(','); + } + } + + this.DependencySpecifiers = this.DependencySpecifiers.Where(x => !x.Contains("python_version")) + .Select(x => x.Trim()) + .ToList(); + } + /// /// Whether or not the package is safe to resolve based on the packagesToIgnore. /// @@ -214,62 +272,4 @@ public bool IsValidParentPackage(Dictionary pythonEnvironmentVar !this.PackageIsUnsafe() && this.PackageConditionsMet(pythonEnvironmentVariables) && !this.ConditionalDependencySpecifiers.Any(s => s.Contains("extra ==", StringComparison.OrdinalIgnoreCase)); - - private void Initialize(string packageString, bool requiresDist) - { - if (requiresDist) - { - var distMatch = RequiresDistRegex.Match(packageString); - - for (var i = 1; i < distMatch.Groups.Count; i++) - { - if (string.IsNullOrWhiteSpace(distMatch.Groups[i].Value)) - { - continue; - } - - if (string.IsNullOrWhiteSpace(this.Name)) - { - this.Name = distMatch.Groups[i].Value.Trim(); - } - else - { - this.DependencySpecifiers = distMatch.Groups[i].Value.Split(','); - } - } - - var conditionalDependenciesMatches = RequiresDistConditionalDependenciesMatch.Matches(packageString); - - for (var i = 0; i < conditionalDependenciesMatches.Count; i++) - { - if (!string.IsNullOrWhiteSpace(conditionalDependenciesMatches[i].Value)) - { - this.ConditionalDependencySpecifiers.Add(conditionalDependenciesMatches[i].Value); - } - } - } - else - { - var nameMatches = PipNameExtractionRegex.Match(packageString); - var versionMatches = PipVersionExtractionRegex.Match(packageString); - - if (nameMatches.Captures.Count > 0) - { - this.Name = nameMatches.Captures[0].Value; - } - else - { - this.Name = packageString; - } - - if (versionMatches.Captures.Count > 0) - { - this.DependencySpecifiers = versionMatches.Captures[0].Value.Split(','); - } - } - - this.DependencySpecifiers = this.DependencySpecifiers.Where(x => !x.Contains("python_version")) - .Select(x => x.Trim()) - .ToList(); - } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/IPyPiClient.cs b/src/Microsoft.ComponentDetection.Detectors/pip/IPyPiClient.cs index dc4829f72..29b89122a 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/IPyPiClient.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/IPyPiClient.cs @@ -235,13 +235,6 @@ public async Task GetProjectAsync(PipDependencySpecification spec return versions; } - public void Dispose() - { - this.cacheTelemetry.FinalCacheSize = this.cachedResponses.Count; - this.cacheTelemetry.Dispose(); - this.cachedResponses.Dispose(); - } - /// /// Returns a cached response if it exists, otherwise returns the response from PyPi REST call. /// The response from PyPi is automatically added to the cache. @@ -292,4 +285,11 @@ private void InitializeNonDefaultMemoryCache() this.checkedMaxEntriesVariable = true; } + + public void Dispose() + { + this.cacheTelemetry.FinalCacheSize = this.cachedResponses.Count; + this.cacheTelemetry.Dispose(); + this.cachedResponses.Dispose(); + } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PipCommandService.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PipCommandService.cs index 2bb874921..8d96944b5 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/PipCommandService.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/PipCommandService.cs @@ -76,6 +76,50 @@ public async Task GetPipVersionAsync(string pipPath = null, string pyth } } + private async Task<(string PipExectuable, string PythonExecutable)> ResolvePipAsync(string pipPath = null, string pythonPath = null) + { + var pipCommand = string.IsNullOrEmpty(pipPath) ? "pip" : pipPath; + var pythonCommand = string.IsNullOrEmpty(pythonPath) ? "python" : pythonPath; + + if (await this.CanCommandBeLocatedAsync(pipCommand)) + { + return (pipCommand, null); + } + else if (await this.commandLineInvocationService.CanCommandBeLocatedAsync(pythonCommand, null, "-m", "pip", "--version")) + { + return (null, pythonCommand); + } + + return (null, null); + } + + private async Task CanCommandBeLocatedAsync(string pipPath) + { + return await this.commandLineInvocationService.CanCommandBeLocatedAsync(pipPath, ["pip3"], "--version"); + } + + private async Task ExecuteCommandAsync( + string pipExecutable = null, + string pythonExecutable = null, + IEnumerable additionalCandidateCommands = null, + DirectoryInfo workingDirectory = null, + CancellationToken cancellationToken = default, + params string[] parameters) + { + if (!string.IsNullOrEmpty(pipExecutable)) + { + return await this.commandLineInvocationService.ExecuteCommandAsync( + pipExecutable, additionalCandidateCommands, workingDirectory, cancellationToken, parameters); + } + else + { + var pythonPipParams = new[] { "-m", "pip" }; + var parametersFull = pythonPipParams.Concat(parameters).ToArray(); + return await this.commandLineInvocationService.ExecuteCommandAsync( + pythonExecutable, additionalCandidateCommands, workingDirectory, cancellationToken, parametersFull); + } + } + public async Task<(PipInstallationReport Report, FileInfo ReportFile)> GenerateInstallationReportAsync( string path, string pipExePath = null, string pythonExePath = null, CancellationToken cancellationToken = default) { @@ -181,48 +225,4 @@ public async Task GetPipVersionAsync(string pipPath = null, string pyth } } } - - private async Task<(string PipExectuable, string PythonExecutable)> ResolvePipAsync(string pipPath = null, string pythonPath = null) - { - var pipCommand = string.IsNullOrEmpty(pipPath) ? "pip" : pipPath; - var pythonCommand = string.IsNullOrEmpty(pythonPath) ? "python" : pythonPath; - - if (await this.CanCommandBeLocatedAsync(pipCommand)) - { - return (pipCommand, null); - } - else if (await this.commandLineInvocationService.CanCommandBeLocatedAsync(pythonCommand, null, "-m", "pip", "--version")) - { - return (null, pythonCommand); - } - - return (null, null); - } - - private async Task CanCommandBeLocatedAsync(string pipPath) - { - return await this.commandLineInvocationService.CanCommandBeLocatedAsync(pipPath, ["pip3"], "--version"); - } - - private async Task ExecuteCommandAsync( - string pipExecutable = null, - string pythonExecutable = null, - IEnumerable additionalCandidateCommands = null, - DirectoryInfo workingDirectory = null, - CancellationToken cancellationToken = default, - params string[] parameters) - { - if (!string.IsNullOrEmpty(pipExecutable)) - { - return await this.commandLineInvocationService.ExecuteCommandAsync( - pipExecutable, additionalCandidateCommands, workingDirectory, cancellationToken, parameters); - } - else - { - var pythonPipParams = new[] { "-m", "pip" }; - var parametersFull = pythonPipParams.Concat(parameters).ToArray(); - return await this.commandLineInvocationService.ExecuteCommandAsync( - pythonExecutable, additionalCandidateCommands, workingDirectory, cancellationToken, parametersFull); - } - } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PipReportUtilities.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PipReportUtilities.cs index a3802e9fa..1495f752f 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/PipReportUtilities.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/PipReportUtilities.cs @@ -6,16 +6,16 @@ namespace Microsoft.ComponentDetection.Detectors.Pip; internal class PipReportUtilities { + private const int MaxLicenseFieldLength = 100; + private const string ClassifierFieldSeparator = " :: "; + private const string ClassifierFieldLicensePrefix = "License"; + // Python regular expression for version schema: // https://www.python.org/dev/peps/pep-0440/#appendix-b-parsing-version-strings-with-regular-expressions public static readonly Regex CanonicalVersionPatternMatch = new Regex( @"^([1-9]\d*!)?(0|[1-9]\d*)(\.(0|[1-9]\d*))*((a|b|rc)(0|[1-9]\d*))?(\.post(0|[1-9]\d*))?(\.dev(0|[1-9]\d*))?(\+(?:(?[a-z0-9]+(?:[.][a-z0-9]+)*))?)?$", RegexOptions.Compiled); - private const int MaxLicenseFieldLength = 100; - private const string ClassifierFieldSeparator = " :: "; - private const string ClassifierFieldLicensePrefix = "License"; - /// /// Normalize the package name format to the standard Python Packaging format. /// See https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization. diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PythonCommandService.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PythonCommandService.cs index d824f1ea9..962003f9c 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/PythonCommandService.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/PythonCommandService.cs @@ -59,22 +59,6 @@ public async Task PythonExistsAsync(string pythonPath = null) } } - public async Task GetPythonVersionAsync(string pythonPath) - { - var pythonCommand = await this.ResolvePythonAsync(pythonPath); - var versionResult = await this.commandLineInvocationService.ExecuteCommandAsync(pythonCommand, ["python3", "python2"], "--version"); - var version = new Regex("Python ([\\d.]+)"); - var match = version.Match(versionResult.StdOut); - return match.Success ? match.Groups[1].Value : null; - } - - public async Task GetOsTypeAsync(string pythonPath) - { - var pythonCommand = await this.ResolvePythonAsync(pythonPath); - var versionResult = await this.commandLineInvocationService.ExecuteCommandAsync(pythonCommand, ["python3", "python2"], "-c", "\"import sys; print(sys.platform);\""); - return versionResult.ExitCode == 0 && string.IsNullOrEmpty(versionResult.StdErr) ? versionResult.StdOut.Trim() : null; - } - private async Task> ParseSetupPyFileAsync(string filePath, string pythonExePath = null) { var pythonExecutable = await this.ResolvePythonAsync(pythonExePath); @@ -189,4 +173,20 @@ private async Task CanCommandBeLocatedAsync(string pythonPath) { return await this.commandLineInvocationService.CanCommandBeLocatedAsync(pythonPath, ["python3", "python2"], "--version"); } + + public async Task GetPythonVersionAsync(string pythonPath) + { + var pythonCommand = await this.ResolvePythonAsync(pythonPath); + var versionResult = await this.commandLineInvocationService.ExecuteCommandAsync(pythonCommand, ["python3", "python2"], "--version"); + var version = new Regex("Python ([\\d.]+)"); + var match = version.Match(versionResult.StdOut); + return match.Success ? match.Groups[1].Value : null; + } + + public async Task GetOsTypeAsync(string pythonPath) + { + var pythonCommand = await this.ResolvePythonAsync(pythonPath); + var versionResult = await this.commandLineInvocationService.ExecuteCommandAsync(pythonCommand, ["python3", "python2"], "-c", "\"import sys; print(sys.platform);\""); + return versionResult.ExitCode == 0 && string.IsNullOrEmpty(versionResult.StdErr) ? versionResult.StdOut.Trim() : null; + } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs index 124279cee..1dd7c34f2 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs @@ -79,32 +79,6 @@ public async Task> ResolveRootsAsync(ISingleFileComponentRec return await this.ProcessQueueAsync(singleFileComponentRecorder, state) ?? []; } - public void SetPythonEnvironmentVariable(string key, string value) - { - this.pythonEnvironmentVariables[key] = value; - } - - public Dictionary GetPythonEnvironmentVariables() - { - return this.pythonEnvironmentVariables; - } - - protected override async Task> FetchPackageDependenciesAsync( - PythonResolverState state, - PipDependencySpecification spec) - { - var candidateVersion = state.NodeReferences[spec.Name].Value.Version; - - var packageToFetch = state.ValidVersionMap[spec.Name][candidateVersion].FirstOrDefault(x => string.Equals("bdist_wheel", x.PackageType, StringComparison.OrdinalIgnoreCase)) ?? - state.ValidVersionMap[spec.Name][candidateVersion].FirstOrDefault(x => string.Equals("bdist_egg", x.PackageType, StringComparison.OrdinalIgnoreCase)); - if (packageToFetch == null) - { - return []; - } - - return await this.pypiClient.FetchPackageDependenciesAsync(spec.Name, candidateVersion, packageToFetch); - } - private async Task> ProcessQueueAsync(ISingleFileComponentRecorder singleFileComponentRecorder, PythonResolverState state) { while (state.ProcessingQueue.Count > 0) @@ -180,6 +154,22 @@ private async Task> ProcessQueueAsync(ISingleFileComponentRe return state.Roots; } + protected override async Task> FetchPackageDependenciesAsync( + PythonResolverState state, + PipDependencySpecification spec) + { + var candidateVersion = state.NodeReferences[spec.Name].Value.Version; + + var packageToFetch = state.ValidVersionMap[spec.Name][candidateVersion].FirstOrDefault(x => string.Equals("bdist_wheel", x.PackageType, StringComparison.OrdinalIgnoreCase)) ?? + state.ValidVersionMap[spec.Name][candidateVersion].FirstOrDefault(x => string.Equals("bdist_egg", x.PackageType, StringComparison.OrdinalIgnoreCase)); + if (packageToFetch == null) + { + return []; + } + + return await this.pypiClient.FetchPackageDependenciesAsync(spec.Name, candidateVersion, packageToFetch); + } + private void AddGraphNode(PythonResolverState state, PipGraphNode parent, string name, string version, string license = null, string author = null) { if (state.NodeReferences.TryGetValue(name, out var value)) @@ -242,4 +232,14 @@ private string GetLicenseFromProject(PythonProject project) return null; } + + public void SetPythonEnvironmentVariable(string key, string value) + { + this.pythonEnvironmentVariables[key] = value; + } + + public Dictionary GetPythonEnvironmentVariables() + { + return this.pythonEnvironmentVariables; + } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolverBase.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolverBase.cs index 3ef861601..fe44081c0 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolverBase.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolverBase.cs @@ -15,32 +15,6 @@ public abstract class PythonResolverBase internal PythonResolverBase(ILogger logger) => this.logger = logger; - /// - /// Multiple dependency specification versions can be given for a single package name. - /// Until a better method is devised, choose the latest entry. - /// See https://github.com/microsoft/component-detection/issues/963. - /// - /// Dictionary of package names to dependency version specifiers. - public Dictionary ResolveDependencySpecifications(PipComponent component, IList fetchedDependences) - { - var dependencies = new Dictionary(); - fetchedDependences.ForEach(d => - { - if (!dependencies.TryAdd(d.Name, d)) - { - this.logger.LogWarning( - "Duplicate package dependencies entry for component:{ComponentName} with dependency:{DependencyName}. Existing dependency specifiers: {ExistingSpecifiers}. New dependency specifiers: {NewSpecifiers}.", - component.Name, - d.Name, - JsonConvert.SerializeObject(dependencies[d.Name].DependencySpecifiers), - JsonConvert.SerializeObject(d.DependencySpecifiers)); - dependencies[d.Name] = d; - } - }); - - return dependencies; - } - /// /// Given a state, node, and new spec, will reprocess a new valid version for the node. /// @@ -102,6 +76,32 @@ protected async Task InvalidateAndReprocessAsync( return true; } + /// + /// Multiple dependency specification versions can be given for a single package name. + /// Until a better method is devised, choose the latest entry. + /// See https://github.com/microsoft/component-detection/issues/963. + /// + /// Dictionary of package names to dependency version specifiers. + public Dictionary ResolveDependencySpecifications(PipComponent component, IList fetchedDependences) + { + var dependencies = new Dictionary(); + fetchedDependences.ForEach(d => + { + if (!dependencies.TryAdd(d.Name, d)) + { + this.logger.LogWarning( + "Duplicate package dependencies entry for component:{ComponentName} with dependency:{DependencyName}. Existing dependency specifiers: {ExistingSpecifiers}. New dependency specifiers: {NewSpecifiers}.", + component.Name, + d.Name, + JsonConvert.SerializeObject(dependencies[d.Name].DependencySpecifiers), + JsonConvert.SerializeObject(d.DependencySpecifiers)); + dependencies[d.Name] = d; + } + }); + + return dependencies; + } + protected abstract Task> FetchPackageDependenciesAsync( PythonResolverState state, PipDependencySpecification spec); diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PythonVersionUtilities.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PythonVersionUtilities.cs index d933113d7..ede50da9b 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/PythonVersionUtilities.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/PythonVersionUtilities.cs @@ -84,23 +84,6 @@ public static bool CheckEquality(string version, string specVer, bool fuzzy = fa } } - public static (string Operator, string Version) ParseSpec(string spec) - { - var opChars = new char[] { '=', '<', '>', '~', '!' }; - var specArray = spec.ToCharArray(); - - var i = 0; - while (i < spec.Length && i < 3 && opChars.Contains(specArray[i])) - { - i++; - } - - var op = spec[..i]; - var specVerSection = spec[i..].Trim(); - - return (op, specVerSection); - } - private static bool VersionValidForSpec(string version, string spec) { (var op, var specVersion) = ParseSpec(spec); @@ -131,4 +114,21 @@ private static bool VersionValidForSpec(string version, string spec) _ => false, }; } + + public static (string Operator, string Version) ParseSpec(string spec) + { + var opChars = new char[] { '=', '<', '>', '~', '!' }; + var specArray = spec.ToCharArray(); + + var i = 0; + while (i < spec.Length && i < 3 && opChars.Contains(specArray[i])) + { + i++; + } + + var op = spec[..i]; + var specVerSection = spec[i..].Trim(); + + return (op, specVerSection); + } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/SimplePypiClient.cs b/src/Microsoft.ComponentDetection.Detectors/pip/SimplePypiClient.cs index 9407d9df9..52be242f9 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/SimplePypiClient.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/SimplePypiClient.cs @@ -18,13 +18,13 @@ namespace Microsoft.ComponentDetection.Detectors.Pip; public sealed class SimplePyPiClient : ISimplePyPiClient, IDisposable { - // max number of retries allowed, to cap the total delay period - public const long MAXRETRIES = 15; - // Values used for cache creation private const long CACHEINTERVALSECONDS = 180; private const long DEFAULTCACHEENTRIES = 4096; + // max number of retries allowed, to cap the total delay period + public const long MAXRETRIES = 15; + private static readonly ProductInfoHeaderValue ProductValue = new( "ComponentDetection", Assembly.GetEntryAssembly().GetCustomAttribute()?.InformationalVersion); @@ -85,16 +85,6 @@ public async Task FetchPackageFileStreamAsync(Uri releaseUrl) return projectStream; } - public void Dispose() - { - this.cacheTelemetry.FinalSimpleProjectCacheSize = this.cachedSimplePyPiProjects.Count; - this.cacheTelemetry.FinalProjectFileCacheSize = this.cachedProjectWheelFiles.Count; - this.cacheTelemetry.Dispose(); - this.cachedProjectWheelFiles.Dispose(); - this.cachedSimplePyPiProjects.Dispose(); - HttpClient.Dispose(); - } - /// /// Returns a cached response if it exists, otherwise returns the response from PyPi REST call. /// The response from PyPi is automatically added to the cache. @@ -284,4 +274,14 @@ private async Task GetPypiResponseAsync(Uri uri) var response = await HttpClient.SendAsync(request); return response; } + + public void Dispose() + { + this.cacheTelemetry.FinalSimpleProjectCacheSize = this.cachedSimplePyPiProjects.Count; + this.cacheTelemetry.FinalProjectFileCacheSize = this.cachedProjectWheelFiles.Count; + this.cacheTelemetry.Dispose(); + this.cachedProjectWheelFiles.Dispose(); + this.cachedSimplePyPiProjects.Dispose(); + HttpClient.Dispose(); + } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/SimplePythonResolver.cs b/src/Microsoft.ComponentDetection.Detectors/pip/SimplePythonResolver.cs index e60e6a5e6..6a6f6c5ed 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/SimplePythonResolver.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/SimplePythonResolver.cs @@ -33,6 +33,60 @@ public SimplePythonResolver(ISimplePyPiClient simplePypiClient, ILogger + /// Uses regex to extract the version from the file name. + /// + /// the name of the file from simple pypi. + /// returns a string representing the release version. + private static string GetVersionFromFileName(string fileName) + { + var version = VersionRegex.Match(fileName).Groups[1]; + return version.Value; + } + + /// + /// Returns the package type based on the file name. + /// + /// the name of the file from simple pypi. + /// a string representing the package type. + private static string GetPackageType(string fileName) + { + if (fileName.EndsWith(".whl")) + { + return "bdist_wheel"; + } + + if (fileName.EndsWith(".tar.gz")) + { + return "sdist"; + } + + return fileName.EndsWith(".egg") ? "bdist_egg" : string.Empty; + } + + /// + /// Adds a node to the graph. + /// + /// The PythonResolverState. + /// The parent node. + /// The package name. + /// The package version. + private static void AddGraphNode(PythonResolverState state, PipGraphNode parent, string name, string version) + { + if (state.NodeReferences.TryGetValue(name, out var value)) + { + parent.Children.Add(value); + value.Parents.Add(parent); + } + else + { + var node = new PipGraphNode(new PipComponent(name, version)); + state.NodeReferences[name] = node; + parent.Children.Add(node); + node.Parents.Add(parent); + } + } + /// public async Task> ResolveRootsAsync(ISingleFileComponentRecorder singleFileComponentRecorder, IList initialPackages) { @@ -97,89 +151,6 @@ await Parallel.ForEachAsync(initialPackages, async (rootPackage, ct) => return await this.ProcessQueueAsync(singleFileComponentRecorder, state) ?? []; } - /// - /// Fetches the dependencies for a package. - /// - /// The PythonResolverState. - /// The PipDependencySpecification. - /// Returns a list of PipDependencySpecification. - protected override async Task> FetchPackageDependenciesAsync( - PythonResolverState state, - PipDependencySpecification spec) - { - var candidateVersion = state.NodeReferences[spec.Name].Value.Version; - - var packageToFetch = state.ValidVersionMap[spec.Name][candidateVersion].FirstOrDefault(x => string.Equals("bdist_wheel", x.PackageType, StringComparison.OrdinalIgnoreCase)) ?? - state.ValidVersionMap[spec.Name][candidateVersion].FirstOrDefault(x => string.Equals("bdist_egg", x.PackageType, StringComparison.OrdinalIgnoreCase)); - if (packageToFetch == null) - { - return []; - } - - var packageFileStream = await this.simplePypiClient.FetchPackageFileStreamAsync(packageToFetch.Url); - - if (packageFileStream.Length == 0) - { - return []; - } - - return await this.FetchDependenciesFromPackageStreamAsync(spec.Name, candidateVersion, packageFileStream); - } - - /// - /// Uses regex to extract the version from the file name. - /// - /// the name of the file from simple pypi. - /// returns a string representing the release version. - private static string GetVersionFromFileName(string fileName) - { - var version = VersionRegex.Match(fileName).Groups[1]; - return version.Value; - } - - /// - /// Returns the package type based on the file name. - /// - /// the name of the file from simple pypi. - /// a string representing the package type. - private static string GetPackageType(string fileName) - { - if (fileName.EndsWith(".whl")) - { - return "bdist_wheel"; - } - - if (fileName.EndsWith(".tar.gz")) - { - return "sdist"; - } - - return fileName.EndsWith(".egg") ? "bdist_egg" : string.Empty; - } - - /// - /// Adds a node to the graph. - /// - /// The PythonResolverState. - /// The parent node. - /// The package name. - /// The package version. - private static void AddGraphNode(PythonResolverState state, PipGraphNode parent, string name, string version) - { - if (state.NodeReferences.TryGetValue(name, out var value)) - { - parent.Children.Add(value); - value.Parents.Add(parent); - } - else - { - var node = new PipGraphNode(new PipComponent(name, version)); - state.NodeReferences[name] = node; - parent.Children.Add(node); - node.Parents.Add(parent); - } - } - private async Task> ProcessQueueAsync(ISingleFileComponentRecorder singleFileComponentRecorder, PythonResolverState state) { while (state.ProcessingQueue.Count > 0) @@ -309,6 +280,35 @@ private SortedDictionary> ConvertSimplePypiP return sortedProjectVersions; } + /// + /// Fetches the dependencies for a package. + /// + /// The PythonResolverState. + /// The PipDependencySpecification. + /// Returns a list of PipDependencySpecification. + protected override async Task> FetchPackageDependenciesAsync( + PythonResolverState state, + PipDependencySpecification spec) + { + var candidateVersion = state.NodeReferences[spec.Name].Value.Version; + + var packageToFetch = state.ValidVersionMap[spec.Name][candidateVersion].FirstOrDefault(x => string.Equals("bdist_wheel", x.PackageType, StringComparison.OrdinalIgnoreCase)) ?? + state.ValidVersionMap[spec.Name][candidateVersion].FirstOrDefault(x => string.Equals("bdist_egg", x.PackageType, StringComparison.OrdinalIgnoreCase)); + if (packageToFetch == null) + { + return []; + } + + var packageFileStream = await this.simplePypiClient.FetchPackageFileStreamAsync(packageToFetch.Url); + + if (packageFileStream.Length == 0) + { + return []; + } + + return await this.FetchDependenciesFromPackageStreamAsync(spec.Name, candidateVersion, packageFileStream); + } + /// /// Given a package stream will unzip and return the dependencies in the metadata file. /// diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/Parsers/RustCargoLockParser.cs b/src/Microsoft.ComponentDetection.Detectors/rust/Parsers/RustCargoLockParser.cs index a2a1407af..a7efc2270 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/Parsers/RustCargoLockParser.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/Parsers/RustCargoLockParser.cs @@ -38,6 +38,8 @@ public class RustCargoLockParser : IRustCargoLockParser /// The logger. public RustCargoLockParser(ILogger logger) => this.logger = logger; + private static bool IsLocalPackage(CargoPackage package) => package.Source == null; + /// /// Parses a Cargo.lock file and records components. /// @@ -65,8 +67,6 @@ public class RustCargoLockParser : IRustCargoLockParser } } - private static bool IsLocalPackage(CargoPackage package) => package.Source == null; - private void ProcessCargoLock(CargoLock cargoLock, ISingleFileComponentRecorder singleFileComponentRecorder, IComponentStream cargoLockFile) { try diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/Parsers/RustSbomParser.cs b/src/Microsoft.ComponentDetection.Detectors/rust/Parsers/RustSbomParser.cs index 5868d7852..b377ef664 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/Parsers/RustSbomParser.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/Parsers/RustSbomParser.cs @@ -61,43 +61,6 @@ public class RustSbomParser : IRustSbomParser } } - /// - /// Parses a Cargo SBOM file and registers each discovered component against all owning Cargo.toml recorders - /// using the provided ownership map (cargo metadata package id -> set of manifest paths). - /// Falls back to the supplied sbomRecorder when ownership info is absent. - /// - /// SBOM stream. - /// Recorder tied to the SBOM file (fallback target). - /// Root component recorder used to create (or reuse) per-manifest recorders. - /// Package ownership map from RustMetadataContextBuilder (may be null). - /// Cancellation token. - /// SBOM version or null on failure. - public async Task ParseWithOwnershipAsync( - IComponentStream componentStream, - ISingleFileComponentRecorder sbomRecorder, - IComponentRecorder parentComponentRecorder, - IReadOnlyDictionary> ownershipMap, - CancellationToken cancellationToken = default) - { - try - { - using var reader = new StreamReader(componentStream.Stream); - var cargoSbom = CargoSbom.FromJson(await reader.ReadToEndAsync(cancellationToken)); - this.ProcessCargoSbomWithOwnership( - cargoSbom, - componentStream, - sbomRecorder, - parentComponentRecorder, - ownershipMap); - return cargoSbom.Version; - } - catch (Exception e) - { - this.logger.LogError(e, "Failed to parse Cargo SBOM (ownership mode) '{FileLocation}'", componentStream.Location); - return null; - } - } - private void ProcessCargoSbom(CargoSbom sbom, ISingleFileComponentRecorder recorder, IComponentStream components) { try @@ -150,6 +113,43 @@ private void ProcessDependency( } } + /// + /// Parses a Cargo SBOM file and registers each discovered component against all owning Cargo.toml recorders + /// using the provided ownership map (cargo metadata package id -> set of manifest paths). + /// Falls back to the supplied sbomRecorder when ownership info is absent. + /// + /// SBOM stream. + /// Recorder tied to the SBOM file (fallback target). + /// Root component recorder used to create (or reuse) per-manifest recorders. + /// Package ownership map from RustMetadataContextBuilder (may be null). + /// Cancellation token. + /// SBOM version or null on failure. + public async Task ParseWithOwnershipAsync( + IComponentStream componentStream, + ISingleFileComponentRecorder sbomRecorder, + IComponentRecorder parentComponentRecorder, + IReadOnlyDictionary> ownershipMap, + CancellationToken cancellationToken = default) + { + try + { + using var reader = new StreamReader(componentStream.Stream); + var cargoSbom = CargoSbom.FromJson(await reader.ReadToEndAsync(cancellationToken)); + this.ProcessCargoSbomWithOwnership( + cargoSbom, + componentStream, + sbomRecorder, + parentComponentRecorder, + ownershipMap); + return cargoSbom.Version; + } + catch (Exception e) + { + this.logger.LogError(e, "Failed to parse Cargo SBOM (ownership mode) '{FileLocation}'", componentStream.Location); + return null; + } + } + private void ProcessCargoSbomWithOwnership( CargoSbom sbom, IComponentStream sbomStream, diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs index b75d8f802..10561cd28 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs @@ -17,8 +17,8 @@ public static class DetectorExperiments /// public static bool Enable { get; set; } - internal static bool AreExperimentsEnabled => Enable || EnvironmentEnabled; - private static bool EnvironmentEnabled => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("CD_DETECTOR_EXPERIMENTS")); + + internal static bool AreExperimentsEnabled => Enable || EnvironmentEnabled; } diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs index b8b2c0dde..4468199f0 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs @@ -125,6 +125,19 @@ public void RecordDetectorRun( } } + private void FilterExperiments(IComponentDetector detector, int count) + { + var experimentsToRemove = this.experiments + .Where(x => !x.Key.ShouldRecord(detector, count)) + .Select(x => x.Key) + .ToList(); + + foreach (var config in experimentsToRemove.Where(config => this.experiments.TryRemove(config, out _))) + { + this.logger.LogDebug("Removing {Experiment} from active experiments", config.Name); + } + } + public void RemoveUnwantedExperimentsbyDetectors(IEnumerable detectors) { if (detectors == null) @@ -188,17 +201,4 @@ public async Task FinishAsync() } } } - - private void FilterExperiments(IComponentDetector detector, int count) - { - var experimentsToRemove = this.experiments - .Where(x => !x.Key.ShouldRecord(detector, count)) - .Select(x => x.Key) - .ToList(); - - foreach (var config in experimentsToRemove.Where(config => this.experiments.TryRemove(config, out _))) - { - this.logger.LogDebug("Removing {Experiment} from active experiments", config.Name); - } - } } diff --git a/src/Microsoft.ComponentDetection.Orchestrator/LoggingEnricher.cs b/src/Microsoft.ComponentDetection.Orchestrator/LoggingEnricher.cs index 2f8a8e17c..3b1d153e0 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/LoggingEnricher.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/LoggingEnricher.cs @@ -13,15 +13,13 @@ public class LoggingEnricher : ILogEventEnricher /// The name of the log file path property. /// public const string LogFilePathPropertyName = "LogFilePath"; + private string cachedLogFilePath; + private LogEventProperty cachedLogFilePathProperty; /// /// The name of the print stderr property. /// public const string PrintStderrPropertyName = "PrintStderr"; - - private string cachedLogFilePath; - private LogEventProperty cachedLogFilePathProperty; - private bool? cachedPrintStderr; private LogEventProperty cachedPrintStderrProperty; diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs index a704e59e9..d4941512a 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs @@ -73,6 +73,106 @@ public DotNetComponentDetectorTests() .Returns((string c, IEnumerable ac, DirectoryInfo d, CancellationToken ct, string[] args) => Task.FromResult(this.CommandResult(c, d))); } + private bool FileExists(string path) + { + var fileName = Path.GetFileName(path); + var directory = Path.GetDirectoryName(path); + + return this.files.TryGetValue(directory, out var fileNames) && + fileNames.TryGetValue(fileName, out _); + } + + private Stream OpenFile(string path) + { + var fileName = Path.GetFileName(path); + var directory = Path.GetDirectoryName(path); + + return this.files.TryGetValue(directory, out var fileNames) && + fileNames.TryGetValue(fileName, out var stream) ? stream : null; + } + + private bool DirectoryExists(string directory) => this.files.ContainsKey(directory); + + private IEnumerable EnumerateFilesRecursive(string directory, string pattern) + { + if (this.files.TryGetValue(directory, out var fileNames)) + { + // a basic approximation of globbing + var patternRegex = new Regex(pattern.Replace(".", "\\.").Replace("*", ".*")); + + foreach (var fileName in fileNames.Keys) + { + var filePath = Path.Combine(directory, fileName); + + if (fileName.EndsWith(Path.DirectorySeparatorChar)) + { + foreach (var subFile in this.EnumerateFilesRecursive(Path.TrimEndingDirectorySeparator(filePath), pattern)) + { + yield return subFile; + } + } + else + { + if (patternRegex.IsMatch(fileName)) + { + yield return filePath; + } + } + } + } + } + + private void AddFile(string path, Stream content) + { + var fileName = Path.GetFileName(path); + var directory = Path.GetDirectoryName(path); + this.AddDirectory(directory); + this.files[directory][fileName] = content; + } + + private void AddDirectory(string path, string subDirectory = null) + { + if (string.IsNullOrEmpty(path)) + { + return; + } + + if (subDirectory is not null) + { + // use a trailing slash to indicate a sub directory in the files collection + subDirectory += Path.DirectorySeparatorChar; + } + + if (this.files.TryGetValue(path, out var directoryFiles)) + { + if (subDirectory is not null) + { + directoryFiles.Add(subDirectory, null); + } + } + else + { + this.files.Add(path, subDirectory is null ? [] : new() { { subDirectory, null } }); + this.AddDirectory(Path.GetDirectoryName(path), Path.GetFileName(path)); + } + } + + private void SetCommandResult(int exitCode, string stdOut = null, string stdErr = null) + { + this.commandLineCallback = null; + this.commandLineExecutionResult.ExitCode = exitCode; + this.commandLineExecutionResult.StdOut = stdOut; + this.commandLineExecutionResult.StdErr = stdErr; + } + + private void SetCommandResult(Func callback) + { + this.commandLineCallback = callback; + } + + private CommandLineExecutionResult CommandResult(string command, DirectoryInfo directory) => + (this.commandLineCallback != null) ? this.commandLineCallback(command, directory) : this.commandLineExecutionResult; + [TestCleanup] public void ClearMocks() { @@ -80,6 +180,63 @@ public void ClearMocks() this.SetCommandResult(-1); } + private static string ProjectAssets(string projectName, string outputPath, string projectPath, params string[] targetFrameworks) + { + LockFileFormat format = new(); + LockFile lockFile = new(); + using var textWriter = new StringWriter(); + + // assets file always includes a trailing separator + if (!Path.EndsInDirectorySeparator(outputPath)) + { + outputPath += Path.DirectorySeparatorChar; + } + + lockFile.Targets = targetFrameworks.Select(tfm => new LockFileTarget() { TargetFramework = NuGetFramework.Parse(tfm) }).ToList(); + lockFile.PackageSpec = new() + { + RestoreMetadata = new() + { + ProjectName = projectName, + OutputPath = outputPath, + ProjectPath = projectPath, + }, + }; + + format.Write(textWriter, lockFile); + return textWriter.ToString(); + } + + private static Stream GlobalJson(string sdkVersion) + { + var stream = new MemoryStream(); + using (var writer = new Utf8JsonWriter(stream, new() { Indented = true })) + { + writer.WriteStartObject(); + writer.WritePropertyName("sdk"); + writer.WriteStartObject(); + writer.WriteString("version", sdkVersion); + writer.WriteEndObject(); + writer.WriteEndObject(); + } + + stream.Position = 0; + return stream; + } + + private static Stream StreamFromString(string content) + { + var stream = new MemoryStream(); + using (var writer = new StreamWriter(stream, leaveOpen: true)) + { + writer.Write(content); + writer.Flush(); + stream.Position = 0; + } + + return stream; + } + [TestMethod] public async Task TestDotNetDetectorWithNoFiles_ReturnsSuccessfullyAsync() { @@ -588,161 +745,4 @@ public async Task TestDotNetDetectorRebasePaths(string additionalPathSegment) discoveredComponents.Where(component => component.Component.Id == "4.5.6 net6.0 library - DotNet").Should().ContainSingle(); discoveredComponents.Where(component => component.Component.Id == "4.5.6 netstandard2.0 library - DotNet").Should().ContainSingle(); } - - private static string ProjectAssets(string projectName, string outputPath, string projectPath, params string[] targetFrameworks) - { - LockFileFormat format = new(); - LockFile lockFile = new(); - using var textWriter = new StringWriter(); - - // assets file always includes a trailing separator - if (!Path.EndsInDirectorySeparator(outputPath)) - { - outputPath += Path.DirectorySeparatorChar; - } - - lockFile.Targets = targetFrameworks.Select(tfm => new LockFileTarget() { TargetFramework = NuGetFramework.Parse(tfm) }).ToList(); - lockFile.PackageSpec = new() - { - RestoreMetadata = new() - { - ProjectName = projectName, - OutputPath = outputPath, - ProjectPath = projectPath, - }, - }; - - format.Write(textWriter, lockFile); - return textWriter.ToString(); - } - - private static Stream GlobalJson(string sdkVersion) - { - var stream = new MemoryStream(); - using (var writer = new Utf8JsonWriter(stream, new() { Indented = true })) - { - writer.WriteStartObject(); - writer.WritePropertyName("sdk"); - writer.WriteStartObject(); - writer.WriteString("version", sdkVersion); - writer.WriteEndObject(); - writer.WriteEndObject(); - } - - stream.Position = 0; - return stream; - } - - private static Stream StreamFromString(string content) - { - var stream = new MemoryStream(); - using (var writer = new StreamWriter(stream, leaveOpen: true)) - { - writer.Write(content); - writer.Flush(); - stream.Position = 0; - } - - return stream; - } - - private bool FileExists(string path) - { - var fileName = Path.GetFileName(path); - var directory = Path.GetDirectoryName(path); - - return this.files.TryGetValue(directory, out var fileNames) && - fileNames.TryGetValue(fileName, out _); - } - - private Stream OpenFile(string path) - { - var fileName = Path.GetFileName(path); - var directory = Path.GetDirectoryName(path); - - return this.files.TryGetValue(directory, out var fileNames) && - fileNames.TryGetValue(fileName, out var stream) ? stream : null; - } - - private bool DirectoryExists(string directory) => this.files.ContainsKey(directory); - - private IEnumerable EnumerateFilesRecursive(string directory, string pattern) - { - if (this.files.TryGetValue(directory, out var fileNames)) - { - // a basic approximation of globbing - var patternRegex = new Regex(pattern.Replace(".", "\\.").Replace("*", ".*")); - - foreach (var fileName in fileNames.Keys) - { - var filePath = Path.Combine(directory, fileName); - - if (fileName.EndsWith(Path.DirectorySeparatorChar)) - { - foreach (var subFile in this.EnumerateFilesRecursive(Path.TrimEndingDirectorySeparator(filePath), pattern)) - { - yield return subFile; - } - } - else - { - if (patternRegex.IsMatch(fileName)) - { - yield return filePath; - } - } - } - } - } - - private void AddFile(string path, Stream content) - { - var fileName = Path.GetFileName(path); - var directory = Path.GetDirectoryName(path); - this.AddDirectory(directory); - this.files[directory][fileName] = content; - } - - private void AddDirectory(string path, string subDirectory = null) - { - if (string.IsNullOrEmpty(path)) - { - return; - } - - if (subDirectory is not null) - { - // use a trailing slash to indicate a sub directory in the files collection - subDirectory += Path.DirectorySeparatorChar; - } - - if (this.files.TryGetValue(path, out var directoryFiles)) - { - if (subDirectory is not null) - { - directoryFiles.Add(subDirectory, null); - } - } - else - { - this.files.Add(path, subDirectory is null ? [] : new() { { subDirectory, null } }); - this.AddDirectory(Path.GetDirectoryName(path), Path.GetFileName(path)); - } - } - - private void SetCommandResult(int exitCode, string stdOut = null, string stdErr = null) - { - this.commandLineCallback = null; - this.commandLineExecutionResult.ExitCode = exitCode; - this.commandLineExecutionResult.StdOut = stdOut; - this.commandLineExecutionResult.StdErr = stdErr; - } - - private void SetCommandResult(Func callback) - { - this.commandLineCallback = callback; - } - - private CommandLineExecutionResult CommandResult(string command, DirectoryInfo directory) => - (this.commandLineCallback != null) ? this.commandLineCallback(command, directory) : this.commandLineExecutionResult; } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/GoComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/GoComponentDetectorTests.cs index 75e6b98a1..97d06c3e3 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/GoComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/GoComponentDetectorTests.cs @@ -53,6 +53,31 @@ public GoComponentDetectorTests() this.DetectorTestUtility.AddServiceMock(this.mockParserFactory); } + private void SetupMockGoModParser() + { + this.mockParserFactory.Setup(f => f.CreateParser(GoParserType.GoMod, It.IsAny())).Returns(this.mockGoModParser.Object); + } + + private void SetupMockGoSumParser() + { + this.mockParserFactory.Setup(f => f.CreateParser(GoParserType.GoSum, It.IsAny())).Returns(this.mockGoSumParser.Object); + } + + private void SetupMockGoCLIParser() + { + this.mockParserFactory.Setup(f => f.CreateParser(GoParserType.GoCLI, It.IsAny())).Returns(this.mockGoCliParser.Object); + } + + private void SetupActualGoModParser() + { + this.mockParserFactory.Setup(f => f.CreateParser(GoParserType.GoMod, It.IsAny())).Returns(new GoModParser(this.mockLogger.Object)); + } + + private void SetupActualGoSumParser() + { + this.mockParserFactory.Setup(f => f.CreateParser(GoParserType.GoSum, It.IsAny())).Returns(new GoSumParser(this.mockLogger.Object)); + } + [TestMethod] public async Task TestGoModDetectorWithValidFile_ReturnsSuccessfullyAsync() { @@ -1183,29 +1208,4 @@ public async Task GoDetector_GoSum_VerifyNestedRootsAreNotSkippedIfParentParseFa Path.Combine(root, "a", "a", "go.mod"), Path.Combine(root, "a", "b", "go.mod")); } - - private void SetupMockGoModParser() - { - this.mockParserFactory.Setup(f => f.CreateParser(GoParserType.GoMod, It.IsAny())).Returns(this.mockGoModParser.Object); - } - - private void SetupMockGoSumParser() - { - this.mockParserFactory.Setup(f => f.CreateParser(GoParserType.GoSum, It.IsAny())).Returns(this.mockGoSumParser.Object); - } - - private void SetupMockGoCLIParser() - { - this.mockParserFactory.Setup(f => f.CreateParser(GoParserType.GoCLI, It.IsAny())).Returns(this.mockGoCliParser.Object); - } - - private void SetupActualGoModParser() - { - this.mockParserFactory.Setup(f => f.CreateParser(GoParserType.GoMod, It.IsAny())).Returns(new GoModParser(this.mockLogger.Object)); - } - - private void SetupActualGoSumParser() - { - this.mockParserFactory.Setup(f => f.CreateParser(GoParserType.GoSum, It.IsAny())).Returns(new GoSumParser(this.mockLogger.Object)); - } } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/PipCommandServiceTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/PipCommandServiceTests.cs index b13341ab2..a8d173801 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/PipCommandServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/PipCommandServiceTests.cs @@ -359,6 +359,29 @@ public async Task PythonPipCommandService_GeneratesReport_RequirementsTxt_Correc this.commandLineInvokationService.Verify(); } + private static void ValidateRequirementsTxtReportFile(PipInstallationReport report, FileInfo reportFile) + { + // the file shouldn't exist since we're not writing to it in the test + reportFile.Should().NotBeNull(); + reportFile.Exists.Should().Be(false); + + // validate report parameters + report.Should().NotBeNull(); + report.Version.Should().Be("1"); + report.InstallItems.Should().NotBeNull(); + report.InstallItems.Should().ContainSingle(); + + // validate packages + report.InstallItems[0].Requested.Should().BeTrue(); + report.InstallItems[0].Metadata.Name.Should().Be("six"); + report.InstallItems[0].Metadata.Version.Should().Be("1.16.0"); + report.InstallItems[0].Metadata.License.Should().Be("MIT"); + report.InstallItems[0].Metadata.Author.Should().Be("Benjamin Peterson"); + report.InstallItems[0].Metadata.AuthorEmail.Should().Be("benjamin@python.org"); + report.InstallItems[0].Metadata.Maintainer.Should().BeNullOrEmpty(); + report.InstallItems[0].Metadata.MaintainerEmail.Should().BeNullOrEmpty(); + } + [TestMethod] public async Task PipCommandService_GeneratesReport_SetupPy_CorrectlyAsync() { @@ -680,27 +703,4 @@ public async Task PipCommandService_CancelledAsync() var action = async () => await service.GenerateInstallationReportAsync(testPath, cancellationToken: cts.Token); await action.Should().ThrowAsync().WithMessage("PipReport: Cancelled*"); } - - private static void ValidateRequirementsTxtReportFile(PipInstallationReport report, FileInfo reportFile) - { - // the file shouldn't exist since we're not writing to it in the test - reportFile.Should().NotBeNull(); - reportFile.Exists.Should().Be(false); - - // validate report parameters - report.Should().NotBeNull(); - report.Version.Should().Be("1"); - report.InstallItems.Should().NotBeNull(); - report.InstallItems.Should().ContainSingle(); - - // validate packages - report.InstallItems[0].Requested.Should().BeTrue(); - report.InstallItems[0].Metadata.Name.Should().Be("six"); - report.InstallItems[0].Metadata.Version.Should().Be("1.16.0"); - report.InstallItems[0].Metadata.License.Should().Be("MIT"); - report.InstallItems[0].Metadata.Author.Should().Be("Benjamin Peterson"); - report.InstallItems[0].Metadata.AuthorEmail.Should().Be("benjamin@python.org"); - report.InstallItems[0].Metadata.Maintainer.Should().BeNullOrEmpty(); - report.InstallItems[0].Metadata.MaintainerEmail.Should().BeNullOrEmpty(); - } } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/PipDependencySpecifierTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/PipDependencySpecifierTests.cs index 3b6e37b18..29faeef0b 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/PipDependencySpecifierTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/PipDependencySpecifierTests.cs @@ -9,6 +9,59 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; [TestClass] public class PipDependencySpecifierTests { + private static void VerifyPipDependencyParsing( + List<(string SpecString, PipDependencySpecification ReferenceDependencySpecification)> testCases, + bool requiresDist = false) + { + foreach (var (specString, referenceDependencySpecification) in testCases) + { + var dependencySpecifier = new PipDependencySpecification(specString, requiresDist); + + dependencySpecifier.Name.Should().Be(referenceDependencySpecification.Name); + dependencySpecifier.DependencySpecifiers.Should().HaveCount(referenceDependencySpecification.DependencySpecifiers.Count); + for (var i = 0; i < referenceDependencySpecification.DependencySpecifiers.Count; i++) + { + dependencySpecifier.DependencySpecifiers.Should().HaveElementAt( + i, referenceDependencySpecification.DependencySpecifiers[i]); + } + + dependencySpecifier.ConditionalDependencySpecifiers.Should().HaveCount(referenceDependencySpecification.ConditionalDependencySpecifiers.Count); + for (var i = 0; i < referenceDependencySpecification.ConditionalDependencySpecifiers.Count; i++) + { + dependencySpecifier.ConditionalDependencySpecifiers.Should().HaveElementAt( + i, referenceDependencySpecification.ConditionalDependencySpecifiers[i]); + } + } + } + + private static void VerifyPipConditionalDependencyParsing( + List<(string SpecString, bool ShouldBeIncluded, PipDependencySpecification ReferenceDependencySpecification)> testCases, + Dictionary pythonEnvironmentVariables, + bool requiresDist = false) + { + foreach (var (specString, shouldBeIncluded, referenceDependencySpecification) in testCases) + { + var dependencySpecifier = new PipDependencySpecification(specString, requiresDist); + + dependencySpecifier.Name.Should().Be(referenceDependencySpecification.Name); + dependencySpecifier.DependencySpecifiers.Should().HaveCount(referenceDependencySpecification.DependencySpecifiers.Count); + for (var i = 0; i < referenceDependencySpecification.DependencySpecifiers.Count; i++) + { + dependencySpecifier.DependencySpecifiers.Should().HaveElementAt( + i, referenceDependencySpecification.DependencySpecifiers[i]); + } + + dependencySpecifier.ConditionalDependencySpecifiers.Should().HaveCount(referenceDependencySpecification.ConditionalDependencySpecifiers.Count); + for (var i = 0; i < referenceDependencySpecification.ConditionalDependencySpecifiers.Count; i++) + { + dependencySpecifier.ConditionalDependencySpecifiers.Should().HaveElementAt( + i, referenceDependencySpecification.ConditionalDependencySpecifiers[i]); + } + + dependencySpecifier.PackageConditionsMet(pythonEnvironmentVariables).Should().Be(shouldBeIncluded, string.Join(',', dependencySpecifier.ConditionalDependencySpecifiers)); + } + } + [TestMethod] public void TestPipDependencySpecifierConstruction() { @@ -148,57 +201,4 @@ public void TestPipDependencyGetHighestExplicitPackageVersion_AllInvalidSpec() var highestVersion = spec.GetHighestExplicitPackageVersion(); highestVersion.Should().BeNull(); } - - private static void VerifyPipDependencyParsing( - List<(string SpecString, PipDependencySpecification ReferenceDependencySpecification)> testCases, - bool requiresDist = false) - { - foreach (var (specString, referenceDependencySpecification) in testCases) - { - var dependencySpecifier = new PipDependencySpecification(specString, requiresDist); - - dependencySpecifier.Name.Should().Be(referenceDependencySpecification.Name); - dependencySpecifier.DependencySpecifiers.Should().HaveCount(referenceDependencySpecification.DependencySpecifiers.Count); - for (var i = 0; i < referenceDependencySpecification.DependencySpecifiers.Count; i++) - { - dependencySpecifier.DependencySpecifiers.Should().HaveElementAt( - i, referenceDependencySpecification.DependencySpecifiers[i]); - } - - dependencySpecifier.ConditionalDependencySpecifiers.Should().HaveCount(referenceDependencySpecification.ConditionalDependencySpecifiers.Count); - for (var i = 0; i < referenceDependencySpecification.ConditionalDependencySpecifiers.Count; i++) - { - dependencySpecifier.ConditionalDependencySpecifiers.Should().HaveElementAt( - i, referenceDependencySpecification.ConditionalDependencySpecifiers[i]); - } - } - } - - private static void VerifyPipConditionalDependencyParsing( - List<(string SpecString, bool ShouldBeIncluded, PipDependencySpecification ReferenceDependencySpecification)> testCases, - Dictionary pythonEnvironmentVariables, - bool requiresDist = false) - { - foreach (var (specString, shouldBeIncluded, referenceDependencySpecification) in testCases) - { - var dependencySpecifier = new PipDependencySpecification(specString, requiresDist); - - dependencySpecifier.Name.Should().Be(referenceDependencySpecification.Name); - dependencySpecifier.DependencySpecifiers.Should().HaveCount(referenceDependencySpecification.DependencySpecifiers.Count); - for (var i = 0; i < referenceDependencySpecification.DependencySpecifiers.Count; i++) - { - dependencySpecifier.DependencySpecifiers.Should().HaveElementAt( - i, referenceDependencySpecification.DependencySpecifiers[i]); - } - - dependencySpecifier.ConditionalDependencySpecifiers.Should().HaveCount(referenceDependencySpecification.ConditionalDependencySpecifiers.Count); - for (var i = 0; i < referenceDependencySpecification.ConditionalDependencySpecifiers.Count; i++) - { - dependencySpecifier.ConditionalDependencySpecifiers.Should().HaveElementAt( - i, referenceDependencySpecification.ConditionalDependencySpecifiers[i]); - } - - dependencySpecifier.PackageConditionsMet(pythonEnvironmentVariables).Should().Be(shouldBeIncluded, string.Join(',', dependencySpecifier.ConditionalDependencySpecifiers)); - } - } } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/RustCargoLockParserTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/RustCargoLockParserTests.cs index c37919188..159e9dcab 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/RustCargoLockParserTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/RustCargoLockParserTests.cs @@ -31,6 +31,39 @@ public void Init() this.parser = new RustCargoLockParser(this.logger.Object); } + private static IComponentStream MakeStream(string name, string toml) + { + return new ComponentStream + { + Location = name, + Pattern = "Cargo.lock", + Stream = new MemoryStream(Encoding.UTF8.GetBytes(toml)), + }; + } + + private static (int Usages, int ExplicitRoots, int Edges, int Failures) Analyze(Mock recorder) + { + var usageInvocations = recorder.Invocations.Where(i => i.Method.Name == "RegisterUsage").ToList(); + var explicitRoots = 0; + var edges = 0; + foreach (var inv in usageInvocations) + { + // Signature: RegisterUsage(DetectedComponent dc, bool isExplicitReferencedDependency = false, string parentComponentId = null, bool isDevelopmentDependency = false) + if (inv.Arguments.Count >= 2 && inv.Arguments[1] is bool explicitFlag && explicitFlag) + { + explicitRoots++; + } + + if (inv.Arguments.Count >= 3 && inv.Arguments[2] is string parentId) + { + edges++; + } + } + + var failures = recorder.Invocations.Count(i => i.Method.Name == "RegisterPackageParseFailure"); + return (usageInvocations.Count, explicitRoots, edges, failures); + } + [TestMethod] public async Task ParseAsync_NoPackages_ReturnsVersion_NoUsage() { @@ -698,37 +731,4 @@ public async Task DifferentVersionsSamePackage_BothRegistered() edges.Should().Be(2); failures.Should().Be(0); } - - private static IComponentStream MakeStream(string name, string toml) - { - return new ComponentStream - { - Location = name, - Pattern = "Cargo.lock", - Stream = new MemoryStream(Encoding.UTF8.GetBytes(toml)), - }; - } - - private static (int Usages, int ExplicitRoots, int Edges, int Failures) Analyze(Mock recorder) - { - var usageInvocations = recorder.Invocations.Where(i => i.Method.Name == "RegisterUsage").ToList(); - var explicitRoots = 0; - var edges = 0; - foreach (var inv in usageInvocations) - { - // Signature: RegisterUsage(DetectedComponent dc, bool isExplicitReferencedDependency = false, string parentComponentId = null, bool isDevelopmentDependency = false) - if (inv.Arguments.Count >= 2 && inv.Arguments[1] is bool explicitFlag && explicitFlag) - { - explicitRoots++; - } - - if (inv.Arguments.Count >= 3 && inv.Arguments[2] is string parentId) - { - edges++; - } - } - - var failures = recorder.Invocations.Count(i => i.Method.Name == "RegisterPackageParseFailure"); - return (usageInvocations.Count, explicitRoots, edges, failures); - } } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/RustCliParserTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/RustCliParserTests.cs index e91d57243..1fb1b7b54 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/RustCliParserTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/RustCliParserTests.cs @@ -41,6 +41,51 @@ public void Init() this.parser = new RustCliParser(this.cli.Object, this.env.Object, new PathUtilityService(new Mock>().Object), this.logger.Object); } + private static IComponentStream MakeTomlStream(string path) => + new ComponentStream { Location = path, Pattern = "Cargo.toml", Stream = new MemoryStream(Encoding.UTF8.GetBytes("[package]\nname=\"x\"")) }; + + // kind: build (non-dev), kind: dev (development), or absent/null. + private static string BuildNormalRootMetadataJson() => """ + { + "packages": [ + { "name":"rootpkg", "version":"1.0.0", "id":"rootpkg 1.0.0", "authors":[""], "license":"", "source":null, "manifest_path":"C:/repo/root/Cargo.toml" }, + { "name":"childA", "version":"2.0.0", "id":"childA 2.0.0", "authors":["Alice"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/root/childA/Cargo.toml" }, + { "name":"childDev", "version":"3.0.0", "id":"childDev 3.0.0", "authors":["Bob"], "license":"Apache-2.0", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/root/childDev/Cargo.toml" } + ], + "resolve": { + "root":"rootpkg 1.0.0", + "nodes":[ + { "id":"rootpkg 1.0.0", + "deps":[ + { "pkg":"childA 2.0.0", "dep_kinds":[{"kind":"build"}] }, + { "pkg":"childDev 3.0.0", "dep_kinds":[{"kind":"dev"}] } + ] + }, + { "id":"childA 2.0.0", "deps":[] }, + { "id":"childDev 3.0.0", "deps":[] } + ] + } + } + """; + + private static string BuildVirtualManifestMetadataJson() => """ + { + "packages": [ + { "name":"virtA", "version":"0.2.0", "id":"virtA 0.2.0", "authors":["Ann"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/virtA/Cargo.toml" }, + { "name":"virtB", "version":"0.3.0", "id":"virtB 0.3.0", "authors":["Ben"], "license":"Apache-2.0", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/virtB/Cargo.toml" } + ], + "resolve": { + "root": null, + "nodes":[ + { "id":"virtA 0.2.0", "deps":[ { "pkg":"virtB 0.3.0", "dep_kinds":[{"kind":"build"}] } ] }, + { "id":"virtB 0.3.0", "deps":[] } + ] + } + } + """; + + private static CargoMetadata ParseMetadata(string json) => CargoMetadata.FromJson(json); + [TestMethod] public async Task ParseAsync_ManuallyDisabled_ReturnsFailure() { @@ -1091,51 +1136,6 @@ public async Task VirtualManifest_MultipleRootNodes_AllProcessed() distinctNames.Should().Contain("pkgB"); } - private static IComponentStream MakeTomlStream(string path) => - new ComponentStream { Location = path, Pattern = "Cargo.toml", Stream = new MemoryStream(Encoding.UTF8.GetBytes("[package]\nname=\"x\"")) }; - - // kind: build (non-dev), kind: dev (development), or absent/null. - private static string BuildNormalRootMetadataJson() => """ - { - "packages": [ - { "name":"rootpkg", "version":"1.0.0", "id":"rootpkg 1.0.0", "authors":[""], "license":"", "source":null, "manifest_path":"C:/repo/root/Cargo.toml" }, - { "name":"childA", "version":"2.0.0", "id":"childA 2.0.0", "authors":["Alice"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/root/childA/Cargo.toml" }, - { "name":"childDev", "version":"3.0.0", "id":"childDev 3.0.0", "authors":["Bob"], "license":"Apache-2.0", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/root/childDev/Cargo.toml" } - ], - "resolve": { - "root":"rootpkg 1.0.0", - "nodes":[ - { "id":"rootpkg 1.0.0", - "deps":[ - { "pkg":"childA 2.0.0", "dep_kinds":[{"kind":"build"}] }, - { "pkg":"childDev 3.0.0", "dep_kinds":[{"kind":"dev"}] } - ] - }, - { "id":"childA 2.0.0", "deps":[] }, - { "id":"childDev 3.0.0", "deps":[] } - ] - } - } - """; - - private static string BuildVirtualManifestMetadataJson() => """ - { - "packages": [ - { "name":"virtA", "version":"0.2.0", "id":"virtA 0.2.0", "authors":["Ann"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/virtA/Cargo.toml" }, - { "name":"virtB", "version":"0.3.0", "id":"virtB 0.3.0", "authors":["Ben"], "license":"Apache-2.0", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/virtB/Cargo.toml" } - ], - "resolve": { - "root": null, - "nodes":[ - { "id":"virtA 0.2.0", "deps":[ { "pkg":"virtB 0.3.0", "dep_kinds":[{"kind":"build"}] } ] }, - { "id":"virtB 0.3.0", "deps":[] } - ] - } - } - """; - - private static CargoMetadata ParseMetadata(string json) => CargoMetadata.FromJson(json); - private async Task InvokeProcessMetadataAsync(string manifestLocation, ISingleFileComponentRecorder fallbackRecorder, CargoMetadata metadata) => await this.parser.ParseFromMetadataAsync( new ComponentStream { Location = manifestLocation, Pattern = "Cargo.toml", Stream = new MemoryStream([]) }, diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/RustMetadataContextBuilderTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/RustMetadataContextBuilderTests.cs index fb3dc7eac..5e70e9b80 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/RustMetadataContextBuilderTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/RustMetadataContextBuilderTests.cs @@ -35,6 +35,65 @@ public void Init() this.envVarService.Object); } + private static string BuildSimpleMetadataJson(string rootManifest, string rootId) => $$""" + { + "packages": [ + { "name":"rootpkg", "version":"1.0.0", "id":"{{rootId}}", "authors":[""], "license":"", "source":null, "manifest_path":"{{rootManifest}}" }, + { "name":"dep1", "version":"2.0.0", "id":"dep1 2.0.0", "authors":["A"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/dep1/Cargo.toml" } + ], + "resolve": { + "root":"{{rootId}}", + "nodes":[ + { "id":"{{rootId}}", "deps":[ { "pkg":"dep1 2.0.0", "dep_kinds":[{"kind":"build"}] } ] }, + { "id":"dep1 2.0.0", "deps":[] } + ] + } + } + """; + + private static string BuildWorkspaceMetadataJson() => """ + { + "packages": [ + { "name":"workspace", "version":"0.1.0", "id":"workspace 0.1.0", "authors":[""], "license":"", "source":null, "manifest_path":"C:/repo/Cargo.toml" }, + { "name":"member1", "version":"0.2.0", "id":"member1 0.2.0", "authors":[""], "license":"", "source":null, "manifest_path":"C:/repo/member1/Cargo.toml" }, + { "name":"member2", "version":"0.3.0", "id":"member2 0.3.0", "authors":[""], "license":"", "source":null, "manifest_path":"C:/repo/member2/Cargo.toml" }, + { "name":"shared", "version":"1.0.0", "id":"shared 1.0.0", "authors":["S"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/shared/Cargo.toml" } + ], + "resolve": { + "root":"workspace 0.1.0", + "nodes":[ + { "id":"workspace 0.1.0", "deps":[] }, + { "id":"member1 0.2.0", "deps":[ { "pkg":"shared 1.0.0", "dep_kinds":[{"kind":"build"}] } ] }, + { "id":"member2 0.3.0", "deps":[ { "pkg":"shared 1.0.0", "dep_kinds":[{"kind":"build"}] } ] }, + { "id":"shared 1.0.0", "deps":[] } + ] + } + } + """; + + private static string BuildDiamondDependencyJson() => """ + { + "packages": [ + { "name":"root", "version":"1.0.0", "id":"root 1.0.0", "authors":[""], "license":"", "source":null, "manifest_path":"C:/repo/Cargo.toml" }, + { "name":"depA", "version":"1.0.0", "id":"depA 1.0.0", "authors":["A"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/depA/Cargo.toml" }, + { "name":"depB", "version":"1.0.0", "id":"depB 1.0.0", "authors":["B"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/depB/Cargo.toml" }, + { "name":"shared", "version":"1.0.0", "id":"shared 1.0.0", "authors":["S"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/shared/Cargo.toml" } + ], + "resolve": { + "root":"root 1.0.0", + "nodes":[ + { "id":"root 1.0.0", "deps":[ + { "pkg":"depA 1.0.0", "dep_kinds":[{"kind":"build"}] }, + { "pkg":"depB 1.0.0", "dep_kinds":[{"kind":"build"}] } + ] }, + { "id":"depA 1.0.0", "deps":[ { "pkg":"shared 1.0.0", "dep_kinds":[{"kind":"build"}] } ] }, + { "id":"depB 1.0.0", "deps":[ { "pkg":"shared 1.0.0", "dep_kinds":[{"kind":"build"}] } ] }, + { "id":"shared 1.0.0", "deps":[] } + ] + } + } + """; + [TestMethod] public async Task BuildPackageOwnershipMapAsync_ManuallyDisabled_ReturnsEmptyResult() { @@ -415,63 +474,4 @@ public async Task BuildPackageOwnershipMapAsync_PathNormalization_UsesNormalized result.LocalPackageManifests.Should().Contain(normalizedPath); result.ManifestToMetadata.Should().ContainKey(normalizedPath); } - - private static string BuildSimpleMetadataJson(string rootManifest, string rootId) => $$""" - { - "packages": [ - { "name":"rootpkg", "version":"1.0.0", "id":"{{rootId}}", "authors":[""], "license":"", "source":null, "manifest_path":"{{rootManifest}}" }, - { "name":"dep1", "version":"2.0.0", "id":"dep1 2.0.0", "authors":["A"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/dep1/Cargo.toml" } - ], - "resolve": { - "root":"{{rootId}}", - "nodes":[ - { "id":"{{rootId}}", "deps":[ { "pkg":"dep1 2.0.0", "dep_kinds":[{"kind":"build"}] } ] }, - { "id":"dep1 2.0.0", "deps":[] } - ] - } - } - """; - - private static string BuildWorkspaceMetadataJson() => """ - { - "packages": [ - { "name":"workspace", "version":"0.1.0", "id":"workspace 0.1.0", "authors":[""], "license":"", "source":null, "manifest_path":"C:/repo/Cargo.toml" }, - { "name":"member1", "version":"0.2.0", "id":"member1 0.2.0", "authors":[""], "license":"", "source":null, "manifest_path":"C:/repo/member1/Cargo.toml" }, - { "name":"member2", "version":"0.3.0", "id":"member2 0.3.0", "authors":[""], "license":"", "source":null, "manifest_path":"C:/repo/member2/Cargo.toml" }, - { "name":"shared", "version":"1.0.0", "id":"shared 1.0.0", "authors":["S"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/shared/Cargo.toml" } - ], - "resolve": { - "root":"workspace 0.1.0", - "nodes":[ - { "id":"workspace 0.1.0", "deps":[] }, - { "id":"member1 0.2.0", "deps":[ { "pkg":"shared 1.0.0", "dep_kinds":[{"kind":"build"}] } ] }, - { "id":"member2 0.3.0", "deps":[ { "pkg":"shared 1.0.0", "dep_kinds":[{"kind":"build"}] } ] }, - { "id":"shared 1.0.0", "deps":[] } - ] - } - } - """; - - private static string BuildDiamondDependencyJson() => """ - { - "packages": [ - { "name":"root", "version":"1.0.0", "id":"root 1.0.0", "authors":[""], "license":"", "source":null, "manifest_path":"C:/repo/Cargo.toml" }, - { "name":"depA", "version":"1.0.0", "id":"depA 1.0.0", "authors":["A"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/depA/Cargo.toml" }, - { "name":"depB", "version":"1.0.0", "id":"depB 1.0.0", "authors":["B"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/depB/Cargo.toml" }, - { "name":"shared", "version":"1.0.0", "id":"shared 1.0.0", "authors":["S"], "license":"MIT", "source":"registry+https://github.com/rust-lang/crates.io-index", "manifest_path":"C:/repo/shared/Cargo.toml" } - ], - "resolve": { - "root":"root 1.0.0", - "nodes":[ - { "id":"root 1.0.0", "deps":[ - { "pkg":"depA 1.0.0", "dep_kinds":[{"kind":"build"}] }, - { "pkg":"depB 1.0.0", "dep_kinds":[{"kind":"build"}] } - ] }, - { "id":"depA 1.0.0", "deps":[ { "pkg":"shared 1.0.0", "dep_kinds":[{"kind":"build"}] } ] }, - { "id":"depB 1.0.0", "deps":[ { "pkg":"shared 1.0.0", "dep_kinds":[{"kind":"build"}] } ] }, - { "id":"shared 1.0.0", "deps":[] } - ] - } - } - """; } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/RustSbomParserTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/RustSbomParserTests.cs index 93cffc708..04ac761ea 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/RustSbomParserTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/RustSbomParserTests.cs @@ -32,6 +32,63 @@ public void Init() this.parser = new RustSbomParser(this.logger.Object); } + private static IComponentStream MakeSbomStream(string location, string json) => + new ComponentStream + { + Location = location, + Pattern = "*.cargo-sbom.json", + Stream = new MemoryStream(Encoding.UTF8.GetBytes(json)), + }; + + private static string BuildSimpleSbomJson() => $$""" + { + "version": 1, + "root": 0, + "crates": [ + { + "id": "path+file:///repo/root#0.1.0", + "features": [], + "dependencies": [ + { "index": 1, "kind": "normal" } + ] + }, + { + "id": "{{CratesIo}}#dep1@1.0.0", + "features": [], + "dependencies": [] + } + ] + } + """; + + private static string BuildNestedSbomJson() => $$""" + { + "version": 1, + "root": 0, + "crates": [ + { + "id": "path+file:///repo/root#0.1.0", + "features": [], + "dependencies": [ + { "index": 1, "kind": "normal" } + ] + }, + { + "id": "{{CratesIo}}#parent@2.0.0", + "features": [], + "dependencies": [ + { "index": 2, "kind": "normal" } + ] + }, + { + "id": "{{CratesIo}}#child@3.0.0", + "features": [], + "dependencies": [] + } + ] + } + """; + [TestMethod] public async Task ParseAsync_ValidSimpleSbom_RegistersComponents() { @@ -935,61 +992,4 @@ await this.parser.ParseWithOwnershipAsync( // Verify fallback happened and parentId was checked in graph sbomRecorder.Invocations.Count(i => i.Method.Name == "RegisterUsage").Should().BePositive(); } - - private static IComponentStream MakeSbomStream(string location, string json) => - new ComponentStream - { - Location = location, - Pattern = "*.cargo-sbom.json", - Stream = new MemoryStream(Encoding.UTF8.GetBytes(json)), - }; - - private static string BuildSimpleSbomJson() => $$""" - { - "version": 1, - "root": 0, - "crates": [ - { - "id": "path+file:///repo/root#0.1.0", - "features": [], - "dependencies": [ - { "index": 1, "kind": "normal" } - ] - }, - { - "id": "{{CratesIo}}#dep1@1.0.0", - "features": [], - "dependencies": [] - } - ] - } - """; - - private static string BuildNestedSbomJson() => $$""" - { - "version": 1, - "root": 0, - "crates": [ - { - "id": "path+file:///repo/root#0.1.0", - "features": [], - "dependencies": [ - { "index": 1, "kind": "normal" } - ] - }, - { - "id": "{{CratesIo}}#parent@2.0.0", - "features": [], - "dependencies": [ - { "index": 2, "kind": "normal" } - ] - }, - { - "id": "{{CratesIo}}#child@3.0.0", - "features": [], - "dependencies": [] - } - ] - } - """; } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/SimplePypiClientTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/SimplePypiClientTests.cs index 9a2612b4d..b41880bb4 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/SimplePypiClientTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/SimplePypiClientTests.cs @@ -20,6 +20,29 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; [TestClass] public class SimplePyPiClientTests { + private Mock MockHttpMessageHandler(string content, HttpStatusCode statusCode) + { + var handlerMock = new Mock(); + handlerMock.Protected() + .Setup>( + "SendAsync", + ItExpr.IsAny(), + ItExpr.IsAny()) + .ReturnsAsync(new HttpResponseMessage() + { + StatusCode = statusCode, + Content = new StringContent(content), + }); + + return handlerMock; + } + + private ISimplePyPiClient CreateSimplePypiClient(HttpMessageHandler messageHandler, IEnvironmentVariableService evs, ILogger logger) + { + SimplePyPiClient.HttpClient = new HttpClient(messageHandler); + return new SimplePyPiClient(evs, logger); + } + [TestMethod] public async Task GetSimplePypiProject_DuplicateEntries_CallsGetAsync_OnceAsync() { @@ -332,27 +355,4 @@ public string SampleValidApiJsonResponse(string packageName, string version) return packageJsonTemplate; } - - private Mock MockHttpMessageHandler(string content, HttpStatusCode statusCode) - { - var handlerMock = new Mock(); - handlerMock.Protected() - .Setup>( - "SendAsync", - ItExpr.IsAny(), - ItExpr.IsAny()) - .ReturnsAsync(new HttpResponseMessage() - { - StatusCode = statusCode, - Content = new StringContent(content), - }); - - return handlerMock; - } - - private ISimplePyPiClient CreateSimplePypiClient(HttpMessageHandler messageHandler, IEnvironmentVariableService evs, ILogger logger) - { - SimplePyPiClient.HttpClient = new HttpClient(messageHandler); - return new SimplePyPiClient(evs, logger); - } } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Utilities/ComponentRecorderTestUtilities.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/Utilities/ComponentRecorderTestUtilities.cs index 404c358cf..ae290cc85 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/Utilities/ComponentRecorderTestUtilities.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/Utilities/ComponentRecorderTestUtilities.cs @@ -113,6 +113,30 @@ public static void AssertAllExplicitlyReferencedComponents( }); } + private static ComponentOrientedGrouping TupleToObject(IEnumerable<(string Location, IDependencyGraph Graph, string ComponentId)> x) + { + var additionalRelatedFiles = new List(x.SelectMany(y => y.Graph.GetAdditionalRelatedFiles())); + additionalRelatedFiles.AddRange(x.Select(y => y.Location)); + + return new ComponentOrientedGrouping + { + ComponentId = x.First().ComponentId, + FoundInGraphs = x.Select(y => (y.Location, y.Graph)).ToList(), + AllFileLocations = additionalRelatedFiles.Distinct().ToList(), + ParentComponentIdsThatAreExplicitReferences = x.SelectMany(y => y.Graph.GetExplicitReferencedDependencyIds(x.First().ComponentId)).Distinct().ToList(), + }; + } + + private static List> GroupByComponentId(IReadOnlyDictionary graphs) + { + return graphs + .Select(x => (Location: x.Key, Graph: x.Value)) + .SelectMany(x => x.Graph.GetComponents() + .Select(componentId => (x.Location, x.Graph, ComponentId: componentId))) + .GroupBy(x => x.ComponentId) + .ToList(); + } + public static void CheckGraphStructure(IDependencyGraph graph, Dictionary graphComponentsWithDeps) { var graphComponents = graph.GetComponents().ToArray(); @@ -148,30 +172,6 @@ public static void CheckChild(IComponentRecorder recorder, string childId, st parentIds.Select(parentId => new Func(x => x.Id == parentId)).ToArray()); } - private static ComponentOrientedGrouping TupleToObject(IEnumerable<(string Location, IDependencyGraph Graph, string ComponentId)> x) - { - var additionalRelatedFiles = new List(x.SelectMany(y => y.Graph.GetAdditionalRelatedFiles())); - additionalRelatedFiles.AddRange(x.Select(y => y.Location)); - - return new ComponentOrientedGrouping - { - ComponentId = x.First().ComponentId, - FoundInGraphs = x.Select(y => (y.Location, y.Graph)).ToList(), - AllFileLocations = additionalRelatedFiles.Distinct().ToList(), - ParentComponentIdsThatAreExplicitReferences = x.SelectMany(y => y.Graph.GetExplicitReferencedDependencyIds(x.First().ComponentId)).Distinct().ToList(), - }; - } - - private static List> GroupByComponentId(IReadOnlyDictionary graphs) - { - return graphs - .Select(x => (Location: x.Key, Graph: x.Value)) - .SelectMany(x => x.Graph.GetComponents() - .Select(componentId => (x.Location, x.Graph, ComponentId: componentId))) - .GroupBy(x => x.ComponentId) - .ToList(); - } - public class ComponentOrientedGrouping { public IEnumerable<(string ManifestFile, IDependencyGraph Graph)> FoundInGraphs { get; set; } diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs index 00b8ced9d..ee369f242 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs @@ -50,6 +50,13 @@ public ExperimentServiceTests() this.experimentConfigMock.Setup(x => x.ShouldRecord(this.detectorMock.Object, It.IsAny())).Returns(true); } + private void SetupGraphMock(IEnumerable components) + { + this.graphTranslationServiceMock + .Setup(x => x.GenerateScanResultFromProcessingResult(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(new ScanResult() { ComponentsFound = components }); + } + [TestInitialize] public void TestInitialize() { @@ -351,11 +358,4 @@ public async Task InitializeAsync_SwallowsExceptionsAsync() await action.Should().NotThrowAsync(); this.experimentConfigMock.Verify(x => x.InitAsync(), Times.Once()); } - - private void SetupGraphMock(IEnumerable components) - { - this.graphTranslationServiceMock - .Setup(x => x.GenerateScanResultFromProcessingResult(It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(new ScanResult() { ComponentsFound = components }); - } } diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs index f33b57914..0d9c87c4d 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs @@ -72,6 +72,15 @@ public DetectorProcessingServiceTests() this.isWin = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); } + private IndividualDetectorScanResult ExpectedResultForDetector(string detectorId) + { + return new IndividualDetectorScanResult + { + AdditionalTelemetryDetails = new Dictionary { { "detectorId", detectorId } }, + ResultCode = ProcessingResultCode.Success, + }; + } + [TestMethod] public async Task ProcessDetectorsAsync_HappyPathReturnsDetectedComponentsAsync() { @@ -579,15 +588,6 @@ public async Task ProcessDetectorsAsync_InitializesExperimentsAsync() this.experimentServiceMock.Verify(x => x.InitializeAsync(), Times.Once); } - private IndividualDetectorScanResult ExpectedResultForDetector(string detectorId) - { - return new IndividualDetectorScanResult - { - AdditionalTelemetryDetails = new Dictionary { { "detectorId", detectorId } }, - ResultCode = ProcessingResultCode.Success, - }; - } - private Mock SetupFileDetectorMock(string id, DirectoryInfo sourceDirectory = null) { var mockFileDetector = new Mock(); From 636b1fe6ad7af6319893c7bb9eb3a59bfed44a9d Mon Sep 17 00:00:00 2001 From: Adam Plaskitt Date: Mon, 24 Nov 2025 18:29:08 +0000 Subject: [PATCH 5/5] Fix SA1202 warnings in component constructors --- .../FileComponentDetector.cs | 124 +++++++++--------- .../TypedComponent/CargoComponent.cs | 10 +- .../TypedComponent/ConanComponent.cs | 10 +- .../TypedComponent/DockerImageComponent.cs | 10 +- .../TypedComponent/DotNetComponent.cs | 10 +- .../TypedComponent/GoComponent.cs | 4 +- .../TypedComponent/LinuxComponent.cs | 10 +- .../TypedComponent/NpmComponent.cs | 10 +- .../TypedComponent/NugetComponent.cs | 10 +- .../TypedComponent/OtherComponent.cs | 10 +- .../TypedComponent/PipComponent.cs | 10 +- .../TypedComponent/PodComponent.cs | 10 +- .../TypedComponent/RubyGemsComponent.cs | 10 +- .../TypedComponent/SpdxComponent.cs | 10 +- .../TypedComponent/VcpkgComponent.cs | 10 +- 15 files changed, 129 insertions(+), 129 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs b/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs index 7669726d4..df7d12fc8 100644 --- a/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs @@ -17,18 +17,6 @@ namespace Microsoft.ComponentDetection.Contracts; /// Specialized base class for file based component detection. public abstract class FileComponentDetector : IComponentDetector { - /// - /// Gets or sets the factory for handing back component streams to File detectors. - /// - protected IComponentStreamEnumerableFactory ComponentStreamEnumerableFactory { get; set; } - - protected IObservableDirectoryWalkerFactory Scanner { get; set; } - - /// - /// Gets or sets the logger for writing basic logging message to both console and file. - /// - protected ILogger Logger { get; set; } - public IComponentRecorder ComponentRecorder { get; private set; } /// @@ -46,25 +34,37 @@ public abstract class FileComponentDetector : IComponentDetector /// Gets the version of this component detector. public abstract int Version { get; } + public virtual bool NeedsAutomaticRootDependencyCalculation { get; protected set; } + /// - /// Gets the folder names that will be skipped by the Component Detector. + /// List of any any additional properties as key-value pairs that we would like to capture for the detector. /// - protected virtual IList SkippedFolders => []; + public List<(string PropertyKey, string PropertyValue)> AdditionalProperties { get; set; } = []; + + protected ConcurrentDictionary Telemetry { get; set; } = []; /// - /// Gets or sets the active scan request -- only populated after a ScanDirectoryAsync is invoked. If ScanDirectoryAsync is overridden, - /// the overrider should ensure this property is populated. + /// Gets or sets the factory for handing back component streams to File detectors. /// - protected ScanRequest CurrentScanRequest { get; set; } + protected IComponentStreamEnumerableFactory ComponentStreamEnumerableFactory { get; set; } - public virtual bool NeedsAutomaticRootDependencyCalculation { get; protected set; } + protected IObservableDirectoryWalkerFactory Scanner { get; set; } - protected ConcurrentDictionary Telemetry { get; set; } = []; + /// + /// Gets or sets the logger for writing basic logging message to both console and file. + /// + protected ILogger Logger { get; set; } /// - /// List of any any additional properties as key-value pairs that we would like to capture for the detector. + /// Gets the folder names that will be skipped by the Component Detector. /// - public List<(string PropertyKey, string PropertyValue)> AdditionalProperties { get; set; } = []; + protected virtual IList SkippedFolders => []; + + /// + /// Gets or sets the active scan request -- only populated after a ScanDirectoryAsync is invoked. If ScanDirectoryAsync is overridden, + /// the overrider should ensure this property is populated. + /// + protected ScanRequest CurrentScanRequest { get; set; } protected IObservable ComponentStreams { get; private set; } @@ -78,16 +78,6 @@ public async virtual Task ExecuteDetectorAsync(Sca return await this.ScanDirectoryAsync(request, cancellationToken); } - private Task ScanDirectoryAsync(ScanRequest request, CancellationToken cancellationToken = default) - { - this.CurrentScanRequest = request; - - var filteredObservable = this.Scanner.GetFilteredComponentStreamObservable(request.SourceDirectory, this.SearchPatterns, request.ComponentRecorder); - - this.Logger.LogDebug("Registered {Detector}", this.GetType().FullName); - return this.ProcessAsync(filteredObservable, request.DetectorArgs, request.MaxThreads, request.CleanupCreatedFiles, cancellationToken); - } - /// /// Gets the file streams for the Detector's declared as an . /// @@ -111,37 +101,6 @@ protected Task> GetFileStreamsAsync(DirectoryInfo /// The lockfile version. protected void RecordLockfileVersion(string lockfileVersion) => this.Telemetry["LockfileVersion"] = lockfileVersion; - private async Task ProcessAsync( - IObservable processRequests, IDictionary detectorArgs, int maxThreads, bool cleanupCreatedFiles, CancellationToken cancellationToken = default) - { - var threadsToUse = this.EnableParallelism ? Math.Min(Environment.ProcessorCount, maxThreads) : 1; - this.Telemetry["ThreadsUsed"] = $"{threadsToUse}"; - - var processor = new ActionBlock( - async processRequest => await this.OnFileFoundAsync(processRequest, detectorArgs, cleanupCreatedFiles, cancellationToken), - new ExecutionDataflowBlockOptions - { - // MaxDegreeOfParallelism is the lower of the processor count and the max threads arg that the customer passed in - MaxDegreeOfParallelism = threadsToUse, - }); - - var preprocessedObserbable = await this.OnPrepareDetectionAsync(processRequests, detectorArgs, cancellationToken); - - await preprocessedObserbable.ForEachAsync(processRequest => processor.Post(processRequest)); - - processor.Complete(); - - await processor.Completion; - - await this.OnDetectionFinishedAsync(); - - return new IndividualDetectorScanResult - { - ResultCode = ProcessingResultCode.Success, - AdditionalTelemetryDetails = this.Telemetry.ToDictionary(kvp => kvp.Key, kvp => kvp.Value), - }; - } - /// /// Auxliary method executed before the actual scanning of a given file takes place. /// This method can be used to modify or create new ProcessRequests that later will @@ -174,4 +133,45 @@ protected virtual Task OnDetectionFinishedAsync() // Do not cleanup by default, only if the detector uses the FileComponentWithCleanup abstract class. protected virtual async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, bool cleanupCreatedFiles, CancellationToken cancellationToken = default) => await this.OnFileFoundAsync(processRequest, detectorArgs, cancellationToken); + + private Task ScanDirectoryAsync(ScanRequest request, CancellationToken cancellationToken = default) + { + this.CurrentScanRequest = request; + + var filteredObservable = this.Scanner.GetFilteredComponentStreamObservable(request.SourceDirectory, this.SearchPatterns, request.ComponentRecorder); + + this.Logger.LogDebug("Registered {Detector}", this.GetType().FullName); + return this.ProcessAsync(filteredObservable, request.DetectorArgs, request.MaxThreads, request.CleanupCreatedFiles, cancellationToken); + } + + private async Task ProcessAsync( + IObservable processRequests, IDictionary detectorArgs, int maxThreads, bool cleanupCreatedFiles, CancellationToken cancellationToken = default) + { + var threadsToUse = this.EnableParallelism ? Math.Min(Environment.ProcessorCount, maxThreads) : 1; + this.Telemetry["ThreadsUsed"] = $"{threadsToUse}"; + + var processor = new ActionBlock( + async processRequest => await this.OnFileFoundAsync(processRequest, detectorArgs, cleanupCreatedFiles, cancellationToken), + new ExecutionDataflowBlockOptions + { + // MaxDegreeOfParallelism is the lower of the processor count and the max threads arg that the customer passed in + MaxDegreeOfParallelism = threadsToUse, + }); + + var preprocessedObserbable = await this.OnPrepareDetectionAsync(processRequests, detectorArgs, cancellationToken); + + await preprocessedObserbable.ForEachAsync(processRequest => processor.Post(processRequest)); + + processor.Complete(); + + await processor.Completion; + + await this.OnDetectionFinishedAsync(); + + return new IndividualDetectorScanResult + { + ResultCode = ProcessingResultCode.Success, + AdditionalTelemetryDetails = this.Telemetry.ToDictionary(kvp => kvp.Key, kvp => kvp.Value), + }; + } } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs index d74edc9d1..f3640b511 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs @@ -7,11 +7,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class CargoComponent : TypedComponent { - private CargoComponent() - { - // reserved for deserialization - } - public CargoComponent(string name, string version, string author = null, string license = null, string source = null) { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Cargo)); @@ -21,6 +16,11 @@ public CargoComponent(string name, string version, string author = null, string this.Source = source; } + private CargoComponent() + { + // reserved for deserialization + } + [JsonPropertyName("name")] public string Name { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ConanComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ConanComponent.cs index b3e0ff64c..764f1afe1 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ConanComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ConanComponent.cs @@ -6,11 +6,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class ConanComponent : TypedComponent { - private ConanComponent() - { - // reserved for deserialization - } - public ConanComponent(string name, string version, string previous, string packageId) { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Conan)); @@ -19,6 +14,11 @@ public ConanComponent(string name, string version, string previous, string packa this.Sha1Hash = this.ValidateRequiredInput(packageId, nameof(this.Sha1Hash), nameof(ComponentType.Conan)); } + private ConanComponent() + { + // reserved for deserialization + } + [JsonPropertyName("name")] public string Name { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs index 80df152cd..e8dfa98c3 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs @@ -5,11 +5,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class DockerImageComponent : TypedComponent { - private DockerImageComponent() - { - /* Reserved for deserialization */ - } - public DockerImageComponent(string hash, string name = null, string tag = null) { this.Digest = this.ValidateRequiredInput(hash, nameof(this.Digest), nameof(ComponentType.DockerImage)); @@ -17,6 +12,11 @@ public DockerImageComponent(string hash, string name = null, string tag = null) this.Tag = tag; } + private DockerImageComponent() + { + /* Reserved for deserialization */ + } + [JsonPropertyName("name")] public string Name { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs index 26a12d431..dd22b2391 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs @@ -8,11 +8,6 @@ public class DotNetComponent : TypedComponent { private const string UnknownValue = "unknown"; - private DotNetComponent() - { - /* Reserved for deserialization */ - } - public DotNetComponent(string sdkVersion, string targetFramework = null, string projectType = null) { if (string.IsNullOrWhiteSpace(sdkVersion) && string.IsNullOrWhiteSpace(targetFramework)) @@ -25,6 +20,11 @@ public DotNetComponent(string sdkVersion, string targetFramework = null, string this.ProjectType = string.IsNullOrWhiteSpace(projectType) ? UnknownValue : projectType; } + private DotNetComponent() + { + /* Reserved for deserialization */ + } + /// /// SDK Version detected, could be null if no global.json exists and no dotnet is on the path. /// diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs index fd0080523..474363243 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs @@ -41,8 +41,6 @@ private GoComponent() public override ComponentType Type => ComponentType.Go; - protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}"; - public override bool Equals(object obj) { return obj is GoComponent otherComponent && this.Equals(otherComponent); @@ -62,4 +60,6 @@ public override int GetHashCode() { return this.Name.GetHashCode() ^ this.Version.GetHashCode() ^ this.Hash.GetHashCode(); } + + protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs index 5622fb90e..e9858954b 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs @@ -7,11 +7,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class LinuxComponent : TypedComponent { - private LinuxComponent() - { - /* Reserved for deserialization */ - } - public LinuxComponent(string distribution, string release, string name, string version, string license = null, string author = null) { this.Distribution = this.ValidateRequiredInput(distribution, nameof(this.Distribution), nameof(ComponentType.Linux)); @@ -22,6 +17,11 @@ public LinuxComponent(string distribution, string release, string name, string v this.Author = author; } + private LinuxComponent() + { + /* Reserved for deserialization */ + } + [JsonPropertyName("distribution")] public string Distribution { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs index 58ecb29e4..62b9fb157 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs @@ -7,11 +7,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class NpmComponent : TypedComponent { - private NpmComponent() - { - /* Reserved for deserialization */ - } - public NpmComponent(string name, string version, string hash = null, NpmAuthor author = null) { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Npm)); @@ -20,6 +15,11 @@ public NpmComponent(string name, string version, string hash = null, NpmAuthor a this.Author = author; } + private NpmComponent() + { + /* Reserved for deserialization */ + } + [JsonPropertyName("name")] public string Name { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs index a7b8870ec..ebac8eef3 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs @@ -6,11 +6,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class NuGetComponent : TypedComponent { - private NuGetComponent() - { - /* Reserved for deserialization */ - } - public NuGetComponent(string name, string version, string[] authors = null) { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.NuGet)); @@ -18,6 +13,11 @@ public NuGetComponent(string name, string version, string[] authors = null) this.Authors = authors; } + private NuGetComponent() + { + /* Reserved for deserialization */ + } + [JsonPropertyName("name")] public string Name { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs index 4eb104801..10414c8dc 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs @@ -6,11 +6,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class OtherComponent : TypedComponent { - private OtherComponent() - { - /* Reserved for deserialization */ - } - public OtherComponent(string name, string version, Uri downloadUrl, string hash) { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Other)); @@ -19,6 +14,11 @@ public OtherComponent(string name, string version, Uri downloadUrl, string hash) this.Hash = hash; } + private OtherComponent() + { + /* Reserved for deserialization */ + } + [JsonPropertyName("name")] public string Name { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs index b7d1a14d1..16cea5997 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs @@ -8,11 +8,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class PipComponent : TypedComponent { - private PipComponent() - { - /* Reserved for deserialization */ - } - public PipComponent(string name, string version, string author = null, string license = null) { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Pip)); @@ -21,6 +16,11 @@ public PipComponent(string name, string version, string author = null, string li this.License = license; } + private PipComponent() + { + /* Reserved for deserialization */ + } + [JsonPropertyName("name")] public string Name { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs index 2e1558f12..d79725455 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs @@ -7,11 +7,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class PodComponent : TypedComponent { - private PodComponent() - { - /* Reserved for deserialization */ - } - public PodComponent(string name, string version, string specRepo = "") { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Pod)); @@ -19,6 +14,11 @@ public PodComponent(string name, string version, string specRepo = "") this.SpecRepo = specRepo; } + private PodComponent() + { + /* Reserved for deserialization */ + } + [JsonPropertyName("name")] public string Name { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs index aa434b203..af053582e 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs @@ -6,11 +6,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class RubyGemsComponent : TypedComponent { - private RubyGemsComponent() - { - /* Reserved for deserialization */ - } - public RubyGemsComponent(string name, string version, string source = "") { this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.RubyGems)); @@ -18,6 +13,11 @@ public RubyGemsComponent(string name, string version, string source = "") this.Source = source; } + private RubyGemsComponent() + { + /* Reserved for deserialization */ + } + [JsonPropertyName("name")] public string Name { get; set; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs index e6cf2cd79..428a02dd5 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs @@ -6,11 +6,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class SpdxComponent : TypedComponent { - private SpdxComponent() - { - /* Reserved for deserialization */ - } - public SpdxComponent(string spdxVersion, Uri documentNamespace, string name, string checksum, string rootElementId, string path) { this.SpdxVersion = this.ValidateRequiredInput(spdxVersion, nameof(this.SpdxVersion), nameof(ComponentType.Spdx)); @@ -21,6 +16,11 @@ public SpdxComponent(string spdxVersion, Uri documentNamespace, string name, str this.Path = this.ValidateRequiredInput(path, nameof(this.Path), nameof(ComponentType.Spdx)); } + private SpdxComponent() + { + /* Reserved for deserialization */ + } + public override ComponentType Type => ComponentType.Spdx; [JsonPropertyName("rootElementId")] diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs index 1b1af859e..07df800e2 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs @@ -6,11 +6,6 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; public class VcpkgComponent : TypedComponent { - private VcpkgComponent() - { - /* Reserved for deserialization */ - } - public VcpkgComponent(string spdxid, string name, string version, string triplet = null, string portVersion = null, string description = null, string downloadLocation = null) { int.TryParse(portVersion, out var port); @@ -24,6 +19,11 @@ public VcpkgComponent(string spdxid, string name, string version, string triplet this.DownloadLocation = downloadLocation; } + private VcpkgComponent() + { + /* Reserved for deserialization */ + } + [JsonPropertyName("spdxid")] public string SPDXID { get; set; }