Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 17 additions & 3 deletions .editorconfig
Original file line number Diff line number Diff line change
Expand Up @@ -463,9 +463,26 @@ dotnet_diagnostic.SA1623.severity = none
# https://github.com/DotNetAnalyzers/StyleCopAnalyzers
##########################################

# https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1009.md
# Closing parenthesis should be spaced correctly
dotnet_diagnostic.SA1009.severity = none

# https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1101.md
# Local calls should be prefixed with `this`
dotnet_diagnostic.SA1101.severity = warning

# https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1111.md
# Closing parenthesis should be on the line of last parameter
dotnet_diagnostic.SA1111.severity = none

# https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1201.md
# Elements should be in the correct order
dotnet_diagnostic.SA1201.severity = warning

# https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1202.md
# Elements should be ordered by access
dotnet_diagnostic.SA1202.severity = none

# https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1600.md
# Elements should be documented
dotnet_diagnostic.SA1600.severity = suggestion
Expand Down Expand Up @@ -612,9 +629,6 @@ dotnet_diagnostic.CA1837.severity = suggestion
# CA1024: Use properties where appropriate
dotnet_diagnostic.CA1024.severity = suggestion

# SA1202: Elements should be ordered by access
dotnet_diagnostic.SA1202.severity = suggestion

# IDE0022: Use expression body for methods
dotnet_diagnostic.IDE0022.severity = suggestion

Expand Down
124 changes: 62 additions & 62 deletions src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,6 @@ namespace Microsoft.ComponentDetection.Contracts;
/// <summary>Specialized base class for file based component detection.</summary>
public abstract class FileComponentDetector : IComponentDetector
{
/// <summary>
/// Gets or sets the factory for handing back component streams to File detectors.
/// </summary>
protected IComponentStreamEnumerableFactory ComponentStreamEnumerableFactory { get; set; }

protected IObservableDirectoryWalkerFactory Scanner { get; set; }

/// <summary>
/// Gets or sets the logger for writing basic logging message to both console and file.
/// </summary>
protected ILogger Logger { get; set; }

public IComponentRecorder ComponentRecorder { get; private set; }

/// <inheritdoc />
Expand All @@ -46,25 +34,37 @@ public abstract class FileComponentDetector : IComponentDetector
/// <summary>Gets the version of this component detector. </summary>
public abstract int Version { get; }

public virtual bool NeedsAutomaticRootDependencyCalculation { get; protected set; }

/// <summary>
/// Gets the folder names that will be skipped by the Component Detector.
/// List of any any additional properties as key-value pairs that we would like to capture for the detector.
/// </summary>
protected virtual IList<string> SkippedFolders => [];
public List<(string PropertyKey, string PropertyValue)> AdditionalProperties { get; set; } = [];

protected ConcurrentDictionary<string, string> Telemetry { get; set; } = [];

/// <summary>
/// Gets or sets the active scan request -- only populated after a ScanDirectoryAsync is invoked. If ScanDirectoryAsync is overridden,
/// the overrider should ensure this property is populated.
/// Gets or sets the factory for handing back component streams to File detectors.
/// </summary>
protected ScanRequest CurrentScanRequest { get; set; }
protected IComponentStreamEnumerableFactory ComponentStreamEnumerableFactory { get; set; }

public virtual bool NeedsAutomaticRootDependencyCalculation { get; protected set; }
protected IObservableDirectoryWalkerFactory Scanner { get; set; }

protected ConcurrentDictionary<string, string> Telemetry { get; set; } = [];
/// <summary>
/// Gets or sets the logger for writing basic logging message to both console and file.
/// </summary>
protected ILogger Logger { get; set; }

/// <summary>
/// List of any any additional properties as key-value pairs that we would like to capture for the detector.
/// Gets the folder names that will be skipped by the Component Detector.
/// </summary>
public List<(string PropertyKey, string PropertyValue)> AdditionalProperties { get; set; } = [];
protected virtual IList<string> SkippedFolders => [];

/// <summary>
/// Gets or sets the active scan request -- only populated after a ScanDirectoryAsync is invoked. If ScanDirectoryAsync is overridden,
/// the overrider should ensure this property is populated.
/// </summary>
protected ScanRequest CurrentScanRequest { get; set; }

protected IObservable<IComponentStream> ComponentStreams { get; private set; }

Expand All @@ -78,16 +78,6 @@ public async virtual Task<IndividualDetectorScanResult> ExecuteDetectorAsync(Sca
return await this.ScanDirectoryAsync(request, cancellationToken);
}

private Task<IndividualDetectorScanResult> ScanDirectoryAsync(ScanRequest request, CancellationToken cancellationToken = default)
{
this.CurrentScanRequest = request;

var filteredObservable = this.Scanner.GetFilteredComponentStreamObservable(request.SourceDirectory, this.SearchPatterns, request.ComponentRecorder);

this.Logger.LogDebug("Registered {Detector}", this.GetType().FullName);
return this.ProcessAsync(filteredObservable, request.DetectorArgs, request.MaxThreads, request.CleanupCreatedFiles, cancellationToken);
}

/// <summary>
/// Gets the file streams for the Detector's declared <see cref="SearchPatterns"/> as an <see cref="IEnumerable{IComponentStream}"/>.
/// </summary>
Expand All @@ -111,37 +101,6 @@ protected Task<IEnumerable<IComponentStream>> GetFileStreamsAsync(DirectoryInfo
/// <param name="lockfileVersion">The lockfile version.</param>
protected void RecordLockfileVersion(string lockfileVersion) => this.Telemetry["LockfileVersion"] = lockfileVersion;

private async Task<IndividualDetectorScanResult> ProcessAsync(
IObservable<ProcessRequest> processRequests, IDictionary<string, string> detectorArgs, int maxThreads, bool cleanupCreatedFiles, CancellationToken cancellationToken = default)
{
var threadsToUse = this.EnableParallelism ? Math.Min(Environment.ProcessorCount, maxThreads) : 1;
this.Telemetry["ThreadsUsed"] = $"{threadsToUse}";

var processor = new ActionBlock<ProcessRequest>(
async processRequest => await this.OnFileFoundAsync(processRequest, detectorArgs, cleanupCreatedFiles, cancellationToken),
new ExecutionDataflowBlockOptions
{
// MaxDegreeOfParallelism is the lower of the processor count and the max threads arg that the customer passed in
MaxDegreeOfParallelism = threadsToUse,
});

var preprocessedObserbable = await this.OnPrepareDetectionAsync(processRequests, detectorArgs, cancellationToken);

await preprocessedObserbable.ForEachAsync(processRequest => processor.Post(processRequest));

processor.Complete();

await processor.Completion;

await this.OnDetectionFinishedAsync();

return new IndividualDetectorScanResult
{
ResultCode = ProcessingResultCode.Success,
AdditionalTelemetryDetails = this.Telemetry.ToDictionary(kvp => kvp.Key, kvp => kvp.Value),
};
}

/// <summary>
/// Auxliary method executed before the actual scanning of a given file takes place.
/// This method can be used to modify or create new ProcessRequests that later will
Expand Down Expand Up @@ -174,4 +133,45 @@ protected virtual Task OnDetectionFinishedAsync()
// Do not cleanup by default, only if the detector uses the FileComponentWithCleanup abstract class.
protected virtual async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary<string, string> detectorArgs, bool cleanupCreatedFiles, CancellationToken cancellationToken = default) =>
await this.OnFileFoundAsync(processRequest, detectorArgs, cancellationToken);

private Task<IndividualDetectorScanResult> ScanDirectoryAsync(ScanRequest request, CancellationToken cancellationToken = default)
{
this.CurrentScanRequest = request;

var filteredObservable = this.Scanner.GetFilteredComponentStreamObservable(request.SourceDirectory, this.SearchPatterns, request.ComponentRecorder);

this.Logger.LogDebug("Registered {Detector}", this.GetType().FullName);
return this.ProcessAsync(filteredObservable, request.DetectorArgs, request.MaxThreads, request.CleanupCreatedFiles, cancellationToken);
}

private async Task<IndividualDetectorScanResult> ProcessAsync(
IObservable<ProcessRequest> processRequests, IDictionary<string, string> detectorArgs, int maxThreads, bool cleanupCreatedFiles, CancellationToken cancellationToken = default)
{
var threadsToUse = this.EnableParallelism ? Math.Min(Environment.ProcessorCount, maxThreads) : 1;
this.Telemetry["ThreadsUsed"] = $"{threadsToUse}";

var processor = new ActionBlock<ProcessRequest>(
async processRequest => await this.OnFileFoundAsync(processRequest, detectorArgs, cleanupCreatedFiles, cancellationToken),
new ExecutionDataflowBlockOptions
{
// MaxDegreeOfParallelism is the lower of the processor count and the max threads arg that the customer passed in
MaxDegreeOfParallelism = threadsToUse,
});

var preprocessedObserbable = await this.OnPrepareDetectionAsync(processRequests, detectorArgs, cancellationToken);

await preprocessedObserbable.ForEachAsync(processRequest => processor.Post(processRequest));

processor.Complete();

await processor.Completion;

await this.OnDetectionFinishedAsync();

return new IndividualDetectorScanResult
{
ResultCode = ProcessingResultCode.Success,
AdditionalTelemetryDetails = this.Telemetry.ToDictionary(kvp => kvp.Key, kvp => kvp.Value),
};
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,6 @@ public GoComponent()

public override ComponentType Type => ComponentType.Go;

protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}";

public override bool Equals(object obj)
{
return obj is GoComponent otherComponent && this.Equals(otherComponent);
Expand All @@ -62,4 +60,6 @@ public override int GetHashCode()
{
return this.Name.GetHashCode() ^ this.Version.GetHashCode() ^ this.Hash.GetHashCode();
}

protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}";
}