From e1d7911c64a0c01dbe1067f590fb5446b8858268 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Thu, 5 Mar 2026 19:28:30 -0800 Subject: [PATCH 01/26] Create BinLogDetector --- Directory.Packages.props | 1 + ...rosoft.ComponentDetection.Detectors.csproj | 1 + .../nuget/BinLogProcessor.cs | 302 ++++++++++++ .../nuget/IBinLogProcessor.cs | 16 + .../nuget/LockFileUtilities.cs | 281 +++++++++++ .../MSBuildBinaryLogComponentDetector.cs | 456 ++++++++++++++++++ .../nuget/MSBuildProjectInfo.cs | 193 ++++++++ .../Configs/MSBuildBinaryLogExperiment.cs | 25 + .../Extensions/ServiceCollectionExtensions.cs | 2 + .../MSBuildBinaryLogComponentDetectorTests.cs | 372 ++++++++++++++ 10 files changed, 1649 insertions(+) create mode 100644 src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs create mode 100644 src/Microsoft.ComponentDetection.Detectors/nuget/IBinLogProcessor.cs create mode 100644 src/Microsoft.ComponentDetection.Detectors/nuget/LockFileUtilities.cs create mode 100644 src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs create mode 100644 src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildProjectInfo.cs create mode 100644 src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/MSBuildBinaryLogExperiment.cs create mode 100644 test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index e96973080..a74da6434 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -16,6 +16,7 @@ + diff --git a/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj b/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj index a0fb3f5e4..281de71ce 100644 --- a/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj +++ b/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj @@ -2,6 +2,7 @@ + diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs new file mode 100644 index 000000000..48a18e314 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs @@ -0,0 +1,302 @@ +namespace Microsoft.ComponentDetection.Detectors.NuGet; + +using System; +using System.Collections; +using System.Collections.Generic; +using Microsoft.Build.Framework; +using Microsoft.Build.Logging.StructuredLogger; +using Microsoft.Extensions.Logging; + +/// +/// Processes MSBuild binary log files to extract project information. +/// +internal class BinLogProcessor : IBinLogProcessor +{ + private readonly Microsoft.Extensions.Logging.ILogger logger; + + /// + /// Initializes a new instance of the class. + /// + /// Logger for diagnostic messages. + public BinLogProcessor(Microsoft.Extensions.Logging.ILogger logger) => this.logger = logger; + + /// + public IReadOnlyList ExtractProjectInfo(string binlogPath) + { + // Maps project path to the primary MSBuildProjectInfo for that project + var projectInfoByPath = new Dictionary(StringComparer.OrdinalIgnoreCase); + + try + { + var reader = new BinLogReader(); + + // Maps evaluation ID to MSBuildProjectInfo being populated + var projectInfoByEvaluationId = new Dictionary(); + + // Maps project instance ID to evaluation ID + var projectInstanceToEvaluationMap = new Dictionary(); + + // Hook into status events to capture property evaluations + reader.StatusEventRaised += (sender, e) => + { + if (e?.BuildEventContext?.EvaluationId >= 0 && + e is ProjectEvaluationFinishedEventArgs projectEvalArgs) + { + var projectInfo = new MSBuildProjectInfo(); + this.PopulateFromEvaluation(projectEvalArgs, projectInfo); + projectInfoByEvaluationId[e.BuildEventContext.EvaluationId] = projectInfo; + } + }; + + // Hook into project started to map project instance to evaluation and capture project path + reader.ProjectStarted += (sender, e) => + { + if (e?.BuildEventContext?.EvaluationId >= 0 && + e?.BuildEventContext?.ProjectInstanceId >= 0) + { + projectInstanceToEvaluationMap[e.BuildEventContext.ProjectInstanceId] = e.BuildEventContext.EvaluationId; + + // Set the project path on the MSBuildProjectInfo + if (!string.IsNullOrEmpty(e.ProjectFile) && + projectInfoByEvaluationId.TryGetValue(e.BuildEventContext.EvaluationId, out var projectInfo)) + { + projectInfo.ProjectPath = e.ProjectFile; + } + } + }; + + // Hook into any event to capture property reassignments and item changes during build + reader.AnyEventRaised += (sender, e) => + { + this.HandleBuildEvent(e, projectInstanceToEvaluationMap, projectInfoByEvaluationId); + }; + + // Hook into project finished to collect final project info and establish hierarchy + reader.ProjectFinished += (sender, e) => + { + if (e?.BuildEventContext?.ProjectInstanceId >= 0 && + projectInstanceToEvaluationMap.TryGetValue(e.BuildEventContext.ProjectInstanceId, out var evaluationId) && + projectInfoByEvaluationId.TryGetValue(evaluationId, out var projectInfo) && + !string.IsNullOrEmpty(projectInfo.ProjectPath)) + { + this.AddOrMergeProjectInfo(projectInfo, projectInfoByPath); + } + }; + + reader.Replay(binlogPath); + } + catch (Exception ex) + { + this.logger.LogWarning(ex, "Error parsing binlog: {BinlogPath}", binlogPath); + } + + return [.. projectInfoByPath.Values]; + } + + /// + /// Adds a project info to the results, merging with existing entries for the same project path. + /// Outer builds become the primary entry; inner builds are added as children. + /// + private void AddOrMergeProjectInfo( + MSBuildProjectInfo projectInfo, + Dictionary projectInfoByPath) + { + if (!projectInfoByPath.TryGetValue(projectInfo.ProjectPath!, out var existing)) + { + // First time seeing this project - add it + projectInfoByPath[projectInfo.ProjectPath!] = projectInfo; + return; + } + + // We've seen this project before - determine how to merge + if (projectInfo.IsOuterBuild && !existing.IsOuterBuild) + { + // New build is outer, existing is inner - outer becomes primary + // Move existing to be an inner build of the new outer build + projectInfo.InnerBuilds.Add(existing); + + // Also move any inner builds that were already collected + foreach (var inner in existing.InnerBuilds) + { + projectInfo.InnerBuilds.Add(inner); + } + + existing.InnerBuilds.Clear(); + + // Replace in the lookup + projectInfoByPath[projectInfo.ProjectPath!] = projectInfo; + } + else if (existing.IsOuterBuild && !projectInfo.IsOuterBuild && !string.IsNullOrEmpty(projectInfo.TargetFramework)) + { + // Existing is outer, new is inner - add new as inner build + existing.InnerBuilds.Add(projectInfo); + } + else if (!existing.IsOuterBuild && !projectInfo.IsOuterBuild && !string.IsNullOrEmpty(projectInfo.TargetFramework)) + { + // Both are inner builds (no outer build seen yet) - add to InnerBuilds of the first one + // The first one acts as a placeholder until we see an outer build + existing.InnerBuilds.Add(projectInfo); + } + + // Otherwise: duplicate builds currently ignored. + } + + /// + /// Populates project info from evaluation results (properties and items). + /// + private void PopulateFromEvaluation(ProjectEvaluationFinishedEventArgs projectEvalArgs, MSBuildProjectInfo projectInfo) + { + // Extract properties + if (projectEvalArgs?.Properties != null) + { + // Handle different property collection types based on MSBuild version + // Newer MSBuild versions may provide IDictionary + if (projectEvalArgs.Properties is IDictionary propertiesDict) + { + foreach (var kvp in propertiesDict) + { + projectInfo.TrySetProperty(kvp.Key, kvp.Value); + } + } + else + { + // Older format uses IEnumerable with DictionaryEntry or KeyValuePair + foreach (var property in projectEvalArgs.Properties) + { + string? key = null; + string? value = null; + + if (property is DictionaryEntry entry) + { + key = entry.Key as string; + value = entry.Value as string; + } + else if (property is KeyValuePair kvp) + { + key = kvp.Key; + value = kvp.Value; + } + + if (!string.IsNullOrEmpty(key)) + { + projectInfo.TrySetProperty(key, value ?? string.Empty); + } + } + } + } + + // Extract items + if (projectEvalArgs?.Items != null) + { + // Items is an IEnumerable that contains item groups + // Each item group has an ItemType (Key) and Items collection (Value) + foreach (var itemGroup in projectEvalArgs.Items) + { + if (itemGroup is DictionaryEntry entry && + entry.Key is string itemType && + MSBuildProjectInfo.IsItemTypeOfInterest(itemType) && + entry.Value is IEnumerable groupItems) + { + foreach (var item in groupItems) + { + if (item is ITaskItem taskItem) + { + projectInfo.TryAddOrUpdateItem(itemType, taskItem); + } + } + } + } + } + } + + /// + /// Handles build events to capture property and item changes during target execution. + /// + private void HandleBuildEvent( + BuildEventArgs? args, + Dictionary projectInstanceToEvaluationMap, + Dictionary projectInfoByEvaluationId) + { + if (!this.TryGetProjectInfo(args, projectInstanceToEvaluationMap, projectInfoByEvaluationId, out var projectInfo)) + { + return; + } + + switch (args) + { + // Property reassignments (when a property value changes during the build) + case PropertyReassignmentEventArgs propertyReassignment: + projectInfo.TrySetProperty(propertyReassignment.PropertyName, propertyReassignment.NewValue); + break; + + // Initial property value set events + case PropertyInitialValueSetEventArgs propertyInitialValueSet: + projectInfo.TrySetProperty(propertyInitialValueSet.PropertyName, propertyInitialValueSet.PropertyValue); + break; + + // Task parameter events which can contain item arrays for add/remove/update + case TaskParameterEventArgs taskParameter when + MSBuildProjectInfo.IsItemTypeOfInterest(taskParameter.ItemType) && + taskParameter.Items is IList taskItems: + this.ProcessTaskParameterItems(taskParameter.Kind, taskParameter.ItemType, taskItems, projectInfo); + break; + + default: + break; + } + } + + /// + /// Tries to get the project info for a build event. + /// + private bool TryGetProjectInfo( + BuildEventArgs? args, + Dictionary projectInstanceToEvaluationMap, + Dictionary projectInfoByEvaluationId, + out MSBuildProjectInfo projectInfo) + { + projectInfo = null!; + + if (args?.BuildEventContext?.ProjectInstanceId == null || args.BuildEventContext.ProjectInstanceId < 0) + { + return false; + } + + if (!projectInstanceToEvaluationMap.TryGetValue(args.BuildEventContext.ProjectInstanceId, out var evaluationId) || + !projectInfoByEvaluationId.TryGetValue(evaluationId, out projectInfo!)) + { + return false; + } + + return true; + } + + /// + /// Processes task parameter items for add/remove operations. + /// + private void ProcessTaskParameterItems( + TaskParameterMessageKind kind, + string itemType, + IList items, + MSBuildProjectInfo projectInfo) + { + if (kind == TaskParameterMessageKind.RemoveItem) + { + foreach (var item in items) + { + projectInfo.TryRemoveItem(itemType, item.ItemSpec); + } + } + else if (kind == TaskParameterMessageKind.TaskInput || + kind == TaskParameterMessageKind.AddItem || + kind == TaskParameterMessageKind.TaskOutput) + { + foreach (var item in items) + { + projectInfo.TryAddOrUpdateItem(itemType, item); + } + } + + // SkippedTargetInputs and SkippedTargetOutputs are informational and don't modify items + } +} diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/IBinLogProcessor.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/IBinLogProcessor.cs new file mode 100644 index 000000000..44a1733f0 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/IBinLogProcessor.cs @@ -0,0 +1,16 @@ +namespace Microsoft.ComponentDetection.Detectors.NuGet; + +using System.Collections.Generic; + +/// +/// Interface for processing MSBuild binary log files to extract project information. +/// +internal interface IBinLogProcessor +{ + /// + /// Extracts project information from a binary log file. + /// + /// Path to the binary log file. + /// Collection of project information extracted from the binlog. + IReadOnlyList ExtractProjectInfo(string binlogPath); +} diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/LockFileUtilities.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/LockFileUtilities.cs new file mode 100644 index 000000000..9cb98d135 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/LockFileUtilities.cs @@ -0,0 +1,281 @@ +namespace Microsoft.ComponentDetection.Detectors.NuGet; + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using global::NuGet.Packaging.Core; +using global::NuGet.ProjectModel; +using global::NuGet.Versioning; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; + +/// +/// Shared utility methods for processing NuGet lock files (project.assets.json). +/// Used by both NuGetProjectModelProjectCentricComponentDetector and MSBuildBinaryLogComponentDetector. +/// +public static class LockFileUtilities +{ + /// + /// Dependency type constant for project references in project.assets.json. + /// + public const string ProjectDependencyType = "project"; + + /// + /// Gets the framework references for a given lock file target. + /// + /// The lock file to analyze. + /// The target framework to get references for. + /// Array of framework reference names. + public static string[] GetFrameworkReferences(LockFile lockFile, LockFileTarget target) + { + var frameworkInformation = lockFile.PackageSpec?.TargetFrameworks + .FirstOrDefault(x => x.FrameworkName.Equals(target.TargetFramework)); + + if (frameworkInformation == null) + { + return []; + } + + // Add directly referenced frameworks + var results = frameworkInformation.FrameworkReferences.Select(x => x.Name); + + // Add transitive framework references + results = results.Concat(target.Libraries.SelectMany(l => l.FrameworkReferences)); + + return results.Distinct().ToArray(); + } + + /// + /// Determines if a library is a development dependency based on its content. + /// A placeholder item is an empty file that doesn't exist with name _._ meant to indicate + /// an empty folder in a nuget package, but also used by NuGet when a package's assets are excluded. + /// + /// The library to check. + /// The lock file containing library metadata. + /// True if the library is a development dependency. + public static bool IsADevelopmentDependency(LockFileTargetLibrary library, LockFile lockFile) + { + static bool IsAPlaceholderItem(LockFileItem item) => + Path.GetFileName(item.Path).Equals(PackagingCoreConstants.EmptyFolder, StringComparison.OrdinalIgnoreCase); + + // All(IsAPlaceholderItem) checks if the collection is empty or all items are placeholders. + return library.RuntimeAssemblies.All(IsAPlaceholderItem) && + library.RuntimeTargets.All(IsAPlaceholderItem) && + library.ResourceAssemblies.All(IsAPlaceholderItem) && + library.NativeLibraries.All(IsAPlaceholderItem) && + library.ContentFiles.All(IsAPlaceholderItem) && + library.Build.All(IsAPlaceholderItem) && + library.BuildMultiTargeting.All(IsAPlaceholderItem) && + + // The SDK looks at the library for analyzers using the following heuristic: + // https://github.com/dotnet/sdk/blob/d7fe6e66d8f67dc93c5c294a75f42a2924889196/src/Tasks/Microsoft.NET.Build.Tasks/NuGetUtils.NuGet.cs#L43 + (!lockFile.GetLibrary(library.Name, library.Version)?.Files + .Any(file => file.StartsWith("analyzers", StringComparison.Ordinal) + && file.EndsWith(".dll", StringComparison.OrdinalIgnoreCase) + && !file.EndsWith(".resources.dll", StringComparison.OrdinalIgnoreCase)) ?? false); + } + + /// + /// Gets the top-level libraries (direct dependencies) from a lock file. + /// + /// The lock file to analyze. + /// List of top-level library information. + public static List<(string Name, Version? Version, VersionRange? VersionRange)> GetTopLevelLibraries(LockFile lockFile) + { + var toBeFilled = new List<(string Name, Version? Version, VersionRange? VersionRange)>(); + + if (lockFile.PackageSpec?.TargetFrameworks != null) + { + foreach (var framework in lockFile.PackageSpec.TargetFrameworks) + { + foreach (var dependency in framework.Dependencies) + { + toBeFilled.Add((dependency.Name, Version: null, dependency.LibraryRange.VersionRange)); + } + } + } + + var projectDirectory = lockFile.PackageSpec?.RestoreMetadata?.ProjectPath != null + ? Path.GetDirectoryName(lockFile.PackageSpec.RestoreMetadata.ProjectPath) + : null; + + if (projectDirectory != null && lockFile.Libraries != null) + { + var librariesWithAbsolutePath = lockFile.Libraries + .Where(x => x.Type == ProjectDependencyType) + .Select(x => (library: x, absoluteProjectPath: Path.GetFullPath(Path.Combine(projectDirectory, x.Path)))) + .ToDictionary(x => x.absoluteProjectPath, x => x.library); + + if (lockFile.PackageSpec?.RestoreMetadata?.TargetFrameworks != null) + { + foreach (var restoreMetadataTargetFramework in lockFile.PackageSpec.RestoreMetadata.TargetFrameworks) + { + foreach (var projectReference in restoreMetadataTargetFramework.ProjectReferences) + { + if (librariesWithAbsolutePath.TryGetValue(Path.GetFullPath(projectReference.ProjectPath), out var library)) + { + toBeFilled.Add((library.Name, library.Version?.Version, null)); + } + } + } + } + } + + return toBeFilled; + } + + /// + /// Looks up a library in project.assets.json given a version (preferred) or version range. + /// + /// The list of libraries to search. + /// The dependency name to find. + /// The specific version to match (mutually exclusive with versionRange). + /// The version range to match (mutually exclusive with version). + /// Optional logger for debug messages. + /// The matching library, or null if not found. + public static LockFileLibrary? GetLibraryComponentWithDependencyLookup( + IList? libraries, + string dependencyId, + Version? version, + VersionRange? versionRange, + ILogger? logger = null) + { + if (libraries == null) + { + return null; + } + + if ((version == null && versionRange == null) || (version != null && versionRange != null)) + { + logger?.LogDebug("Either version or versionRange must be specified, but not both for {DependencyId}.", dependencyId); + return null; + } + + var matchingLibraryNames = libraries.Where(x => string.Equals(x.Name, dependencyId, StringComparison.OrdinalIgnoreCase)).ToList(); + + if (matchingLibraryNames.Count == 0) + { + logger?.LogDebug("No library found matching: {DependencyId}", dependencyId); + return null; + } + + LockFileLibrary? matchingLibrary; + if (version != null) + { + matchingLibrary = matchingLibraryNames.FirstOrDefault(x => x.Version?.Version?.Equals(version) ?? false); + } + else + { + matchingLibrary = matchingLibraryNames.FirstOrDefault(x => x.Version != null && versionRange!.Satisfies(x.Version)); + } + + if (matchingLibrary == null) + { + matchingLibrary = matchingLibraryNames.First(); + var versionString = versionRange != null ? versionRange.ToNormalizedString() : version?.ToString(); + logger?.LogDebug( + "Couldn't satisfy lookup for {Version}. Falling back to first found component for {MatchingLibraryName}, resolving to version {MatchingLibraryVersion}.", + versionString, + matchingLibrary.Name, + matchingLibrary.Version); + } + + return matchingLibrary; + } + + /// + /// Navigates the dependency graph and registers components with the component recorder. + /// + /// The lock file target containing dependency information. + /// Set of component IDs that are explicitly referenced. + /// The component recorder to register with. + /// The library to process. + /// The parent component ID, or null for root dependencies. + /// Function to determine if a library is a development dependency. + /// Set of already visited dependency IDs to prevent cycles. + public static void NavigateAndRegister( + LockFileTarget target, + HashSet explicitlyReferencedComponentIds, + ISingleFileComponentRecorder singleFileComponentRecorder, + LockFileTargetLibrary library, + string? parentComponentId, + Func isDevelopmentDependency, + HashSet? visited = null) + { + if (library.Type == ProjectDependencyType) + { + return; + } + + visited ??= []; + + var libraryComponent = new DetectedComponent(new NuGetComponent(library.Name, library.Version?.ToNormalizedString() ?? "0.0.0")); + + singleFileComponentRecorder.RegisterUsage( + libraryComponent, + explicitlyReferencedComponentIds.Contains(libraryComponent.Component.Id), + parentComponentId, + isDevelopmentDependency: isDevelopmentDependency(library), + targetFramework: target.TargetFramework?.GetShortFolderName()); + + foreach (var dependency in library.Dependencies) + { + if (visited.Contains(dependency.Id)) + { + continue; + } + + var targetLibrary = target.GetTargetLibrary(dependency.Id); + + if (targetLibrary != null) + { + visited.Add(dependency.Id); + NavigateAndRegister( + target, + explicitlyReferencedComponentIds, + singleFileComponentRecorder, + targetLibrary, + libraryComponent.Component.Id, + isDevelopmentDependency, + visited); + } + } + } + + /// + /// Registers PackageDownload dependencies from the lock file. + /// PackageDownload is always a development dependency since its usage does not make it part of the application. + /// + /// The component recorder to register with. + /// The lock file containing PackageDownload references. + public static void RegisterPackageDownloads(ISingleFileComponentRecorder singleFileComponentRecorder, LockFile lockFile) + { + if (lockFile.PackageSpec?.TargetFrameworks == null) + { + return; + } + + foreach (var framework in lockFile.PackageSpec.TargetFrameworks) + { + foreach (var packageDownload in framework.DownloadDependencies) + { + if (packageDownload?.Name is null || packageDownload?.VersionRange?.MinVersion is null) + { + continue; + } + + var libraryComponent = new DetectedComponent(new NuGetComponent(packageDownload.Name, packageDownload.VersionRange.MinVersion.ToNormalizedString())); + + // PackageDownload is always a development dependency since its usage does not make it part of the application + singleFileComponentRecorder.RegisterUsage( + libraryComponent, + isExplicitReferencedDependency: true, + parentComponentId: null, + isDevelopmentDependency: true, + targetFramework: framework.FrameworkName?.GetShortFolderName()); + } + } + } +} diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs new file mode 100644 index 000000000..7c3b5db53 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs @@ -0,0 +1,456 @@ +namespace Microsoft.ComponentDetection.Detectors.NuGet; + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; +using System.Threading; +using System.Threading.Tasks; +using global::NuGet.ProjectModel; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.Internal; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; +using Task = System.Threading.Tasks.Task; + +/// +/// An experimental detector that combines MSBuild binlog information with NuGet project.assets.json +/// to provide enhanced component detection with project-level classifications. +/// This detector is intended to replace both DotNetComponentDetector and NuGetProjectModelProjectCentricComponentDetector. +/// +/// +/// +/// Logic consistency notes with replaced detectors: +/// +/// +/// NuGet component detection (from NuGetProjectModelProjectCentricComponentDetector): +/// - Uses the same LockFileUtilities methods for processing project.assets.json +/// - Maintains the same logic for determining framework packages and development dependencies +/// - Registers PackageDownload dependencies the same way +/// - Uses project path from RestoreMetadata.ProjectPath for component recorder (consistent behavior). +/// +/// +/// DotNet component detection (from DotNetComponentDetector): +/// - SDK version: Binlog provides NETCoreSdkVersion which is the actual version used during build +/// (more accurate than running `dotnet --version` which may differ due to global.json rollforward) +/// - Target type: Uses OutputType property from binlog to determine "application" vs "library" +/// (DotNetComponentDetector uses PE header inspection which requires compiled output to exist.) +/// - Target frameworks: Uses TargetFramework/TargetFrameworks properties from binlog. +/// (DotNetComponentDetector uses targets from project.assets.json which is equivalent.) +/// +/// +/// Additional enhancements: +/// - IsTestProject classification: All dependencies of test projects are marked as dev dependencies. +/// - Fallback mode: When no binlog info is available, falls back to standard NuGet detection. +/// +/// +public class MSBuildBinaryLogComponentDetector : FileComponentDetector, IExperimentalDetector +{ + private readonly IBinLogProcessor binLogProcessor; + private readonly IFileUtilityService fileUtilityService; + private readonly LockFileFormat lockFileFormat = new(); + + // Track which assets files have been processed to avoid duplicate processing + private readonly ConcurrentDictionary processedAssetsFiles = new(StringComparer.OrdinalIgnoreCase); + + // Store project information extracted from binlogs keyed by project path + private readonly ConcurrentDictionary projectInfoByProjectPath = new(StringComparer.OrdinalIgnoreCase); + + // Store project information extracted from binlogs keyed by assets file path + private readonly ConcurrentDictionary projectInfoByAssetsFile = new(StringComparer.OrdinalIgnoreCase); + + /// + /// Initializes a new instance of the class. + /// + /// Factory for creating component streams. + /// Factory for directory walking. + /// Service for file operations. + /// Logger for diagnostic messages. + public MSBuildBinaryLogComponentDetector( + IComponentStreamEnumerableFactory componentStreamEnumerableFactory, + IObservableDirectoryWalkerFactory walkerFactory, + IFileUtilityService fileUtilityService, + ILogger logger) + { + this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; + this.Scanner = walkerFactory; + this.binLogProcessor = new BinLogProcessor(logger); + this.fileUtilityService = fileUtilityService; + this.Logger = logger; + } + + /// + public override string Id => "MSBuildBinaryLog"; + + /// + public override IEnumerable Categories => [Enum.GetName(typeof(DetectorClass), DetectorClass.NuGet)!]; + + /// + public override IList SearchPatterns { get; } = ["*.binlog", "project.assets.json"]; + + /// + public override IEnumerable SupportedComponentTypes { get; } = [ComponentType.NuGet, ComponentType.DotNet]; + + /// + public override int Version { get; } = 1; + + /// + protected override async Task> OnPrepareDetectionAsync( + IObservable processRequests, + IDictionary detectorArgs, + CancellationToken cancellationToken = default) + { + // Collect all requests and sort them so binlogs are processed first + // This ensures we have project info available when processing assets files + var allRequests = await processRequests.ToList(); + + this.Logger.LogDebug("Preparing detection: collected {Count} files", allRequests.Count); + + // Separate binlogs and assets files + var binlogRequests = allRequests + .Where(r => r.ComponentStream.Location.EndsWith(".binlog", StringComparison.OrdinalIgnoreCase)) + .ToList(); + + var assetsRequests = allRequests + .Where(r => r.ComponentStream.Location.EndsWith("project.assets.json", StringComparison.OrdinalIgnoreCase)) + .ToList(); + + this.Logger.LogDebug("Found {BinlogCount} binlog files and {AssetsCount} assets files", binlogRequests.Count, assetsRequests.Count); + + // Return binlogs first, then assets files + var orderedRequests = binlogRequests.Concat(assetsRequests); + + return orderedRequests.ToObservable(); + } + + /// + protected override Task OnFileFoundAsync( + ProcessRequest processRequest, + IDictionary detectorArgs, + CancellationToken cancellationToken = default) + { + var fileExtension = Path.GetExtension(processRequest.ComponentStream.Location); + + if (fileExtension.Equals(".binlog", StringComparison.OrdinalIgnoreCase)) + { + this.ProcessBinlogFile(processRequest); + } + else if (processRequest.ComponentStream.Location.EndsWith("project.assets.json", StringComparison.OrdinalIgnoreCase)) + { + this.ProcessAssetsFile(processRequest); + } + + return Task.CompletedTask; + } + + private void ProcessBinlogFile(ProcessRequest processRequest) + { + var binlogPath = processRequest.ComponentStream.Location; + var assetsFilesFound = new List(); + + try + { + this.Logger.LogDebug("Processing binlog file: {BinlogPath}", binlogPath); + + var projectInfos = this.binLogProcessor.ExtractProjectInfo(binlogPath); + + if (projectInfos.Count == 0) + { + this.Logger.LogInformation("No project information could be extracted from binlog: {BinlogPath}", binlogPath); + return; + } + + foreach (var projectInfo in projectInfos) + { + this.IndexProjectInfo(projectInfo, assetsFilesFound); + this.RegisterDotNetComponent(projectInfo); + this.LogMissingAssetsWarnings(projectInfo); + } + + // Log summary warning if no assets files were found + if (assetsFilesFound.Count == 0 && projectInfos.Count > 0) + { + this.Logger.LogWarning( + "Binlog {BinlogPath} contained {ProjectCount} project(s) but no project.assets.json files were referenced. NuGet restore may not have run.", + binlogPath, + projectInfos.Count); + } + } + catch (Exception ex) + { + this.Logger.LogWarning(ex, "Failed to process binlog file: {BinlogPath}", binlogPath); + } + } + + private void IndexProjectInfo(MSBuildProjectInfo projectInfo, List assetsFilesFound) + { + // Store the project info for later use when processing assets files + var projectPath = projectInfo.ProjectPath; + if (!string.IsNullOrEmpty(projectPath)) + { + this.projectInfoByProjectPath.TryAdd(projectPath, projectInfo); + } + + // Also index by assets file path for direct lookup + if (!string.IsNullOrEmpty(projectInfo.ProjectAssetsFile)) + { + this.projectInfoByAssetsFile.TryAdd(projectInfo.ProjectAssetsFile, projectInfo); + assetsFilesFound.Add(projectInfo.ProjectAssetsFile); + } + } + + private void LogMissingAssetsWarnings(MSBuildProjectInfo projectInfo) + { + if (string.IsNullOrEmpty(projectInfo.ProjectAssetsFile)) + { + this.Logger.LogWarning( + "No ProjectAssetsFile property found in binlog for project: {ProjectPath}. NuGet dependencies may not be detected.", + projectInfo.ProjectPath); + } + else if (!this.fileUtilityService.Exists(projectInfo.ProjectAssetsFile)) + { + this.Logger.LogWarning( + "Project.assets.json referenced in binlog does not exist: {AssetsFile} (from project {ProjectPath})", + projectInfo.ProjectAssetsFile, + projectInfo.ProjectPath); + } + } + + /// + /// Registers a DotNet component based on SDK version from the binlog. + /// + /// + /// This is equivalent to DotNetComponentDetector's behavior but uses the SDK version + /// directly from the binlog (NETCoreSdkVersion property) rather than running `dotnet --version`. + /// The binlog value is more accurate as it represents the actual SDK used during the build. + /// + /// For target type (application/library), we use the OutputType property from the binlog + /// which is equivalent to what DotNetComponentDetector determines by inspecting the PE headers. + /// + private void RegisterDotNetComponent(MSBuildProjectInfo projectInfo) + { + if (string.IsNullOrEmpty(projectInfo.NETCoreSdkVersion) || string.IsNullOrEmpty(projectInfo.ProjectPath)) + { + return; + } + + var singleFileComponentRecorder = this.ComponentRecorder.CreateSingleFileComponentRecorder(projectInfo.ProjectPath); + + // Determine target type from OutputType property + // This is equivalent to DotNetComponentDetector's IsApplication check via PE headers + string? targetType = null; + if (!string.IsNullOrEmpty(projectInfo.OutputType)) + { + targetType = projectInfo.OutputType.Equals("Exe", StringComparison.OrdinalIgnoreCase) || + projectInfo.OutputType.Equals("WinExe", StringComparison.OrdinalIgnoreCase) + ? "application" + : "library"; + } + + // Get target frameworks - equivalent to iterating lockFile.Targets in DotNetComponentDetector + var targetFrameworks = new List(); + if (!string.IsNullOrEmpty(projectInfo.TargetFramework)) + { + targetFrameworks.Add(projectInfo.TargetFramework); + } + else if (!string.IsNullOrEmpty(projectInfo.TargetFrameworks)) + { + targetFrameworks.AddRange(projectInfo.TargetFrameworks.Split(';', StringSplitOptions.RemoveEmptyEntries)); + } + + // Register a DotNet component for each target framework + // This matches DotNetComponentDetector's loop over lockFile.Targets + if (targetFrameworks.Count > 0) + { + foreach (var framework in targetFrameworks) + { + var dotNetComponent = new DotNetComponent(projectInfo.NETCoreSdkVersion, framework, targetType); + singleFileComponentRecorder.RegisterUsage(new DetectedComponent(dotNetComponent)); + } + } + else + { + // No target framework info available, register with just SDK version + var dotNetComponent = new DotNetComponent(projectInfo.NETCoreSdkVersion, targetFramework: null, targetType); + singleFileComponentRecorder.RegisterUsage(new DetectedComponent(dotNetComponent)); + } + } + + private void ProcessAssetsFile(ProcessRequest processRequest) + { + var assetsFilePath = processRequest.ComponentStream.Location; + + // Check if this assets file was already processed + if (this.processedAssetsFiles.ContainsKey(assetsFilePath)) + { + this.Logger.LogDebug("Assets file already processed: {AssetsFile}", assetsFilePath); + return; + } + + try + { + var lockFile = this.lockFileFormat.Read(processRequest.ComponentStream.Stream, assetsFilePath); + + this.RecordLockfileVersion(lockFile.Version); + + if (lockFile.PackageSpec == null) + { + this.Logger.LogDebug("Lock file {LockFilePath} does not contain a PackageSpec.", assetsFilePath); + return; + } + + // Try to find matching binlog info + var projectInfo = this.FindProjectInfoForAssetsFile(assetsFilePath, lockFile); + + // Mark as processed + this.processedAssetsFiles.TryAdd(assetsFilePath, true); + + if (projectInfo != null) + { + // We have binlog info, use enhanced processing + this.ProcessLockFileWithProjectInfo(lockFile, projectInfo); + } + else + { + // Fallback to standard processing without binlog info + // This matches NuGetProjectModelProjectCentricComponentDetector's behavior exactly + this.Logger.LogDebug( + "No binlog information found for assets file: {AssetsFile}. Using fallback processing.", + assetsFilePath); + this.ProcessLockFileFallback(lockFile, assetsFilePath); + } + } + catch (Exception ex) + { + this.Logger.LogWarning(ex, "Failed to process NuGet lockfile: {LockFile}", assetsFilePath); + } + } + + private MSBuildProjectInfo? FindProjectInfoForAssetsFile(string assetsFilePath, LockFile lockFile) + { + // Try to find by assets file path first + if (this.projectInfoByAssetsFile.TryGetValue(assetsFilePath, out var infoByAssets)) + { + return infoByAssets; + } + + // Try to find by project path from the lock file + var projectPath = lockFile.PackageSpec?.RestoreMetadata?.ProjectPath; + if (!string.IsNullOrEmpty(projectPath) && + this.projectInfoByProjectPath.TryGetValue(projectPath, out var infoByProject)) + { + return infoByProject; + } + + return null; + } + + /// + /// Processes a lock file with enhanced project info from the binlog. + /// + /// + /// This method uses the same core logic as NuGetProjectModelProjectCentricComponentDetector: + /// - Gets top-level libraries via GetTopLevelLibraries + /// - Determines framework packages and dev dependencies + /// - Navigates dependency graph via NavigateAndRegister + /// - Registers PackageDownload dependencies + /// + /// The enhancement is that we can mark all dependencies of test projects as dev dependencies + /// based on the IsTestProject property from the binlog. + /// + private void ProcessLockFileWithProjectInfo(LockFile lockFile, MSBuildProjectInfo projectInfo) + { + var explicitReferencedDependencies = LockFileUtilities.GetTopLevelLibraries(lockFile) + .Select(x => LockFileUtilities.GetLibraryComponentWithDependencyLookup(lockFile.Libraries, x.Name, x.Version, x.VersionRange, this.Logger)) + .Where(x => x != null) + .ToList(); + + var explicitlyReferencedComponentIds = explicitReferencedDependencies + .Select(x => new NuGetComponent(x!.Name, x.Version.ToNormalizedString()).Id) + .ToHashSet(); + + // Use project path from RestoreMetadata (consistent with NuGetProjectModelProjectCentricComponentDetector) + var singleFileComponentRecorder = this.ComponentRecorder.CreateSingleFileComponentRecorder( + lockFile.PackageSpec?.RestoreMetadata?.ProjectPath ?? projectInfo.ProjectPath ?? string.Empty); + + foreach (var target in lockFile.Targets) + { + var frameworkReferences = LockFileUtilities.GetFrameworkReferences(lockFile, target); + var frameworkPackages = FrameworkPackages.GetFrameworkPackages(target.TargetFramework, frameworkReferences, target); + + // Same logic as NuGetProjectModelProjectCentricComponentDetector.IsFrameworkOrDevelopmentDependency + bool IsFrameworkOrDevDependency(LockFileTargetLibrary library) => + frameworkPackages.Any(fp => fp.IsAFrameworkComponent(library.Name, library.Version)) || + LockFileUtilities.IsADevelopmentDependency(library, lockFile); + + // Enhancement: Apply test project classification - all dependencies of test projects are dev dependencies + bool IsDevelopmentDependencyWithClassification(LockFileTargetLibrary library) => + projectInfo.IsTestProject == true || IsFrameworkOrDevDependency(library); + + foreach (var library in explicitReferencedDependencies.Select(x => target.GetTargetLibrary(x!.Name)).Where(x => x != null)) + { + LockFileUtilities.NavigateAndRegister( + target, + explicitlyReferencedComponentIds, + singleFileComponentRecorder, + library!, + null, + IsDevelopmentDependencyWithClassification); + } + } + + // Register PackageDownload dependencies (same as NuGetProjectModelProjectCentricComponentDetector) + LockFileUtilities.RegisterPackageDownloads(singleFileComponentRecorder, lockFile); + } + + /// + /// Processes a lock file without binlog info (fallback mode). + /// + /// + /// This method exactly matches NuGetProjectModelProjectCentricComponentDetector's behavior + /// to ensure no loss of information when binlog data is not available. + /// + private void ProcessLockFileFallback(LockFile lockFile, string location) + { + var explicitReferencedDependencies = LockFileUtilities.GetTopLevelLibraries(lockFile) + .Select(x => LockFileUtilities.GetLibraryComponentWithDependencyLookup(lockFile.Libraries, x.Name, x.Version, x.VersionRange, this.Logger)) + .Where(x => x != null) + .ToList(); + + var explicitlyReferencedComponentIds = explicitReferencedDependencies + .Select(x => new NuGetComponent(x!.Name, x.Version.ToNormalizedString()).Id) + .ToHashSet(); + + // Use project path from RestoreMetadata (consistent with NuGetProjectModelProjectCentricComponentDetector) + var projectPath = lockFile.PackageSpec?.RestoreMetadata?.ProjectPath ?? location; + var singleFileComponentRecorder = this.ComponentRecorder.CreateSingleFileComponentRecorder(projectPath); + + foreach (var target in lockFile.Targets) + { + var frameworkReferences = LockFileUtilities.GetFrameworkReferences(lockFile, target); + var frameworkPackages = FrameworkPackages.GetFrameworkPackages(target.TargetFramework, frameworkReferences, target); + + // Same logic as NuGetProjectModelProjectCentricComponentDetector.IsFrameworkOrDevelopmentDependency + bool IsFrameworkOrDevDependency(LockFileTargetLibrary library) => + frameworkPackages.Any(fp => fp.IsAFrameworkComponent(library.Name, library.Version)) || + LockFileUtilities.IsADevelopmentDependency(library, lockFile); + + foreach (var library in explicitReferencedDependencies.Select(x => target.GetTargetLibrary(x!.Name)).Where(x => x != null)) + { + LockFileUtilities.NavigateAndRegister( + target, + explicitlyReferencedComponentIds, + singleFileComponentRecorder, + library!, + null, + IsFrameworkOrDevDependency); + } + } + + // Register PackageDownload dependencies (same as NuGetProjectModelProjectCentricComponentDetector) + LockFileUtilities.RegisterPackageDownloads(singleFileComponentRecorder, lockFile); + } +} diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildProjectInfo.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildProjectInfo.cs new file mode 100644 index 000000000..97ab6cc93 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildProjectInfo.cs @@ -0,0 +1,193 @@ +namespace Microsoft.ComponentDetection.Detectors.NuGet; + +using System; +using System.Collections.Generic; +using Microsoft.Build.Framework; + +/// +/// Represents project information extracted from an MSBuild binlog file. +/// Contains properties of interest for component classification. +/// +internal class MSBuildProjectInfo +{ + /// + /// Maps MSBuild property names to their setter actions. + /// + private static readonly Dictionary> PropertySetters = new(StringComparer.OrdinalIgnoreCase) + { + [nameof(IsDevelopment)] = (info, value) => info.IsDevelopment = string.Equals(value, "true", StringComparison.OrdinalIgnoreCase), + [nameof(IsPackable)] = (info, value) => info.IsPackable = string.Equals(value, "true", StringComparison.OrdinalIgnoreCase), + [nameof(IsShipping)] = (info, value) => info.IsShipping = string.Equals(value, "true", StringComparison.OrdinalIgnoreCase), + [nameof(IsTestProject)] = (info, value) => info.IsTestProject = string.Equals(value, "true", StringComparison.OrdinalIgnoreCase), + [nameof(NETCoreSdkVersion)] = (info, value) => info.NETCoreSdkVersion = value, + [nameof(OutputType)] = (info, value) => info.OutputType = value, + [nameof(ProjectAssetsFile)] = (info, value) => + { + if (!string.IsNullOrEmpty(value)) + { + info.ProjectAssetsFile = value; + } + }, + [nameof(SelfContained)] = (info, value) => info.SelfContained = string.Equals(value, "true", StringComparison.OrdinalIgnoreCase), + [nameof(TargetFramework)] = (info, value) => info.TargetFramework = value, + [nameof(TargetFrameworks)] = (info, value) => info.TargetFrameworks = value, + }; + + /// + /// Maps MSBuild item type names to their dictionary accessor. + /// + private static readonly Dictionary>> ItemDictionaries = new(StringComparer.OrdinalIgnoreCase) + { + [nameof(PackageReference)] = info => info.PackageReference, + [nameof(PackageDownload)] = info => info.PackageDownload, + }; + + /// + /// Gets or sets the full path to the project file. + /// + public string? ProjectPath { get; set; } + + /// + /// Gets or sets a value indicating whether this is a development-only project. + /// Corresponds to the MSBuild IsDevelopment property. + /// + public bool? IsDevelopment { get; set; } + + /// + /// Gets or sets a value indicating whether the project is packable. + /// Corresponds to the MSBuild IsPackable property. + /// + public bool? IsPackable { get; set; } + + /// + /// Gets or sets a value indicating whether this project produces shipping artifacts. + /// Corresponds to the MSBuild IsShipping property. + /// + public bool? IsShipping { get; set; } + + /// + /// Gets or sets a value indicating whether this is a test project. + /// Corresponds to the MSBuild IsTestProject property. + /// When true, all dependencies of this project should be classified as development dependencies. + /// + public bool? IsTestProject { get; set; } + + /// + /// Gets or sets the output type of the project (e.g., "Exe", "Library", "WinExe"). + /// Corresponds to the MSBuild OutputType property. + /// + public string? OutputType { get; set; } + + /// + /// Gets or sets the .NET Core SDK version used to build the project. + /// Corresponds to the MSBuild NETCoreSdkVersion property. + /// + public string? NETCoreSdkVersion { get; set; } + + /// + /// Gets or sets the path to the project.assets.json file. + /// Corresponds to the MSBuild ProjectAssetsFile property. + /// + public string? ProjectAssetsFile { get; set; } + + /// + /// Gets or sets a value indicating whether the project is self-contained. + /// Corresponds to the MSBuild SelfContained property. + /// + public bool? SelfContained { get; set; } + + /// + /// Gets or sets the target framework for single-targeted projects. + /// Corresponds to the MSBuild TargetFramework property. + /// + public string? TargetFramework { get; set; } + + /// + /// Gets or sets the target frameworks for multi-targeted projects. + /// Corresponds to the MSBuild TargetFrameworks property. + /// + public string? TargetFrameworks { get; set; } + + /// + /// Gets the PackageReference items captured from the project. + /// Keyed by package name (ItemSpec). + /// + public IDictionary PackageReference { get; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + + /// + /// Gets the PackageDownload items captured from the project. + /// Keyed by package name (ItemSpec). + /// + public IDictionary PackageDownload { get; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + + /// + /// Gets the inner builds for multi-targeted projects. + /// For multi-targeted projects, the outer build has TargetFrameworks set and dispatches to inner builds. + /// Each inner build has a specific TargetFramework and its own set of properties and items. + /// + public IList InnerBuilds { get; } = []; + + /// + /// Gets a value indicating whether this is an outer build of a multi-targeted project. + /// + public bool IsOuterBuild => !string.IsNullOrEmpty(this.TargetFrameworks); + + /// + /// Determines whether the specified item type is one that this class captures. + /// + /// The MSBuild item type. + /// True if the item type is of interest; otherwise, false. + public static bool IsItemTypeOfInterest(string itemType) => ItemDictionaries.ContainsKey(itemType); + + /// + /// Sets a property value if it is one of the properties of interest. + /// + /// The MSBuild property name. + /// The property value. + /// True if the property was set; otherwise, false. + public bool TrySetProperty(string propertyName, string value) + { + if (PropertySetters.TryGetValue(propertyName, out var setter)) + { + setter(this, value); + return true; + } + + return false; + } + + /// + /// Adds or updates an item if it is one of the item types of interest. + /// + /// The item type (e.g., "PackageReference"). + /// The item to add or update. + /// True if the item was added or updated; otherwise, false. + public bool TryAddOrUpdateItem(string itemType, ITaskItem item) + { + if (item == null || !ItemDictionaries.TryGetValue(itemType, out var getDictionary)) + { + return false; + } + + var dictionary = getDictionary(this); + dictionary[item.ItemSpec] = item; + return true; + } + + /// + /// Removes an item if it exists. + /// + /// The item type (e.g., "PackageReference"). + /// The item spec (e.g., package name). + /// True if the item was removed; otherwise, false. + public bool TryRemoveItem(string itemType, string itemSpec) + { + if (!ItemDictionaries.TryGetValue(itemType, out var getDictionary)) + { + return false; + } + + var dictionary = getDictionary(this); + return dictionary.Remove(itemSpec); + } +} diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/MSBuildBinaryLogExperiment.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/MSBuildBinaryLogExperiment.cs new file mode 100644 index 000000000..6b11f5611 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/MSBuildBinaryLogExperiment.cs @@ -0,0 +1,25 @@ +namespace Microsoft.ComponentDetection.Orchestrator.Experiments.Configs; + +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Detectors.NuGet; + +/// +/// Experiment configuration for validating the +/// against the existing . +/// +public class MSBuildBinaryLogExperiment : IExperimentConfiguration +{ + /// + public string Name => "MSBuildBinaryLogDetector"; + + /// + public bool IsInControlGroup(IComponentDetector componentDetector) => + componentDetector is NuGetProjectModelProjectCentricComponentDetector; + + /// + public bool IsInExperimentGroup(IComponentDetector componentDetector) => + componentDetector is MSBuildBinaryLogComponentDetector; + + /// + public bool ShouldRecord(IComponentDetector componentDetector, int numComponents) => true; +} diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs index ab86692c6..837964bfc 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs @@ -73,6 +73,7 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); // Detectors // CocoaPods @@ -131,6 +132,7 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); // PIP services.AddSingleton(); diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs new file mode 100644 index 000000000..f1626c1e0 --- /dev/null +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs @@ -0,0 +1,372 @@ +namespace Microsoft.ComponentDetection.Detectors.Tests.NuGet; + +using System.Linq; +using System.Threading.Tasks; +using AwesomeAssertions; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.ComponentDetection.Detectors.NuGet; +using Microsoft.ComponentDetection.Detectors.Tests.Utilities; +using Microsoft.ComponentDetection.TestsUtilities; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +[TestClass] +public class MSBuildBinaryLogComponentDetectorTests : BaseDetectorTest +{ + private readonly Mock fileUtilityServiceMock; + + public MSBuildBinaryLogComponentDetectorTests() + { + this.fileUtilityServiceMock = new Mock(); + this.fileUtilityServiceMock.Setup(x => x.Exists(It.IsAny())).Returns(true); + this.DetectorTestUtility.AddServiceMock(this.fileUtilityServiceMock); + } + + [TestMethod] + public async Task ScanDirectoryAsync_WithSimpleAssetsFile_DetectsComponents() + { + var projectAssetsJson = @"{ + ""version"": 3, + ""targets"": { + ""net8.0"": { + ""Newtonsoft.Json/13.0.1"": { + ""type"": ""package"", + ""compile"": { + ""lib/net8.0/Newtonsoft.Json.dll"": {} + }, + ""runtime"": { + ""lib/net8.0/Newtonsoft.Json.dll"": {} + } + } + } + }, + ""libraries"": { + ""Newtonsoft.Json/13.0.1"": { + ""sha512"": ""test"", + ""type"": ""package"", + ""path"": ""newtonsoft.json/13.0.1"", + ""files"": [ + ""lib/net8.0/Newtonsoft.Json.dll"" + ] + } + }, + ""projectFileDependencyGroups"": { + ""net8.0"": [ + ""Newtonsoft.Json >= 13.0.1"" + ] + }, + ""packageFolders"": { + ""C:\\Users\\test\\.nuget\\packages\\"": {} + }, + ""project"": { + ""version"": ""1.0.0"", + ""restore"": { + ""projectName"": ""TestProject"", + ""projectPath"": ""C:\\test\\TestProject.csproj"", + ""outputPath"": ""C:\\test\\obj"" + }, + ""frameworks"": { + ""net8.0"": { + ""targetAlias"": ""net8.0"", + ""dependencies"": { + ""Newtonsoft.Json"": { + ""target"": ""Package"", + ""version"": ""[13.0.1, )"" + } + } + } + } + } + }"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("project.assets.json", projectAssetsJson) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().HaveCount(1); + + var component = detectedComponents.First(); + var nugetComponent = component.Component as NuGetComponent; + nugetComponent.Should().NotBeNull(); + nugetComponent!.Name.Should().Be("Newtonsoft.Json"); + nugetComponent.Version.Should().Be("13.0.1"); + } + + [TestMethod] + public async Task ScanDirectoryAsync_WithTransitiveDependencies_BuildsDependencyGraph() + { + var projectAssetsJson = @"{ + ""version"": 3, + ""targets"": { + ""net8.0"": { + ""Microsoft.Extensions.Logging/8.0.0"": { + ""type"": ""package"", + ""dependencies"": { + ""Microsoft.Extensions.Logging.Abstractions"": ""8.0.0"" + }, + ""compile"": { + ""lib/net8.0/Microsoft.Extensions.Logging.dll"": {} + }, + ""runtime"": { + ""lib/net8.0/Microsoft.Extensions.Logging.dll"": {} + } + }, + ""Microsoft.Extensions.Logging.Abstractions/8.0.0"": { + ""type"": ""package"", + ""compile"": { + ""lib/net8.0/Microsoft.Extensions.Logging.Abstractions.dll"": {} + }, + ""runtime"": { + ""lib/net8.0/Microsoft.Extensions.Logging.Abstractions.dll"": {} + } + } + } + }, + ""libraries"": { + ""Microsoft.Extensions.Logging/8.0.0"": { + ""sha512"": ""test"", + ""type"": ""package"", + ""path"": ""microsoft.extensions.logging/8.0.0"", + ""files"": [ + ""lib/net8.0/Microsoft.Extensions.Logging.dll"" + ] + }, + ""Microsoft.Extensions.Logging.Abstractions/8.0.0"": { + ""sha512"": ""test"", + ""type"": ""package"", + ""path"": ""microsoft.extensions.logging.abstractions/8.0.0"", + ""files"": [ + ""lib/net8.0/Microsoft.Extensions.Logging.Abstractions.dll"" + ] + } + }, + ""projectFileDependencyGroups"": { + ""net8.0"": [ + ""Microsoft.Extensions.Logging >= 8.0.0"" + ] + }, + ""packageFolders"": { + ""C:\\Users\\test\\.nuget\\packages\\"": {} + }, + ""project"": { + ""version"": ""1.0.0"", + ""restore"": { + ""projectName"": ""TestProject"", + ""projectPath"": ""C:\\test\\TestProject.csproj"", + ""outputPath"": ""C:\\test\\obj"" + }, + ""frameworks"": { + ""net8.0"": { + ""targetAlias"": ""net8.0"", + ""dependencies"": { + ""Microsoft.Extensions.Logging"": { + ""target"": ""Package"", + ""version"": ""[8.0.0, )"" + } + } + } + } + } + }"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("project.assets.json", projectAssetsJson) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().HaveCount(2); + + var graphsByLocation = componentRecorder.GetDependencyGraphsByLocation(); + graphsByLocation.Should().NotBeEmpty(); + + var graph = graphsByLocation.Values.First(); + var loggingComponent = detectedComponents.First(x => ((NuGetComponent)x.Component).Name == "Microsoft.Extensions.Logging"); + var abstractionsComponent = detectedComponents.First(x => ((NuGetComponent)x.Component).Name == "Microsoft.Extensions.Logging.Abstractions"); + + graph.IsComponentExplicitlyReferenced(loggingComponent.Component.Id).Should().BeTrue(); + graph.IsComponentExplicitlyReferenced(abstractionsComponent.Component.Id).Should().BeFalse(); + + var dependencies = graph.GetDependenciesForComponent(loggingComponent.Component.Id); + dependencies.Should().Contain(abstractionsComponent.Component.Id); + } + + [TestMethod] + public async Task ScanDirectoryAsync_WithNoPackageSpec_HandlesGracefully() + { + var projectAssetsJson = @"{ + ""version"": 3, + ""targets"": { + ""net8.0"": {} + }, + ""libraries"": {}, + ""packageFolders"": {} + }"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("project.assets.json", projectAssetsJson) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().BeEmpty(); + } + + [TestMethod] + public async Task ScanDirectoryAsync_WithProjectReference_ExcludesProjectDependencies() + { + var projectAssetsJson = @"{ + ""version"": 3, + ""targets"": { + ""net8.0"": { + ""Newtonsoft.Json/13.0.1"": { + ""type"": ""package"", + ""compile"": { + ""lib/net8.0/Newtonsoft.Json.dll"": {} + }, + ""runtime"": { + ""lib/net8.0/Newtonsoft.Json.dll"": {} + } + }, + ""MyOtherProject/1.0.0"": { + ""type"": ""project"" + } + } + }, + ""libraries"": { + ""Newtonsoft.Json/13.0.1"": { + ""sha512"": ""test"", + ""type"": ""package"", + ""path"": ""newtonsoft.json/13.0.1"", + ""files"": [ + ""lib/net8.0/Newtonsoft.Json.dll"" + ] + }, + ""MyOtherProject/1.0.0"": { + ""type"": ""project"", + ""path"": ""../MyOtherProject/MyOtherProject.csproj"", + ""msbuildProject"": ""../MyOtherProject/MyOtherProject.csproj"" + } + }, + ""projectFileDependencyGroups"": { + ""net8.0"": [ + ""Newtonsoft.Json >= 13.0.1"" + ] + }, + ""packageFolders"": { + ""C:\\Users\\test\\.nuget\\packages\\"": {} + }, + ""project"": { + ""version"": ""1.0.0"", + ""restore"": { + ""projectName"": ""TestProject"", + ""projectPath"": ""C:\\test\\TestProject.csproj"", + ""outputPath"": ""C:\\test\\obj"" + }, + ""frameworks"": { + ""net8.0"": { + ""targetAlias"": ""net8.0"", + ""dependencies"": { + ""Newtonsoft.Json"": { + ""target"": ""Package"", + ""version"": ""[13.0.1, )"" + } + } + } + } + } + }"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("project.assets.json", projectAssetsJson) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var detectedComponents = componentRecorder.GetDetectedComponents(); + + // Should only detect the NuGet package, not the project reference + detectedComponents.Should().HaveCount(1); + var component = detectedComponents.First().Component as NuGetComponent; + component!.Name.Should().Be("Newtonsoft.Json"); + } + + [TestMethod] + public async Task ScanDirectoryAsync_WithDevelopmentDependency_MarksAsDev() + { + // A package with only compile/build assets and no runtime assemblies should be marked as a dev dependency + var projectAssetsJson = @"{ + ""version"": 3, + ""targets"": { + ""net8.0"": { + ""StyleCop.Analyzers/1.2.0-beta.556"": { + ""type"": ""package"", + ""compile"": { + ""lib/netstandard2.0/_._"": {} + }, + ""runtime"": { + ""lib/netstandard2.0/_._"": {} + }, + ""build"": { + ""build/StyleCop.Analyzers.props"": {} + } + } + } + }, + ""libraries"": { + ""StyleCop.Analyzers/1.2.0-beta.556"": { + ""sha512"": ""test"", + ""type"": ""package"", + ""path"": ""stylecop.analyzers/1.2.0-beta.556"", + ""files"": [ + ""analyzers/dotnet/cs/StyleCop.Analyzers.dll"", + ""lib/netstandard2.0/_._"" + ] + } + }, + ""projectFileDependencyGroups"": { + ""net8.0"": [ + ""StyleCop.Analyzers >= 1.2.0-beta.556"" + ] + }, + ""packageFolders"": { + ""C:\\Users\\test\\.nuget\\packages\\"": {} + }, + ""project"": { + ""version"": ""1.0.0"", + ""restore"": { + ""projectName"": ""TestProject"", + ""projectPath"": ""C:\\test\\TestProject.csproj"", + ""outputPath"": ""C:\\test\\obj"" + }, + ""frameworks"": { + ""net8.0"": { + ""targetAlias"": ""net8.0"", + ""dependencies"": { + ""StyleCop.Analyzers"": { + ""target"": ""Package"", + ""version"": ""[1.2.0-beta.556, )"" + } + } + } + } + } + }"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("project.assets.json", projectAssetsJson) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().HaveCount(1); + + var component = detectedComponents.First(); + + // Analyzers are detected as development dependencies because they have analyzers in files + // but their runtime assets are placeholders + componentRecorder.GetEffectiveDevDependencyValue(component.Component.Id).Should().NotBeNull(); + } +} From cfd3e1c5ea2536d3e1c8605f371bbe0507987ce7 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Thu, 5 Mar 2026 20:03:31 -0800 Subject: [PATCH 02/26] Implement dev dependency overrides --- .../nuget/LockFileUtilities.cs | 22 ++- .../MSBuildBinaryLogComponentDetector.cs | 125 ++++++++++++++++-- 2 files changed, 130 insertions(+), 17 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/LockFileUtilities.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/LockFileUtilities.cs index 9cb98d135..433579277 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/LockFileUtilities.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/LockFileUtilities.cs @@ -4,6 +4,7 @@ namespace Microsoft.ComponentDetection.Detectors.NuGet; using System.Collections.Generic; using System.IO; using System.Linq; +using global::NuGet.Frameworks; using global::NuGet.Packaging.Core; using global::NuGet.ProjectModel; using global::NuGet.Versioning; @@ -246,19 +247,31 @@ public static void NavigateAndRegister( /// /// Registers PackageDownload dependencies from the lock file. - /// PackageDownload is always a development dependency since its usage does not make it part of the application. /// /// The component recorder to register with. /// The lock file containing PackageDownload references. - public static void RegisterPackageDownloads(ISingleFileComponentRecorder singleFileComponentRecorder, LockFile lockFile) + /// + /// Optional callback to determine if a package download is a development dependency. + /// Parameters are (packageName, targetFramework). Defaults to always returning true since + /// PackageDownload usage does not make it part of the application. + /// + public static void RegisterPackageDownloads( + ISingleFileComponentRecorder singleFileComponentRecorder, + LockFile lockFile, + Func? isDevelopmentDependency = null) { if (lockFile.PackageSpec?.TargetFrameworks == null) { return; } + // Default: PackageDownload is always a development dependency + isDevelopmentDependency ??= (_, _) => true; + foreach (var framework in lockFile.PackageSpec.TargetFrameworks) { + var tfm = framework.FrameworkName; + foreach (var packageDownload in framework.DownloadDependencies) { if (packageDownload?.Name is null || packageDownload?.VersionRange?.MinVersion is null) @@ -268,13 +281,12 @@ public static void RegisterPackageDownloads(ISingleFileComponentRecorder singleF var libraryComponent = new DetectedComponent(new NuGetComponent(packageDownload.Name, packageDownload.VersionRange.MinVersion.ToNormalizedString())); - // PackageDownload is always a development dependency since its usage does not make it part of the application singleFileComponentRecorder.RegisterUsage( libraryComponent, isExplicitReferencedDependency: true, parentComponentId: null, - isDevelopmentDependency: true, - targetFramework: framework.FrameworkName?.GetShortFolderName()); + isDevelopmentDependency: isDevelopmentDependency(packageDownload.Name, tfm), + targetFramework: tfm?.GetShortFolderName()); } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs index 7c3b5db53..4e890ec31 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs @@ -9,7 +9,9 @@ namespace Microsoft.ComponentDetection.Detectors.NuGet; using System.Reactive.Threading.Tasks; using System.Threading; using System.Threading.Tasks; +using global::NuGet.Frameworks; using global::NuGet.ProjectModel; +using Microsoft.Build.Framework; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; @@ -146,6 +148,39 @@ protected override Task OnFileFoundAsync( return Task.CompletedTask; } + /// + /// Determines whether a project should be classified as development-only. + /// A project is development-only if IsTestProject=true, IsShipping=false, or IsDevelopment=true. + /// + private static bool IsDevelopmentOnlyProject(MSBuildProjectInfo projectInfo) => + projectInfo.IsTestProject == true || + projectInfo.IsShipping == false || + projectInfo.IsDevelopment == true; + + /// + /// Gets the IsDevelopmentDependency metadata override for a package from the specified items. + /// + /// The item dictionary to check (e.g., PackageReference or PackageDownload). + /// The package name to look up. + /// + /// True if explicitly marked as a development dependency, + /// false if explicitly marked as NOT a development dependency, + /// null if no explicit override is set or the package is not in the items. + /// + private static bool? GetDevelopmentDependencyOverride(IDictionary items, string packageName) + { + if (items.TryGetValue(packageName, out var item)) + { + var metadataValue = item.GetMetadata("IsDevelopmentDependency"); + if (!string.IsNullOrEmpty(metadataValue)) + { + return string.Equals(metadataValue, "true", StringComparison.OrdinalIgnoreCase); + } + } + + return null; + } + private void ProcessBinlogFile(ProcessRequest processRequest) { var binlogPath = processRequest.ComponentStream.Location; @@ -358,8 +393,10 @@ private void ProcessAssetsFile(ProcessRequest processRequest) /// - Navigates dependency graph via NavigateAndRegister /// - Registers PackageDownload dependencies /// - /// The enhancement is that we can mark all dependencies of test projects as dev dependencies - /// based on the IsTestProject property from the binlog. + /// Enhancements from binlog: + /// - If project sets IsTestProject=true, IsShipping=false, or IsDevelopment=true, + /// all dependencies are marked as development dependencies. + /// - Per-package IsDevelopmentDependency metadata overrides are applied transitively. /// private void ProcessLockFileWithProjectInfo(LockFile lockFile, MSBuildProjectInfo projectInfo) { @@ -376,34 +413,98 @@ private void ProcessLockFileWithProjectInfo(LockFile lockFile, MSBuildProjectInf var singleFileComponentRecorder = this.ComponentRecorder.CreateSingleFileComponentRecorder( lockFile.PackageSpec?.RestoreMetadata?.ProjectPath ?? projectInfo.ProjectPath ?? string.Empty); + // Get the project info for the target framework (use inner build if available) + MSBuildProjectInfo GetProjectInfoForTarget(LockFileTarget target) + { + if (target.TargetFramework != null) + { + var innerBuild = projectInfo.InnerBuilds.FirstOrDefault( + ib => !string.IsNullOrEmpty(ib.TargetFramework) && + NuGetFramework.Parse(ib.TargetFramework).Equals(target.TargetFramework)); + if (innerBuild != null) + { + return innerBuild; + } + } + + return projectInfo; + } + foreach (var target in lockFile.Targets) { + var targetProjectInfo = GetProjectInfoForTarget(target); var frameworkReferences = LockFileUtilities.GetFrameworkReferences(lockFile, target); var frameworkPackages = FrameworkPackages.GetFrameworkPackages(target.TargetFramework, frameworkReferences, target); - // Same logic as NuGetProjectModelProjectCentricComponentDetector.IsFrameworkOrDevelopmentDependency + // Base logic: check if library is a framework component or dev dependency in lock file bool IsFrameworkOrDevDependency(LockFileTargetLibrary library) => frameworkPackages.Any(fp => fp.IsAFrameworkComponent(library.Name, library.Version)) || LockFileUtilities.IsADevelopmentDependency(library, lockFile); - // Enhancement: Apply test project classification - all dependencies of test projects are dev dependencies - bool IsDevelopmentDependencyWithClassification(LockFileTargetLibrary library) => - projectInfo.IsTestProject == true || IsFrameworkOrDevDependency(library); - - foreach (var library in explicitReferencedDependencies.Select(x => target.GetTargetLibrary(x!.Name)).Where(x => x != null)) + foreach (var dependency in explicitReferencedDependencies) { + var library = target.GetTargetLibrary(dependency!.Name); + if (library?.Name == null) + { + continue; + } + + // Combine project-level and per-package overrides into a single value. + // When set, this applies transitively to all dependencies of this package. + var devDependencyOverride = IsDevelopmentOnlyProject(targetProjectInfo) + ? true + : GetDevelopmentDependencyOverride(targetProjectInfo.PackageReference, library.Name); + LockFileUtilities.NavigateAndRegister( target, explicitlyReferencedComponentIds, singleFileComponentRecorder, - library!, + library, null, - IsDevelopmentDependencyWithClassification); + devDependencyOverride.HasValue ? _ => devDependencyOverride.Value : IsFrameworkOrDevDependency); } } - // Register PackageDownload dependencies (same as NuGetProjectModelProjectCentricComponentDetector) - LockFileUtilities.RegisterPackageDownloads(singleFileComponentRecorder, lockFile); + // Register PackageDownload dependencies with dev-dependency overrides + LockFileUtilities.RegisterPackageDownloads( + singleFileComponentRecorder, + lockFile, + (packageName, framework) => this.IsPackageDownloadDevDependency(packageName, framework, projectInfo)); + } + + /// + /// Determines if a PackageDownload is a development dependency based on project info. + /// + private bool IsPackageDownloadDevDependency(string packageName, NuGetFramework? framework, MSBuildProjectInfo projectInfo) + { + // Get the project info for this framework (use inner build if available) + var targetProjectInfo = projectInfo; + if (framework != null) + { + var innerBuild = projectInfo.InnerBuilds.FirstOrDefault( + ib => !string.IsNullOrEmpty(ib.TargetFramework) && + NuGetFramework.Parse(ib.TargetFramework).Equals(framework)); + if (innerBuild != null) + { + targetProjectInfo = innerBuild; + } + } + + // Project-level override: all deps are dev deps + if (IsDevelopmentOnlyProject(targetProjectInfo)) + { + return true; + } + + // Check for explicit item metadata override + var devOverride = GetDevelopmentDependencyOverride(targetProjectInfo.PackageDownload, packageName); + if (devOverride.HasValue) + { + return devOverride.Value; + } + + // Default: PackageDownload is a dev dependency + return true; } /// From 630cd3ea28034ebfd2e99b27d62e77a985c13cf7 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Tue, 10 Mar 2026 14:27:04 -0700 Subject: [PATCH 03/26] Add more tests, add selfcontained detection --- .../nuget/BinLogProcessor.cs | 61 +- .../MSBuildBinaryLogComponentDetector.cs | 141 +- .../nuget/MSBuildProjectInfo.cs | 59 +- ....ComponentDetection.Detectors.Tests.csproj | 1 + .../nuget/BinLogProcessorTests.cs | 1642 +++++++++++++++++ .../MSBuildBinaryLogComponentDetectorTests.cs | 1011 +++++++--- 6 files changed, 2600 insertions(+), 315 deletions(-) create mode 100644 test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs index 48a18e314..5b649f690 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs @@ -3,6 +3,7 @@ namespace Microsoft.ComponentDetection.Detectors.NuGet; using System; using System.Collections; using System.Collections.Generic; +using System.Linq; using Microsoft.Build.Framework; using Microsoft.Build.Logging.StructuredLogger; using Microsoft.Extensions.Logging; @@ -133,12 +134,45 @@ private void AddOrMergeProjectInfo( } else if (!existing.IsOuterBuild && !projectInfo.IsOuterBuild && !string.IsNullOrEmpty(projectInfo.TargetFramework)) { - // Both are inner builds (no outer build seen yet) - add to InnerBuilds of the first one - // The first one acts as a placeholder until we see an outer build - existing.InnerBuilds.Add(projectInfo); + // Both are single-TFM builds - check if they share the same TFM + if (string.Equals(existing.TargetFramework, projectInfo.TargetFramework, StringComparison.OrdinalIgnoreCase)) + { + // Same project, same TFM (e.g. build + publish) - merge as superset + existing.MergeWith(projectInfo); + } + else + { + // Different TFMs (no outer build seen yet) - add to InnerBuilds of the first one + // The first one acts as a placeholder until we see an outer build + existing.InnerBuilds.Add(projectInfo); + } } + else if (existing.IsOuterBuild && projectInfo.IsOuterBuild) + { + // Both are outer builds (e.g. build + publish of a multi-targeted project) + // Merge inner builds: for matching TFMs, merge; for new TFMs, add + foreach (var newInner in projectInfo.InnerBuilds) + { + var matchingInner = existing.InnerBuilds.FirstOrDefault( + ib => string.Equals(ib.TargetFramework, newInner.TargetFramework, StringComparison.OrdinalIgnoreCase)); + if (matchingInner != null) + { + matchingInner.MergeWith(newInner); + } + else + { + existing.InnerBuilds.Add(newInner); + } + } - // Otherwise: duplicate builds currently ignored. + // Merge the outer build properties/items too + existing.MergeWith(projectInfo); + } + else + { + // Fallback: merge properties/items as superset + existing.MergeWith(projectInfo); + } } /// @@ -188,22 +222,17 @@ private void PopulateFromEvaluation(ProjectEvaluationFinishedEventArgs projectEv // Extract items if (projectEvalArgs?.Items != null) { - // Items is an IEnumerable that contains item groups - // Each item group has an ItemType (Key) and Items collection (Value) - foreach (var itemGroup in projectEvalArgs.Items) + // Items is a flat IList where each entry has: + // Key = item type (string, e.g., "PackageReference") + // Value = single ITaskItem (TaskItemData from binlog deserialization) + foreach (var itemEntry in projectEvalArgs.Items) { - if (itemGroup is DictionaryEntry entry && + if (itemEntry is DictionaryEntry entry && entry.Key is string itemType && MSBuildProjectInfo.IsItemTypeOfInterest(itemType) && - entry.Value is IEnumerable groupItems) + entry.Value is ITaskItem taskItem) { - foreach (var item in groupItems) - { - if (item is ITaskItem taskItem) - { - projectInfo.TryAddOrUpdateItem(itemType, taskItem); - } - } + projectInfo.TryAddOrUpdateItem(itemType, taskItem); } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs index 4e890ec31..835e63c3c 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs @@ -84,6 +84,24 @@ public MSBuildBinaryLogComponentDetector( this.Logger = logger; } + /// + /// Initializes a new instance of the class + /// with an explicit for testing. + /// + internal MSBuildBinaryLogComponentDetector( + IComponentStreamEnumerableFactory componentStreamEnumerableFactory, + IObservableDirectoryWalkerFactory walkerFactory, + IFileUtilityService fileUtilityService, + IBinLogProcessor binLogProcessor, + ILogger logger) + { + this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; + this.Scanner = walkerFactory; + this.binLogProcessor = binLogProcessor; + this.fileUtilityService = fileUtilityService; + this.Logger = logger; + } + /// public override string Id => "MSBuildBinaryLog"; @@ -181,6 +199,71 @@ private static bool IsDevelopmentOnlyProject(MSBuildProjectInfo projectInfo) => return null; } + /// + /// Determines if a project is self-contained using MSBuild properties from the binlog. + /// Reads the SelfContained and PublishAot properties directly. + /// + private static bool IsSelfContainedFromProjectInfo(MSBuildProjectInfo projectInfo) => + projectInfo.SelfContained == true || projectInfo.PublishAot == true; + + /// + /// Determines if a project is self-contained by inspecting the lock file. + /// This is the same heuristic used by DotNetComponentDetector: + /// 1. Check for Microsoft.DotNet.ILCompiler in target libraries (indicates PublishAot). + /// 2. Check for runtime download dependencies matching framework references (indicates SelfContained). + /// + private static bool IsSelfContainedFromLockFile(PackageSpec? packageSpec, NuGetFramework? targetFramework, LockFileTarget target) + { + // PublishAot projects reference Microsoft.DotNet.ILCompiler + if (target.Libraries.Any(lib => "Microsoft.DotNet.ILCompiler".Equals(lib.Name, StringComparison.OrdinalIgnoreCase))) + { + return true; + } + + if (packageSpec?.TargetFrameworks == null || targetFramework == null) + { + return false; + } + + var targetFrameworkInfo = packageSpec.TargetFrameworks.FirstOrDefault(tf => tf.FrameworkName == targetFramework); + if (targetFrameworkInfo == null) + { + return false; + } + + var frameworkReferences = targetFrameworkInfo.FrameworkReferences; + var packageDownloads = targetFrameworkInfo.DownloadDependencies; + + if (frameworkReferences == null || frameworkReferences.Count == 0 || packageDownloads.IsDefaultOrEmpty) + { + return false; + } + + foreach (var frameworkRef in frameworkReferences) + { + if (packageDownloads.Any(pd => pd.Name.StartsWith($"{frameworkRef.Name}.Runtime", StringComparison.OrdinalIgnoreCase))) + { + return true; + } + } + + return false; + } + + /// + /// Appends "-selfcontained" suffix to the project type when the project is self-contained. + /// Matches DotNetComponentDetector's GetTargetTypeWithSelfContained behavior. + /// + private static string? GetTargetTypeWithSelfContained(string? targetType, bool isSelfContained) + { + if (string.IsNullOrWhiteSpace(targetType)) + { + return targetType; + } + + return isSelfContained ? $"{targetType}-selfcontained" : targetType; + } + private void ProcessBinlogFile(ProcessRequest processRequest) { var binlogPath = processRequest.ComponentStream.Location; @@ -201,10 +284,18 @@ private void ProcessBinlogFile(ProcessRequest processRequest) foreach (var projectInfo in projectInfos) { this.IndexProjectInfo(projectInfo, assetsFilesFound); - this.RegisterDotNetComponent(projectInfo); this.LogMissingAssetsWarnings(projectInfo); } + // Register DotNet components for projects that won't have lock files + // (projects with lock files get DotNet registration in ProcessLockFileWithProjectInfo + // where we can combine binlog and lock-file self-contained heuristics) + foreach (var projectInfo in projectInfos.Where( + pi => string.IsNullOrEmpty(pi.ProjectAssetsFile) || !this.fileUtilityService.Exists(pi.ProjectAssetsFile))) + { + this.RegisterDotNetComponent(projectInfo); + } + // Log summary warning if no assets files were found if (assetsFilesFound.Count == 0 && projectInfos.Count > 0) { @@ -264,8 +355,12 @@ private void LogMissingAssetsWarnings(MSBuildProjectInfo projectInfo) /// /// For target type (application/library), we use the OutputType property from the binlog /// which is equivalent to what DotNetComponentDetector determines by inspecting the PE headers. + /// + /// When a lock file is available, self-contained detection uses both binlog properties + /// (SelfContained, PublishAot) and the lock file heuristic (ILCompiler in libraries, + /// runtime download dependencies matching framework references) for comprehensive coverage. /// - private void RegisterDotNetComponent(MSBuildProjectInfo projectInfo) + private void RegisterDotNetComponent(MSBuildProjectInfo projectInfo, LockFile? lockFile = null) { if (string.IsNullOrEmpty(projectInfo.NETCoreSdkVersion) || string.IsNullOrEmpty(projectInfo.ProjectPath)) { @@ -285,7 +380,35 @@ private void RegisterDotNetComponent(MSBuildProjectInfo projectInfo) : "library"; } - // Get target frameworks - equivalent to iterating lockFile.Targets in DotNetComponentDetector + // Primary self-contained check from binlog properties (SelfContained, PublishAot) + var isSelfContainedFromBinlog = IsSelfContainedFromProjectInfo(projectInfo); + + if (lockFile != null) + { + // When lock file is available, check per-target self-contained + // combining binlog properties and lock file heuristics + foreach (var target in lockFile.Targets) + { + var isSelfContained = isSelfContainedFromBinlog || + IsSelfContainedFromLockFile(lockFile.PackageSpec, target.TargetFramework, target); + var projectType = GetTargetTypeWithSelfContained(targetType, isSelfContained); + var frameworkName = target.TargetFramework?.GetShortFolderName(); + + singleFileComponentRecorder.RegisterUsage( + new DetectedComponent(new DotNetComponent(projectInfo.NETCoreSdkVersion, frameworkName, projectType))); + } + + // If no targets in lock file, fall through to binlog-only registration below + if (lockFile.Targets.Count > 0) + { + return; + } + } + + // Binlog-only path: no lock file available or no targets in lock file + var projectTypeFromBinlog = GetTargetTypeWithSelfContained(targetType, isSelfContainedFromBinlog); + + // Get target frameworks from binlog properties var targetFrameworks = new List(); if (!string.IsNullOrEmpty(projectInfo.TargetFramework)) { @@ -297,20 +420,19 @@ private void RegisterDotNetComponent(MSBuildProjectInfo projectInfo) } // Register a DotNet component for each target framework - // This matches DotNetComponentDetector's loop over lockFile.Targets if (targetFrameworks.Count > 0) { foreach (var framework in targetFrameworks) { - var dotNetComponent = new DotNetComponent(projectInfo.NETCoreSdkVersion, framework, targetType); - singleFileComponentRecorder.RegisterUsage(new DetectedComponent(dotNetComponent)); + singleFileComponentRecorder.RegisterUsage( + new DetectedComponent(new DotNetComponent(projectInfo.NETCoreSdkVersion, framework, projectTypeFromBinlog))); } } else { // No target framework info available, register with just SDK version - var dotNetComponent = new DotNetComponent(projectInfo.NETCoreSdkVersion, targetFramework: null, targetType); - singleFileComponentRecorder.RegisterUsage(new DetectedComponent(dotNetComponent)); + singleFileComponentRecorder.RegisterUsage( + new DetectedComponent(new DotNetComponent(projectInfo.NETCoreSdkVersion, targetFramework: null, projectTypeFromBinlog))); } } @@ -470,6 +592,9 @@ bool IsFrameworkOrDevDependency(LockFileTargetLibrary library) => singleFileComponentRecorder, lockFile, (packageName, framework) => this.IsPackageDownloadDevDependency(packageName, framework, projectInfo)); + + // Register DotNet component with combined binlog + lock file self-contained detection + this.RegisterDotNetComponent(projectInfo, lockFile); } /// diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildProjectInfo.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildProjectInfo.cs index 97ab6cc93..13d4d0ce7 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildProjectInfo.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildProjectInfo.cs @@ -21,6 +21,7 @@ internal class MSBuildProjectInfo [nameof(IsTestProject)] = (info, value) => info.IsTestProject = string.Equals(value, "true", StringComparison.OrdinalIgnoreCase), [nameof(NETCoreSdkVersion)] = (info, value) => info.NETCoreSdkVersion = value, [nameof(OutputType)] = (info, value) => info.OutputType = value, + [nameof(PublishAot)] = (info, value) => info.PublishAot = string.Equals(value, "true", StringComparison.OrdinalIgnoreCase), [nameof(ProjectAssetsFile)] = (info, value) => { if (!string.IsNullOrEmpty(value)) @@ -78,6 +79,12 @@ internal class MSBuildProjectInfo /// public string? OutputType { get; set; } + /// + /// Gets or sets a value indicating whether this project uses native AOT compilation. + /// Corresponds to the MSBuild PublishAot property. + /// + public bool? PublishAot { get; set; } + /// /// Gets or sets the .NET Core SDK version used to build the project. /// Corresponds to the MSBuild NETCoreSdkVersion property. @@ -129,8 +136,10 @@ internal class MSBuildProjectInfo /// /// Gets a value indicating whether this is an outer build of a multi-targeted project. + /// The outer build has TargetFrameworks set but TargetFramework is empty (it dispatches to inner builds). + /// Inner builds have both TargetFrameworks and TargetFramework set. /// - public bool IsOuterBuild => !string.IsNullOrEmpty(this.TargetFrameworks); + public bool IsOuterBuild => !string.IsNullOrEmpty(this.TargetFrameworks) && string.IsNullOrEmpty(this.TargetFramework); /// /// Determines whether the specified item type is one that this class captures. @@ -190,4 +199,52 @@ public bool TryRemoveItem(string itemType, string itemSpec) var dictionary = getDictionary(this); return dictionary.Remove(itemSpec); } + + /// + /// Merges another project info into this one, forming a superset. + /// Properties from override null values in this instance. + /// Boolean properties use logical OR (true wins). + /// Items from are added if not already present. + /// + /// The other project info to merge from. + public void MergeWith(MSBuildProjectInfo other) + { + // Merge boolean properties: true wins (if either says true, result is true) + this.IsDevelopment = MergeBool(this.IsDevelopment, other.IsDevelopment); + this.IsPackable = MergeBool(this.IsPackable, other.IsPackable); + this.IsShipping = MergeBool(this.IsShipping, other.IsShipping); + this.IsTestProject = MergeBool(this.IsTestProject, other.IsTestProject); + this.PublishAot = MergeBool(this.PublishAot, other.PublishAot); + this.SelfContained = MergeBool(this.SelfContained, other.SelfContained); + + // Merge string properties: prefer non-null/non-empty + this.OutputType ??= other.OutputType; + this.NETCoreSdkVersion ??= other.NETCoreSdkVersion; + this.ProjectAssetsFile ??= other.ProjectAssetsFile; + this.TargetFramework ??= other.TargetFramework; + this.TargetFrameworks ??= other.TargetFrameworks; + + // Merge items: add items from other that are not already present + MergeItems(this.PackageReference, other.PackageReference); + MergeItems(this.PackageDownload, other.PackageDownload); + } + + private static bool? MergeBool(bool? existing, bool? incoming) + { + if (existing == true || incoming == true) + { + return true; + } + + return existing ?? incoming; + } + + private static void MergeItems(IDictionary target, IDictionary source) + { + foreach (var kvp in source) + { + // TryAdd: only add if not already present (existing items win) + target.TryAdd(kvp.Key, kvp.Value); + } + } } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Microsoft.ComponentDetection.Detectors.Tests.csproj b/test/Microsoft.ComponentDetection.Detectors.Tests/Microsoft.ComponentDetection.Detectors.Tests.csproj index 082707904..45a66cfd5 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/Microsoft.ComponentDetection.Detectors.Tests.csproj +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/Microsoft.ComponentDetection.Detectors.Tests.csproj @@ -8,6 +8,7 @@ + diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs new file mode 100644 index 000000000..b7e6c2813 --- /dev/null +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs @@ -0,0 +1,1642 @@ +namespace Microsoft.ComponentDetection.Detectors.Tests.NuGet; + +using System; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using AwesomeAssertions; +using Microsoft.ComponentDetection.Detectors.NuGet; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// Integration tests for that build real MSBuild projects +/// to produce binlog files, then parse them to verify extracted project information. +/// +[TestClass] +[TestCategory("Integration")] +public class BinLogProcessorTests +{ + private readonly BinLogProcessor processor; + private string testDir = null!; + + public BinLogProcessorTests() => this.processor = new BinLogProcessor(NullLogger.Instance); + + [TestInitialize] + public void TestInitialize() + { + this.testDir = Path.Combine(Path.GetTempPath(), "BinLogProcessorTests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(this.testDir); + + // Pin the SDK version so temp projects use the same SDK as the workspace + var globalJson = """{ "sdk": { "version": "8.0.100", "rollForward": "latestFeature" } }"""; + WriteFile(this.testDir, "global.json", globalJson); + } + + [TestCleanup] + public void TestCleanup() + { + try + { + if (Directory.Exists(this.testDir)) + { + Directory.Delete(this.testDir, recursive: true); + } + } + catch + { + // Best effort cleanup + } + } + + [TestMethod] + public async Task SingleTargetProject_ExtractsBasicProperties() + { + var projectDir = Path.Combine(this.testDir, "SingleTarget"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0 + Exe + + + + + + """; + WriteFile(projectDir, "SingleTarget.csproj", content); + WriteMinimalProgram(projectDir); + + var binlogPath = await BuildProjectAsync(projectDir, "SingleTarget.csproj"); + var results = this.processor.ExtractProjectInfo(binlogPath); + + results.Should().NotBeEmpty(); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("SingleTarget.csproj", StringComparison.OrdinalIgnoreCase)); + + projectInfo.TargetFramework.Should().Be("net8.0"); + projectInfo.OutputType.Should().Be("Exe"); + projectInfo.NETCoreSdkVersion.Should().NotBeNullOrEmpty(); + projectInfo.ProjectAssetsFile.Should().NotBeNullOrEmpty(); + projectInfo.IsOuterBuild.Should().BeFalse(); + + // PackageReference should be captured + projectInfo.PackageReference.Should().ContainKey("Newtonsoft.Json"); + } + + [TestMethod] + public async Task MultiTargetProject_ExtractsOuterAndInnerBuilds() + { + var projectDir = Path.Combine(this.testDir, "MultiTarget"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0;net7.0 + + + + + + """; + WriteFile(projectDir, "MultiTarget.csproj", content); + + var binlogPath = await BuildProjectAsync(projectDir, "MultiTarget.csproj"); + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("MultiTarget.csproj", StringComparison.OrdinalIgnoreCase)); + + // The outer build should have TargetFrameworks set + projectInfo.IsOuterBuild.Should().BeTrue(); + projectInfo.TargetFrameworks.Should().Contain("net8.0"); + projectInfo.TargetFrameworks.Should().Contain("net7.0"); + + // Inner builds should be captured + projectInfo.InnerBuilds.Should().HaveCountGreaterThanOrEqualTo(2); + + var net8Inner = projectInfo.InnerBuilds.FirstOrDefault( + ib => ib.TargetFramework != null && ib.TargetFramework.Contains("net8.0")); + var net7Inner = projectInfo.InnerBuilds.FirstOrDefault( + ib => ib.TargetFramework != null && ib.TargetFramework.Contains("net7.0")); + + net8Inner.Should().NotBeNull(); + net7Inner.Should().NotBeNull(); + + // Each inner build should have its own PackageReference + net8Inner!.PackageReference.Should().ContainKey("Newtonsoft.Json"); + net7Inner!.PackageReference.Should().ContainKey("Newtonsoft.Json"); + } + + [TestMethod] + public async Task TestProject_ExtractsIsTestProject() + { + var projectDir = Path.Combine(this.testDir, "TestProject"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0 + true + + + + + + """; + WriteFile(projectDir, "TestProject.csproj", content); + + var binlogPath = await BuildProjectAsync(projectDir, "TestProject.csproj"); + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("TestProject.csproj", StringComparison.OrdinalIgnoreCase)); + + projectInfo.IsTestProject.Should().Be(true); + } + + [TestMethod] + public async Task ProjectWithIsShippingFalse_ExtractsIsShipping() + { + var projectDir = Path.Combine(this.testDir, "NonShipping"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0 + false + + + """; + WriteFile(projectDir, "NonShipping.csproj", content); + + var binlogPath = await BuildProjectAsync(projectDir, "NonShipping.csproj"); + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("NonShipping.csproj", StringComparison.OrdinalIgnoreCase)); + + projectInfo.IsShipping.Should().Be(false); + } + + [TestMethod] + public async Task MultipleProjectsInSameBuild_ExtractsAll() + { + // Create a solution with two projects + var solutionDir = Path.Combine(this.testDir, "MultiProject"); + Directory.CreateDirectory(solutionDir); + + var projectADir = Path.Combine(solutionDir, "ProjectA"); + var projectBDir = Path.Combine(solutionDir, "ProjectB"); + Directory.CreateDirectory(projectADir); + Directory.CreateDirectory(projectBDir); + + var projectAContent = """ + + + net8.0 + Exe + + + + + + """; + WriteFile(projectADir, "ProjectA.csproj", projectAContent); + WriteMinimalProgram(projectADir); + + var projectBContent = """ + + + net8.0 + Library + + + + + + """; + WriteFile(projectBDir, "ProjectB.csproj", projectBContent); + + // Create a solution to build both projects in a single binlog + await RunDotNetAsync(solutionDir, "new sln --name MultiProject"); + await RunDotNetAsync(solutionDir, $"sln add \"{Path.Combine(projectADir, "ProjectA.csproj")}\""); + await RunDotNetAsync(solutionDir, $"sln add \"{Path.Combine(projectBDir, "ProjectB.csproj")}\""); + var binlogPath = Path.Combine(solutionDir, "build.binlog"); + await RunDotNetAsync(solutionDir, $"build \"MultiProject.sln\" -bl:\"{binlogPath}\" /p:UseAppHost=false"); + + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectA = results.FirstOrDefault(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("ProjectA.csproj", StringComparison.OrdinalIgnoreCase)); + var projectB = results.FirstOrDefault(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("ProjectB.csproj", StringComparison.OrdinalIgnoreCase)); + + projectA.Should().NotBeNull(); + projectB.Should().NotBeNull(); + + projectA!.OutputType.Should().Be("Exe"); + projectA.PackageReference.Should().ContainKey("Newtonsoft.Json"); + + projectB!.OutputType.Should().Be("Library"); + projectB.PackageReference.Should().ContainKey("System.Text.Json"); + } + + [TestMethod] + public async Task ProjectToProjectReference_ExtractsBothProjects() + { + var solutionDir = Path.Combine(this.testDir, "P2P"); + Directory.CreateDirectory(solutionDir); + + var libDir = Path.Combine(solutionDir, "MyLib"); + var appDir = Path.Combine(solutionDir, "MyApp"); + Directory.CreateDirectory(libDir); + Directory.CreateDirectory(appDir); + + var libContent = """ + + + net8.0 + + + + + + """; + WriteFile(libDir, "MyLib.csproj", libContent); + + var appContent = $""" + + + net8.0 + Exe + + + + + + + """; + WriteFile(appDir, "MyApp.csproj", appContent); + WriteMinimalProgram(appDir); + + // Build the app (which also builds the lib) + var binlogPath = await BuildProjectAsync(appDir, "MyApp.csproj"); + var results = this.processor.ExtractProjectInfo(binlogPath); + + var appInfo = results.FirstOrDefault(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("MyApp.csproj", StringComparison.OrdinalIgnoreCase)); + var libInfo = results.FirstOrDefault(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("MyLib.csproj", StringComparison.OrdinalIgnoreCase)); + + appInfo.Should().NotBeNull(); + libInfo.Should().NotBeNull(); + + appInfo!.OutputType.Should().Be("Exe"); + appInfo.PackageReference.Should().ContainKey("System.Text.Json"); + + libInfo!.PackageReference.Should().ContainKey("Newtonsoft.Json"); + } + + [TestMethod] + public async Task CustomTargetModifiesProperties_PropertyIsOverridden() + { + var projectDir = Path.Combine(this.testDir, "CustomTarget"); + Directory.CreateDirectory(projectDir); + + // A custom target that runs before Restore and changes a property + var content = """ + + + net8.0 + false + + + + true + + + + + + + """; + WriteFile(projectDir, "CustomTarget.csproj", content); + + var binlogPath = await BuildProjectAsync(projectDir, "CustomTarget.csproj"); + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("CustomTarget.csproj", StringComparison.OrdinalIgnoreCase)); + + // The custom target's property override should be captured + // depending on when the binlog captures it. + // At minimum the evaluation-time value should be captured. + projectInfo.IsTestProject.Should().NotBeNull(); + } + + [TestMethod] + public async Task CustomTargetAddsItems_ItemsAreCaptured() + { + var projectDir = Path.Combine(this.testDir, "CustomItems"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0 + + + + + + + + + + + """; + WriteFile(projectDir, "CustomItems.csproj", content); + + var binlogPath = await BuildProjectAsync(projectDir, "CustomItems.csproj"); + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("CustomItems.csproj", StringComparison.OrdinalIgnoreCase)); + + projectInfo.PackageReference.Should().ContainKey("Newtonsoft.Json"); + + // The PackageDownload added by the custom target should be captured + // (depends on whether the task runs and binlog captures TaskParameter events) + // This tests that task-parameter-level item tracking works + projectInfo.PackageDownload.Should().ContainKey("System.Memory"); + } + + [TestMethod] + public async Task PackageDownloadItems_AreCaptured() + { + var projectDir = Path.Combine(this.testDir, "PkgDownload"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0 + + + + + + + """; + WriteFile(projectDir, "PkgDownload.csproj", content); + + var binlogPath = await BuildProjectAsync(projectDir, "PkgDownload.csproj"); + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("PkgDownload.csproj", StringComparison.OrdinalIgnoreCase)); + + projectInfo.PackageReference.Should().ContainKey("Newtonsoft.Json"); + projectInfo.PackageDownload.Should().ContainKey("System.Memory"); + } + + [TestMethod] + public async Task MultiTargetWithDifferentPackagesPerTfm_InnerBuildsHaveDifferentItems() + { + var projectDir = Path.Combine(this.testDir, "PerTfmPackages"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0;net7.0 + + + + + + + + + """; + WriteFile(projectDir, "PerTfmPackages.csproj", content); + + var binlogPath = await BuildProjectAsync(projectDir, "PerTfmPackages.csproj"); + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("PerTfmPackages.csproj", StringComparison.OrdinalIgnoreCase)); + + projectInfo.IsOuterBuild.Should().BeTrue(); + + var net8Inner = projectInfo.InnerBuilds.FirstOrDefault( + ib => ib.TargetFramework != null && ib.TargetFramework.Contains("net8.0")); + var net7Inner = projectInfo.InnerBuilds.FirstOrDefault( + ib => ib.TargetFramework != null && ib.TargetFramework.Contains("net7.0")); + + net8Inner.Should().NotBeNull(); + net7Inner.Should().NotBeNull(); + + // net8.0 inner build should have both packages + net8Inner!.PackageReference.Should().ContainKey("Newtonsoft.Json"); + net8Inner.PackageReference.Should().ContainKey("System.Text.Json"); + + // net7.0 inner build should only have Newtonsoft.Json + net7Inner!.PackageReference.Should().ContainKey("Newtonsoft.Json"); + net7Inner.PackageReference.Should().NotContainKey("System.Text.Json"); + } + + [TestMethod] + public async Task SelfContainedProject_ExtractsSelfContained() + { + var projectDir = Path.Combine(this.testDir, "SelfContained"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0 + Exe + true + win-x64 + + + """; + WriteFile(projectDir, "SelfContained.csproj", content); + WriteMinimalProgram(projectDir); + + // Use restore-only build: evaluation captures properties in the binlog + // without needing a full build (which conflicts UseAppHost=false + SelfContained=true) + var binlogPath = Path.Combine(projectDir, "build.binlog"); + await RunDotNetAsync(projectDir, $"msbuild \"{Path.Combine(projectDir, "SelfContained.csproj")}\" -t:Restore -bl:\"{binlogPath}\""); + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("SelfContained.csproj", StringComparison.OrdinalIgnoreCase)); + + projectInfo.SelfContained.Should().Be(true); + projectInfo.OutputType.Should().Be("Exe"); + } + + [TestMethod] + public async Task ItemWithMetadata_MetadataIsCaptured() + { + var projectDir = Path.Combine(this.testDir, "ItemMetadata"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0 + + + + + all + true + + + + """; + WriteFile(projectDir, "ItemMetadata.csproj", content); + + var binlogPath = await BuildProjectAsync(projectDir, "ItemMetadata.csproj"); + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("ItemMetadata.csproj", StringComparison.OrdinalIgnoreCase)); + + projectInfo.PackageReference.Should().ContainKey("StyleCop.Analyzers"); + + var styleCopItem = projectInfo.PackageReference["StyleCop.Analyzers"]; + styleCopItem.GetMetadata("IsDevelopmentDependency").Should().Be("true"); + } + + [TestMethod] + public void EmptyBinlogPath_ReturnsEmptyList() + { + // Test with a non-existent file + var results = this.processor.ExtractProjectInfo(Path.Combine(this.testDir, "nonexistent.binlog")); + results.Should().BeEmpty(); + } + + [TestMethod] + public async Task EvaluationPropertyReassignment_LaterDefinitionWins() + { + // An imported .targets file overrides a property set in the project file. + // This tests that PropertyReassignment events during evaluation are captured. + var projectDir = Path.Combine(this.testDir, "PropReassign"); + Directory.CreateDirectory(projectDir); + + // Create a .targets file that overrides OutputType + var targetsContent = """ + + + Exe + + + """; + WriteFile(projectDir, "override.targets", targetsContent); + + var content = """ + + + net8.0 + Library + + + + + + + """; + WriteFile(projectDir, "PropReassign.csproj", content); + WriteMinimalProgram(projectDir); + + var binlogPath = await BuildProjectAsync(projectDir, "PropReassign.csproj"); + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("PropReassign.csproj", StringComparison.OrdinalIgnoreCase)); + + // The imported .targets overrides OutputType from Library to Exe during evaluation + projectInfo.OutputType.Should().Be("Exe"); + } + + [TestMethod] + public async Task TargetSetsProperty_OnlyCapturedAtEvaluationTime() + { + // Verifies that target-level changes are NOT captured in binlog + // property events. This documents a known limitation: only evaluation-time + // property values are tracked. + var projectDir = Path.Combine(this.testDir, "TargetPropLimit"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0 + Library + + + + Exe + + + + """; + WriteFile(projectDir, "TargetPropLimit.csproj", content); + + // Use restore-only to avoid compile errors (the target changes OutputType to Exe + // which expects a Main method that doesn't exist). Evaluation properties are still captured. + var binlogPath = Path.Combine(projectDir, "build.binlog"); + await RunDotNetAsync(projectDir, $"msbuild TargetPropLimit.csproj -t:Restore -bl:\"{binlogPath}\""); + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("TargetPropLimit.csproj", StringComparison.OrdinalIgnoreCase)); + + // Target-level property changes don't emit PropertyReassignment events in binlog. + // We only capture the evaluation-time value. + projectInfo.OutputType.Should().Be( + "Library", + "target-level property changes are not visible in binlog events"); + } + + [TestMethod] + public async Task EvaluationBoolPropertyReassignment_LaterDefinitionWins() + { + // An imported .targets file overrides IsTestProject from false to true. + // This tests that boolean property reassignment during evaluation is captured. + var projectDir = Path.Combine(this.testDir, "BoolReassign"); + Directory.CreateDirectory(projectDir); + + var targetsContent = """ + + + true + + + """; + WriteFile(projectDir, "override.targets", targetsContent); + + var content = """ + + + net8.0 + false + + + + """; + WriteFile(projectDir, "BoolReassign.csproj", content); + + var binlogPath = await BuildProjectAsync(projectDir, "BoolReassign.csproj"); + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("BoolReassign.csproj", StringComparison.OrdinalIgnoreCase)); + + // The imported .targets overrides IsTestProject to true during evaluation + projectInfo.IsTestProject.Should().Be(true); + } + + [TestMethod] + public async Task TargetBeforeRestoreAddsPackageReference_ItemIsCaptured() + { + // A target running before Restore adds a PackageReference + var projectDir = Path.Combine(this.testDir, "TargetAddsRef"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0 + + + + + + + + + + + """; + WriteFile(projectDir, "TargetAddsRef.csproj", content); + + var binlogPath = await BuildProjectAsync(projectDir, "TargetAddsRef.csproj"); + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("TargetAddsRef.csproj", StringComparison.OrdinalIgnoreCase)); + + // The static PackageReference from evaluation + projectInfo.PackageReference.Should().ContainKey("Newtonsoft.Json"); + + // The dynamically-added PackageReference from the target + projectInfo.PackageReference.Should().ContainKey("System.Memory"); + } + + [TestMethod] + public async Task TargetBeforeRestoreAddsPackageDownload_ItemIsCaptured() + { + // A target running before Restore adds a PackageDownload + var projectDir = Path.Combine(this.testDir, "TargetAddsDownload"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0 + + + + + + + + + + + """; + WriteFile(projectDir, "TargetAddsDownload.csproj", content); + + var binlogPath = await BuildProjectAsync(projectDir, "TargetAddsDownload.csproj"); + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("TargetAddsDownload.csproj", StringComparison.OrdinalIgnoreCase)); + + projectInfo.PackageReference.Should().ContainKey("Newtonsoft.Json"); + projectInfo.PackageDownload.Should().ContainKey("System.Memory"); + } + + [TestMethod] + public async Task TraversalBuildAndPublish_MergesProperties() + { + // An orchestrator project builds a child project, then restores it with SelfContained. + // This simulates a traversal or CI script that does build + publish in one binlog. + var solutionDir = Path.Combine(this.testDir, "TraversalMerge"); + Directory.CreateDirectory(solutionDir); + + var appDir = Path.Combine(solutionDir, "App"); + Directory.CreateDirectory(appDir); + + var appContent = """ + + + net8.0 + Exe + + + + + + """; + WriteFile(appDir, "App.csproj", appContent); + WriteMinimalProgram(appDir); + + // Orchestrator: restore, build, then restore with SelfContained (all in one binlog) + var orchestratorContent = """ + + + + + + + + """; + WriteFile(solutionDir, "Orchestrator.proj", orchestratorContent); + + var binlogPath = Path.Combine(solutionDir, "build.binlog"); + await RunDotNetAsync(solutionDir, $"msbuild Orchestrator.proj -t:BuildAndPublish -bl:\"{binlogPath}\""); + + var results = this.processor.ExtractProjectInfo(binlogPath); + + var appInfo = results.FirstOrDefault(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("App.csproj", StringComparison.OrdinalIgnoreCase)); + + appInfo.Should().NotBeNull(); + + // After merge, SelfContained should be true (from the second restore pass) + appInfo!.SelfContained.Should().Be(true); + + // The original PackageReference should still be present + appInfo.PackageReference.Should().ContainKey("Newtonsoft.Json"); + } + + [TestMethod] + public async Task BuildThenPublishSelfContained_MergesSelfContained() + { + // Simulates a common CI pattern: build first (non-self-contained), then publish (self-contained). + // Both produce entries in the same binlog. The merge should yield SelfContained=true. + var projectDir = Path.Combine(this.testDir, "BuildPublishMerge"); + Directory.CreateDirectory(projectDir); + + var appContent = """ + + + net8.0 + Exe + + + + + + """; + WriteFile(projectDir, "App.csproj", appContent); + WriteMinimalProgram(projectDir); + + // First build (not self-contained), then publish (self-contained), both into same binlog + // We use an orchestrator project that invokes MSBuild twice + var orchestratorContent = """ + + + + + + + + """; + WriteFile(projectDir, "Orchestrator.proj", orchestratorContent); + + var binlogPath = Path.Combine(projectDir, "build.binlog"); + await RunDotNetAsync(projectDir, $"msbuild Orchestrator.proj -t:BuildAndPublish -bl:\"{binlogPath}\""); + + var results = this.processor.ExtractProjectInfo(binlogPath); + + var appInfo = results.FirstOrDefault(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("App.csproj", StringComparison.OrdinalIgnoreCase)); + + appInfo.Should().NotBeNull(); + appInfo!.SelfContained.Should().Be(true, "the publish pass sets SelfContained=true, which should be merged"); + appInfo.OutputType.Should().Be("Exe"); + appInfo.PackageReference.Should().ContainKey("Newtonsoft.Json"); + } + + [TestMethod] + public async Task GlobalPropertyFromCommandLine_CapturedInBinlog() + { + // Pass IsTestProject=true as a global property via /p: on the command line + var projectDir = Path.Combine(this.testDir, "GlobalProp"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0 + + + + + + """; + WriteFile(projectDir, "GlobalProp.csproj", content); + + var binlogPath = Path.Combine(projectDir, "build.binlog"); + await RunDotNetAsync(projectDir, $"build GlobalProp.csproj -bl:\"{binlogPath}\" /p:UseAppHost=false /p:IsTestProject=true"); + + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("GlobalProp.csproj", StringComparison.OrdinalIgnoreCase)); + + // Global property should be captured from evaluation + projectInfo.IsTestProject.Should().Be(true); + } + + [TestMethod] + public async Task GlobalPropertyOverridesProjectFile_CommandLineWins() + { + // Project file says OutputType=Library, command line says OutputType=Exe + var projectDir = Path.Combine(this.testDir, "GlobalOverride"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0 + Library + + + """; + WriteFile(projectDir, "GlobalOverride.csproj", content); + WriteMinimalProgram(projectDir); + + var binlogPath = Path.Combine(projectDir, "build.binlog"); + await RunDotNetAsync(projectDir, $"build GlobalOverride.csproj -bl:\"{binlogPath}\" /p:UseAppHost=false /p:OutputType=Exe"); + + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("GlobalOverride.csproj", StringComparison.OrdinalIgnoreCase)); + + // Command-line global property should override the project file value + projectInfo.OutputType.Should().Be("Exe"); + } + + [TestMethod] + public async Task EnvironmentVariableProperty_CapturedInBinlog() + { + // MSBuild automatically promotes environment variables to properties. + // Setting IsTestProject as an env var (without referencing it in the project) + // should still be captured in the binlog evaluation. + var projectDir = Path.Combine(this.testDir, "EnvVar"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0 + + + + + + """; + WriteFile(projectDir, "EnvVar.csproj", content); + + var binlogPath = Path.Combine(projectDir, "build.binlog"); + await RunProcessWithEnvAsync( + projectDir, + "dotnet", + $"build EnvVar.csproj -bl:\"{binlogPath}\" /p:UseAppHost=false", + ("IsTestProject", "true")); + + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("EnvVar.csproj", StringComparison.OrdinalIgnoreCase)); + + // The env var is automatically promoted to an MSBuild property during evaluation + projectInfo.IsTestProject.Should().Be(true); + } + + [TestMethod] + public async Task TargetRemovesPackageReference_ItemIsRemoved() + { + // A PackageReference is defined in the project, but a target removes it before restore + var projectDir = Path.Combine(this.testDir, "RemoveItem"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0 + + + + + + + + + + + + """; + WriteFile(projectDir, "RemoveItem.csproj", content); + + var binlogPath = await BuildProjectAsync(projectDir, "RemoveItem.csproj"); + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("RemoveItem.csproj", StringComparison.OrdinalIgnoreCase)); + + // The target removes System.Memory before restore + projectInfo.PackageReference.Should().ContainKey("Newtonsoft.Json"); + projectInfo.PackageReference.Should().NotContainKey( + "System.Memory", + "the target removes System.Memory before restore"); + } + + [TestMethod] + public async Task ItemUpdateTag_MetadataIsUpdated() + { + // Uses to change metadata + var projectDir = Path.Combine(this.testDir, "ItemUpdate"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0 + + + + + + + + + """; + WriteFile(projectDir, "ItemUpdate.csproj", content); + + var binlogPath = await BuildProjectAsync(projectDir, "ItemUpdate.csproj"); + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("ItemUpdate.csproj", StringComparison.OrdinalIgnoreCase)); + + projectInfo.PackageReference.Should().ContainKey("Newtonsoft.Json"); + + // After the Update, PrivateAssets metadata should be set + var item = projectInfo.PackageReference["Newtonsoft.Json"]; + item.GetMetadata("PrivateAssets").Should().Be("all"); + } + + [TestMethod] + public async Task ItemDefinitionGroup_DefaultMetadataApplied() + { + // ItemDefinitionGroup sets default PrivateAssets for all PackageReference items + var projectDir = Path.Combine(this.testDir, "ItemDefGroup"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0 + + + + all + + + + + + + + """; + WriteFile(projectDir, "ItemDefGroup.csproj", content); + + var binlogPath = await BuildProjectAsync(projectDir, "ItemDefGroup.csproj"); + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("ItemDefGroup.csproj", StringComparison.OrdinalIgnoreCase)); + + projectInfo.PackageReference.Should().ContainKey("Newtonsoft.Json"); + projectInfo.PackageReference.Should().ContainKey("System.Memory"); + + // ItemDefinitionGroup should apply PrivateAssets=all to both items + var newtonsoftItem = projectInfo.PackageReference["Newtonsoft.Json"]; + newtonsoftItem.GetMetadata("PrivateAssets").Should().Be("all"); + + var memoryItem = projectInfo.PackageReference["System.Memory"]; + memoryItem.GetMetadata("PrivateAssets").Should().Be("all"); + } + + [TestMethod] + public async Task TargetRemovesAndReAddsWithMetadata_MetadataReflectsChange() + { + // A target runs and modifies metadata via Remove+Include pattern. + // This is necessary because Item Update inside targets does not emit + // TaskParameter events in binlog, but Remove+Include does. + var projectDir = Path.Combine(this.testDir, "TargetMetadata"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0 + + + + + + + + + all + + + + + """; + WriteFile(projectDir, "TargetMetadata.csproj", content); + + var binlogPath = await BuildProjectAsync(projectDir, "TargetMetadata.csproj"); + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("TargetMetadata.csproj", StringComparison.OrdinalIgnoreCase)); + + projectInfo.PackageReference.Should().ContainKey("Newtonsoft.Json"); + + // The Remove+Include in target produces TaskParameter events + var item = projectInfo.PackageReference["Newtonsoft.Json"]; + item.GetMetadata("PrivateAssets").Should().Be("all"); + } + + [TestMethod] + public async Task TargetItemUpdateLimitation_UpdateNotVisibleInBinlog() + { + // Documents a known limitation: inside a + // does NOT emit TaskParameter events in the binlog. The metadata + // change is invisible to the BinLogProcessor. + var projectDir = Path.Combine(this.testDir, "TargetUpdateLimit"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0 + + + + + + + + + + + """; + WriteFile(projectDir, "TargetUpdateLimit.csproj", content); + + var binlogPath = await BuildProjectAsync(projectDir, "TargetUpdateLimit.csproj"); + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("TargetUpdateLimit.csproj", StringComparison.OrdinalIgnoreCase)); + + projectInfo.PackageReference.Should().ContainKey("Newtonsoft.Json"); + + // Item Update inside targets doesn't emit TaskParameter events. + // PrivateAssets remains at the evaluation-time value (not set). + var item = projectInfo.PackageReference["Newtonsoft.Json"]; + item.GetMetadata("PrivateAssets").Should().BeNullOrEmpty( + "item Update inside targets is not visible in binlog TaskParameter events"); + } + + [TestMethod] + public async Task TargetAddsItemButPropertyInvisible_ItemCapturedPropertyNot() + { + // A single target modifies both a property and adds an item. + // Property changes in targets are invisible, but item additions are visible. + var projectDir = Path.Combine(this.testDir, "ComboPropsItems"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0 + false + + + + + + + true + + + + + + + """; + WriteFile(projectDir, "ComboPropsItems.csproj", content); + + var binlogPath = await BuildProjectAsync(projectDir, "ComboPropsItems.csproj"); + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("ComboPropsItems.csproj", StringComparison.OrdinalIgnoreCase)); + + // Property change in target is invisible (stays at evaluation value) + projectInfo.IsTestProject.Should().Be( + false, + "target-level property changes are not visible in binlog events"); + projectInfo.PackageReference.Should().ContainKey("Newtonsoft.Json"); + + // Item addition in target IS captured via TaskParameter events + projectInfo.PackageDownload.Should().ContainKey("System.Memory"); + } + + [TestMethod] + public async Task MultiTargetWithTargetConditionalOnTfm_PerInnerBuildChanges() + { + // A multi-targeted project where a target conditionally adds a PackageDownload + // only for the net8.0 TFM + var projectDir = Path.Combine(this.testDir, "MultiTargetConditional"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0;net7.0 + + + + + + + + + + + """; + WriteFile(projectDir, "MultiTargetConditional.csproj", content); + + var binlogPath = await BuildProjectAsync(projectDir, "MultiTargetConditional.csproj"); + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("MultiTargetConditional.csproj", StringComparison.OrdinalIgnoreCase)); + + projectInfo.IsOuterBuild.Should().BeTrue(); + + var net8Inner = projectInfo.InnerBuilds.First( + ib => ib.TargetFramework != null && ib.TargetFramework.Contains("net8.0")); + var net7Inner = projectInfo.InnerBuilds.First( + ib => ib.TargetFramework != null && ib.TargetFramework.Contains("net7.0")); + + // net8.0 should have the target-added PackageDownload + net8Inner.PackageDownload.Should().ContainKey("System.Memory"); + + // net7.0 should NOT have it (condition not met) + net7Inner.PackageDownload.Should().NotContainKey("System.Memory"); + } + + [TestMethod] + public async Task GlobalPropertyAndTargetOverride_TargetWins() + { + // Global property sets SelfContained=false, but a target changes it to true + var projectDir = Path.Combine(this.testDir, "GlobalAndTarget"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0 + Exe + + + + + + + true + + + + """; + WriteFile(projectDir, "GlobalAndTarget.csproj", content); + WriteMinimalProgram(projectDir); + + // Pass SelfContained=false on command line, but target overrides to true + var binlogPath = Path.Combine(projectDir, "build.binlog"); + await RunDotNetAsync(projectDir, $"msbuild GlobalAndTarget.csproj -t:Restore -bl:\"{binlogPath}\" /p:SelfContained=false"); + + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("GlobalAndTarget.csproj", StringComparison.OrdinalIgnoreCase)); + + // Note: Global properties cannot be overridden by targets in MSBuild. + // The global property wins. This tests that we correctly capture this MSBuild behavior. + projectInfo.SelfContained.Should().Be( + false, + "global properties cannot be overridden by targets in MSBuild"); + } + + [TestMethod] + public async Task ImportedPropsFile_PropertyFromImportCaptured() + { + // A project imports a .props file that sets a property + var projectDir = Path.Combine(this.testDir, "ImportedProps"); + Directory.CreateDirectory(projectDir); + + var propsContent = """ + + + true + false + + + """; + WriteFile(projectDir, "Directory.Build.props", propsContent); + + var content = """ + + + net8.0 + + + + + + """; + WriteFile(projectDir, "ImportedProps.csproj", content); + + var binlogPath = await BuildProjectAsync(projectDir, "ImportedProps.csproj"); + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("ImportedProps.csproj", StringComparison.OrdinalIgnoreCase)); + + // Properties from the imported file should be captured during evaluation + projectInfo.IsTestProject.Should().Be(true); + projectInfo.IsShipping.Should().Be(false); + } + + [TestMethod] + public async Task TargetRemovesAndReAddsItem_FinalStateReflectsReAdd() + { + // A target removes a PackageReference and then re-adds it with different metadata + var projectDir = Path.Combine(this.testDir, "RemoveReAdd"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0 + + + + + + + + + all + + + + + """; + WriteFile(projectDir, "RemoveReAdd.csproj", content); + + var binlogPath = await BuildProjectAsync(projectDir, "RemoveReAdd.csproj"); + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("RemoveReAdd.csproj", StringComparison.OrdinalIgnoreCase)); + + projectInfo.PackageReference.Should().ContainKey("Newtonsoft.Json"); + + // The re-added item should have the updated version and metadata + var item = projectInfo.PackageReference["Newtonsoft.Json"]; + item.GetMetadata("Version").Should().Be("13.0.3"); + item.GetMetadata("PrivateAssets").Should().Be("all"); + } + + [TestMethod] + public async Task MultiTarget_BuildAndPublishSelfContained_MergesInnerBuilds() + { + // Multi-targeted project where a second pass restores with SelfContained=true. + // The merge should mark the inner builds as self-contained. + var solutionDir = Path.Combine(this.testDir, "MultiTargetPublish"); + Directory.CreateDirectory(solutionDir); + + var appContent = """ + + + net8.0;net7.0 + Exe + + + + + + """; + var appDir = Path.Combine(solutionDir, "App"); + Directory.CreateDirectory(appDir); + WriteFile(appDir, "App.csproj", appContent); + WriteMinimalProgram(appDir); + + // Orchestrator: restore, build, then restore with SelfContained (all in one binlog) + var orchestratorContent = """ + + + + + + + + """; + WriteFile(solutionDir, "Orchestrator.proj", orchestratorContent); + + var binlogPath = Path.Combine(solutionDir, "build.binlog"); + await RunDotNetAsync(solutionDir, $"msbuild Orchestrator.proj -t:BuildThenRestore -bl:\"{binlogPath}\""); + + var results = this.processor.ExtractProjectInfo(binlogPath); + + var appInfo = results.FirstOrDefault(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("App.csproj", StringComparison.OrdinalIgnoreCase)); + + appInfo.Should().NotBeNull(); + + // The first build creates inner builds; the second restore adds SelfContained. + // Verify at least the project is found and contains expected packages. + appInfo!.PackageReference.Should().ContainKey("Newtonsoft.Json"); + + // After merging, some representation should have SelfContained=true + // (either on the project directly, or in inner builds) + var allInfos = new[] { appInfo } + .Concat(appInfo.InnerBuilds) + .ToList(); + allInfos.Should().Contain( + p => p.SelfContained == true, + "the second pass sets SelfContained=true which should be merged"); + } + + [TestMethod] + public async Task ItemDefinitionGroupWithPerItemOverride_OverrideWins() + { + // ItemDefinitionGroup sets PrivateAssets=all for all PackageReferences, + // but one specific PackageReference overrides it + var projectDir = Path.Combine(this.testDir, "DefGroupOverride"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0 + + + + all + + + + + + none + + + + """; + WriteFile(projectDir, "DefGroupOverride.csproj", content); + + var binlogPath = await BuildProjectAsync(projectDir, "DefGroupOverride.csproj"); + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("DefGroupOverride.csproj", StringComparison.OrdinalIgnoreCase)); + + // Newtonsoft.Json inherits PrivateAssets=all from ItemDefinitionGroup + var newtonsoftItem = projectInfo.PackageReference["Newtonsoft.Json"]; + newtonsoftItem.GetMetadata("PrivateAssets").Should().Be("all"); + + // System.Memory overrides to none + var memoryItem = projectInfo.PackageReference["System.Memory"]; + memoryItem.GetMetadata("PrivateAssets").Should().Be("none"); + } + + [TestMethod] + public async Task PropertyPrecedence_EnvVarOverriddenByProjectFile() + { + // Environment sets OutputType=Library, project file sets OutputType=Exe. + // MSBuild precedence: project file wins over env vars. + var projectDir = Path.Combine(this.testDir, "EnvVarPrecedence"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0 + Exe + + + + + + """; + WriteFile(projectDir, "EnvVarPrecedence.csproj", content); + WriteMinimalProgram(projectDir); + + var binlogPath = Path.Combine(projectDir, "build.binlog"); + await RunProcessWithEnvAsync( + projectDir, + "dotnet", + $"build EnvVarPrecedence.csproj -bl:\"{binlogPath}\" /p:UseAppHost=false", + ("OutputType", "Library")); + + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("EnvVarPrecedence.csproj", StringComparison.OrdinalIgnoreCase)); + + // Project file property wins over environment variable + projectInfo.OutputType.Should().Be("Exe"); + } + + [TestMethod] + public async Task PublishAotWithGlobalSelfContained_BothCaptured() + { + // Project has PublishAot=true, global property adds SelfContained=true + var projectDir = Path.Combine(this.testDir, "AotAndSC"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0 + Exe + true + + + + + + """; + WriteFile(projectDir, "AotAndSC.csproj", content); + WriteMinimalProgram(projectDir); + + var binlogPath = Path.Combine(projectDir, "build.binlog"); + await RunDotNetAsync(projectDir, $"msbuild AotAndSC.csproj -t:Restore -bl:\"{binlogPath}\" /p:SelfContained=true /p:RuntimeIdentifier=win-x64"); + + var results = this.processor.ExtractProjectInfo(binlogPath); + + var projectInfo = results.First(p => + p.ProjectPath != null && + p.ProjectPath.EndsWith("AotAndSC.csproj", StringComparison.OrdinalIgnoreCase)); + + projectInfo.PublishAot.Should().Be(true); + projectInfo.SelfContained.Should().Be(true); + } + + private static void WriteFile(string directory, string fileName, string content) + { + File.WriteAllText(Path.Combine(directory, fileName), content); + } + + private static void WriteMinimalProgram(string directory) + { + WriteFile(directory, "Program.cs", "System.Console.WriteLine();"); + } + + private static async Task BuildProjectAsync(string projectDir, string projectFile) + { + var binlogPath = Path.Combine(projectDir, "build.binlog"); + await RunDotNetAsync(projectDir, $"build \"{projectFile}\" -bl:\"{binlogPath}\" /p:UseAppHost=false"); + + if (!File.Exists(binlogPath)) + { + throw new InvalidOperationException($"Build did not produce binlog at: {binlogPath}"); + } + + return binlogPath; + } + + private static async Task RunDotNetAsync(string workingDirectory, string arguments) + { + // dotnet build already includes restore by default, no need for separate restore + await RunProcessAsync(workingDirectory, "dotnet", arguments); + } + + private static async Task RunProcessAsync(string workingDirectory, string fileName, string arguments) + { + var psi = new ProcessStartInfo + { + FileName = fileName, + Arguments = arguments, + WorkingDirectory = workingDirectory, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + + using var process = Process.Start(psi) + ?? throw new InvalidOperationException($"Failed to start: {fileName} {arguments}"); + + var stdout = await process.StandardOutput.ReadToEndAsync(); + var stderr = await process.StandardError.ReadToEndAsync(); + + await process.WaitForExitAsync(); + + if (process.ExitCode != 0) + { + throw new InvalidOperationException( + $"Process exited with code {process.ExitCode}.\nCommand: {fileName} {arguments}\nStdout:\n{stdout}\nStderr:\n{stderr}"); + } + } + + private static async Task RunProcessWithEnvAsync( + string workingDirectory, + string fileName, + string arguments, + params (string Key, string Value)[] envVars) + { + var psi = new ProcessStartInfo + { + FileName = fileName, + Arguments = arguments, + WorkingDirectory = workingDirectory, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + + foreach (var (key, value) in envVars) + { + psi.Environment[key] = value; + } + + using var process = Process.Start(psi) + ?? throw new InvalidOperationException($"Failed to start: {fileName} {arguments}"); + + var stdout = await process.StandardOutput.ReadToEndAsync(); + var stderr = await process.StandardError.ReadToEndAsync(); + + await process.WaitForExitAsync(); + + if (process.ExitCode != 0) + { + throw new InvalidOperationException( + $"Process exited with code {process.ExitCode}.\nCommand: {fileName} {arguments}\nStdout:\n{stdout}\nStderr:\n{stderr}"); + } + } +} diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs index f1626c1e0..9893d1f3c 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs @@ -1,19 +1,37 @@ namespace Microsoft.ComponentDetection.Detectors.Tests.NuGet; +using System.Collections.Generic; +using System.IO; using System.Linq; +using System.Reactive.Linq; +using System.Text; using System.Threading.Tasks; using AwesomeAssertions; +using Microsoft.Build.Framework; +using Microsoft.ComponentDetection.Common.DependencyGraph; using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Detectors.NuGet; using Microsoft.ComponentDetection.Detectors.Tests.Utilities; using Microsoft.ComponentDetection.TestsUtilities; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; +/// +/// Tests for . +/// Fallback tests use (public constructor, no binlog mock). +/// Binlog-enhanced tests construct the detector manually via the internal constructor with a mocked +/// . +/// [TestClass] public class MSBuildBinaryLogComponentDetectorTests : BaseDetectorTest { + private const string ProjectPath = @"C:\test\TestProject.csproj"; + private const string AssetsFilePath = @"C:\test\obj\project.assets.json"; + private const string BinlogFilePath = @"C:\test\build.binlog"; + private readonly Mock fileUtilityServiceMock; public MSBuildBinaryLogComponentDetectorTests() @@ -23,350 +41,763 @@ public MSBuildBinaryLogComponentDetectorTests() this.DetectorTestUtility.AddServiceMock(this.fileUtilityServiceMock); } + // ================================================================ + // Fallback tests – no binlog info available. + // Uses DetectorTestUtility / BaseDetectorTest (public constructor). + // ================================================================ [TestMethod] - public async Task ScanDirectoryAsync_WithSimpleAssetsFile_DetectsComponents() + public async Task Fallback_SimpleAssetsFile_DetectsComponents() { - var projectAssetsJson = @"{ - ""version"": 3, - ""targets"": { - ""net8.0"": { - ""Newtonsoft.Json/13.0.1"": { - ""type"": ""package"", - ""compile"": { - ""lib/net8.0/Newtonsoft.Json.dll"": {} - }, - ""runtime"": { - ""lib/net8.0/Newtonsoft.Json.dll"": {} - } - } - } - }, - ""libraries"": { - ""Newtonsoft.Json/13.0.1"": { - ""sha512"": ""test"", - ""type"": ""package"", - ""path"": ""newtonsoft.json/13.0.1"", - ""files"": [ - ""lib/net8.0/Newtonsoft.Json.dll"" - ] - } - }, - ""projectFileDependencyGroups"": { - ""net8.0"": [ - ""Newtonsoft.Json >= 13.0.1"" - ] - }, - ""packageFolders"": { - ""C:\\Users\\test\\.nuget\\packages\\"": {} - }, - ""project"": { - ""version"": ""1.0.0"", - ""restore"": { - ""projectName"": ""TestProject"", - ""projectPath"": ""C:\\test\\TestProject.csproj"", - ""outputPath"": ""C:\\test\\obj"" - }, - ""frameworks"": { - ""net8.0"": { - ""targetAlias"": ""net8.0"", - ""dependencies"": { - ""Newtonsoft.Json"": { - ""target"": ""Package"", - ""version"": ""[13.0.1, )"" - } - } - } - } - } - }"; + var assetsJson = SimpleAssetsJson("Newtonsoft.Json", "13.0.1"); - var (scanResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("project.assets.json", projectAssetsJson) + var (result, recorder) = await this.DetectorTestUtility + .WithFile("project.assets.json", assetsJson) .ExecuteDetectorAsync(); - scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); - var detectedComponents = componentRecorder.GetDetectedComponents(); - detectedComponents.Should().HaveCount(1); + result.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = recorder.GetDetectedComponents(); + components.Should().HaveCount(1); - var component = detectedComponents.First(); - var nugetComponent = component.Component as NuGetComponent; - nugetComponent.Should().NotBeNull(); - nugetComponent!.Name.Should().Be("Newtonsoft.Json"); - nugetComponent.Version.Should().Be("13.0.1"); + var nuget = (NuGetComponent)components.Single().Component; + nuget.Name.Should().Be("Newtonsoft.Json"); + nuget.Version.Should().Be("13.0.1"); } [TestMethod] - public async Task ScanDirectoryAsync_WithTransitiveDependencies_BuildsDependencyGraph() + public async Task Fallback_TransitiveDependencies_BuildsDependencyGraph() { - var projectAssetsJson = @"{ - ""version"": 3, - ""targets"": { - ""net8.0"": { - ""Microsoft.Extensions.Logging/8.0.0"": { - ""type"": ""package"", - ""dependencies"": { - ""Microsoft.Extensions.Logging.Abstractions"": ""8.0.0"" - }, - ""compile"": { - ""lib/net8.0/Microsoft.Extensions.Logging.dll"": {} - }, - ""runtime"": { - ""lib/net8.0/Microsoft.Extensions.Logging.dll"": {} - } - }, - ""Microsoft.Extensions.Logging.Abstractions/8.0.0"": { - ""type"": ""package"", - ""compile"": { - ""lib/net8.0/Microsoft.Extensions.Logging.Abstractions.dll"": {} - }, - ""runtime"": { - ""lib/net8.0/Microsoft.Extensions.Logging.Abstractions.dll"": {} - } - } - } - }, - ""libraries"": { - ""Microsoft.Extensions.Logging/8.0.0"": { - ""sha512"": ""test"", - ""type"": ""package"", - ""path"": ""microsoft.extensions.logging/8.0.0"", - ""files"": [ - ""lib/net8.0/Microsoft.Extensions.Logging.dll"" - ] - }, - ""Microsoft.Extensions.Logging.Abstractions/8.0.0"": { - ""sha512"": ""test"", - ""type"": ""package"", - ""path"": ""microsoft.extensions.logging.abstractions/8.0.0"", - ""files"": [ - ""lib/net8.0/Microsoft.Extensions.Logging.Abstractions.dll"" - ] - } - }, - ""projectFileDependencyGroups"": { - ""net8.0"": [ - ""Microsoft.Extensions.Logging >= 8.0.0"" - ] - }, - ""packageFolders"": { - ""C:\\Users\\test\\.nuget\\packages\\"": {} - }, - ""project"": { - ""version"": ""1.0.0"", - ""restore"": { - ""projectName"": ""TestProject"", - ""projectPath"": ""C:\\test\\TestProject.csproj"", - ""outputPath"": ""C:\\test\\obj"" - }, - ""frameworks"": { - ""net8.0"": { - ""targetAlias"": ""net8.0"", - ""dependencies"": { - ""Microsoft.Extensions.Logging"": { - ""target"": ""Package"", - ""version"": ""[8.0.0, )"" - } - } - } - } - } - }"; + var assetsJson = TransitiveAssetsJson(); - var (scanResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("project.assets.json", projectAssetsJson) + var (result, recorder) = await this.DetectorTestUtility + .WithFile("project.assets.json", assetsJson) .ExecuteDetectorAsync(); - scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); - var detectedComponents = componentRecorder.GetDetectedComponents(); - detectedComponents.Should().HaveCount(2); + result.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = recorder.GetDetectedComponents(); + components.Should().HaveCount(2); + + var graphs = recorder.GetDependencyGraphsByLocation(); + graphs.Should().NotBeEmpty(); + var graph = graphs.Values.First(); - var graphsByLocation = componentRecorder.GetDependencyGraphsByLocation(); - graphsByLocation.Should().NotBeEmpty(); + var logging = components.First(c => ((NuGetComponent)c.Component).Name == "Microsoft.Extensions.Logging"); + var abstractions = components.First(c => ((NuGetComponent)c.Component).Name == "Microsoft.Extensions.Logging.Abstractions"); + + graph.IsComponentExplicitlyReferenced(logging.Component.Id).Should().BeTrue(); + graph.IsComponentExplicitlyReferenced(abstractions.Component.Id).Should().BeFalse(); + graph.GetDependenciesForComponent(logging.Component.Id).Should().Contain(abstractions.Component.Id); + } - var graph = graphsByLocation.Values.First(); - var loggingComponent = detectedComponents.First(x => ((NuGetComponent)x.Component).Name == "Microsoft.Extensions.Logging"); - var abstractionsComponent = detectedComponents.First(x => ((NuGetComponent)x.Component).Name == "Microsoft.Extensions.Logging.Abstractions"); + [TestMethod] + public async Task Fallback_NoPackageSpec_HandlesGracefully() + { + var assetsJson = @"{ ""version"": 3, ""targets"": { ""net8.0"": {} }, ""libraries"": {}, ""packageFolders"": {} }"; - graph.IsComponentExplicitlyReferenced(loggingComponent.Component.Id).Should().BeTrue(); - graph.IsComponentExplicitlyReferenced(abstractionsComponent.Component.Id).Should().BeFalse(); + var (result, recorder) = await this.DetectorTestUtility + .WithFile("project.assets.json", assetsJson) + .ExecuteDetectorAsync(); - var dependencies = graph.GetDependenciesForComponent(loggingComponent.Component.Id); - dependencies.Should().Contain(abstractionsComponent.Component.Id); + result.ResultCode.Should().Be(ProcessingResultCode.Success); + recorder.GetDetectedComponents().Should().BeEmpty(); } [TestMethod] - public async Task ScanDirectoryAsync_WithNoPackageSpec_HandlesGracefully() + public async Task Fallback_ProjectReference_ExcludesProjectDependencies() { - var projectAssetsJson = @"{ - ""version"": 3, - ""targets"": { - ""net8.0"": {} - }, - ""libraries"": {}, - ""packageFolders"": {} - }"; + var assetsJson = ProjectReferenceAssetsJson(); + + var (result, recorder) = await this.DetectorTestUtility + .WithFile("project.assets.json", assetsJson) + .ExecuteDetectorAsync(); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = recorder.GetDetectedComponents(); + components.Should().HaveCount(1); + ((NuGetComponent)components.Single().Component).Name.Should().Be("Newtonsoft.Json"); + } + + [TestMethod] + public async Task Fallback_PackageDownload_RegisteredAsDevDependency() + { + var assetsJson = PackageDownloadAssetsJson(); - var (scanResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("project.assets.json", projectAssetsJson) + var (result, recorder) = await this.DetectorTestUtility + .WithFile("project.assets.json", assetsJson) .ExecuteDetectorAsync(); - scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); - var detectedComponents = componentRecorder.GetDetectedComponents(); - detectedComponents.Should().BeEmpty(); + result.ResultCode.Should().Be(ProcessingResultCode.Success); + var download = recorder.GetDetectedComponents() + .Single(c => ((NuGetComponent)c.Component).Name == "Microsoft.Net.Compilers.Toolset"); + recorder.GetEffectiveDevDependencyValue(download.Component.Id).Should().BeTrue(); + } + + // ================================================================ + // Binlog-enhanced tests – mock IBinLogProcessor injected via + // the internal constructor (accessible through InternalsVisibleTo). + // ================================================================ + [TestMethod] + public async Task WithBinlog_NormalProject_DetectsNuGetComponents() + { + var info = CreateProjectInfo(); + var assetsJson = SimpleAssetsJson("Newtonsoft.Json", "13.0.1"); + + var (result, recorder) = await ExecuteWithBinlogAsync([info], assetsJson); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + var nuget = recorder.GetDetectedComponents() + .Where(c => c.Component is NuGetComponent) + .ToList(); + nuget.Should().HaveCount(1); + ((NuGetComponent)nuget[0].Component).Name.Should().Be("Newtonsoft.Json"); + } + + [TestMethod] + public async Task WithBinlog_TestProject_AllDependenciesMarkedAsDev() + { + var info = CreateProjectInfo(); + info.IsTestProject = true; + var assetsJson = SimpleAssetsJson("Newtonsoft.Json", "13.0.1"); + + var (result, recorder) = await ExecuteWithBinlogAsync([info], assetsJson); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + var component = recorder.GetDetectedComponents().Single(c => c.Component is NuGetComponent); + recorder.GetEffectiveDevDependencyValue(component.Component.Id).Should().BeTrue(); + } + + [TestMethod] + public async Task WithBinlog_IsShippingFalse_AllDependenciesMarkedAsDev() + { + var info = CreateProjectInfo(); + info.IsShipping = false; + var assetsJson = SimpleAssetsJson("Newtonsoft.Json", "13.0.1"); + + var (result, recorder) = await ExecuteWithBinlogAsync([info], assetsJson); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + var component = recorder.GetDetectedComponents().Single(c => c.Component is NuGetComponent); + recorder.GetEffectiveDevDependencyValue(component.Component.Id).Should().BeTrue(); + } + + [TestMethod] + public async Task WithBinlog_IsDevelopmentTrue_AllDependenciesMarkedAsDev() + { + var info = CreateProjectInfo(); + info.IsDevelopment = true; + var assetsJson = SimpleAssetsJson("Newtonsoft.Json", "13.0.1"); + + var (result, recorder) = await ExecuteWithBinlogAsync([info], assetsJson); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + var component = recorder.GetDetectedComponents().Single(c => c.Component is NuGetComponent); + recorder.GetEffectiveDevDependencyValue(component.Component.Id).Should().BeTrue(); + } + + [TestMethod] + public async Task WithBinlog_ShippingProject_DependenciesNotMarkedAsDev() + { + // All dev-only flags are null or positive, so this is a normal shipping project + var info = CreateProjectInfo(); + var assetsJson = SimpleAssetsJson("Newtonsoft.Json", "13.0.1"); + + var (result, recorder) = await ExecuteWithBinlogAsync([info], assetsJson); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + var component = recorder.GetDetectedComponents().Single(c => c.Component is NuGetComponent); + recorder.GetEffectiveDevDependencyValue(component.Component.Id).Should().BeFalse(); + } + + [TestMethod] + public async Task WithBinlog_ExplicitDevDependencyTrue_OverridesPackageToDev() + { + var info = CreateProjectInfo(); + AddPackageReference(info, "Newtonsoft.Json", isDevelopmentDependency: true); + var assetsJson = SimpleAssetsJson("Newtonsoft.Json", "13.0.1"); + + var (result, recorder) = await ExecuteWithBinlogAsync([info], assetsJson); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + var component = recorder.GetDetectedComponents().Single(c => c.Component is NuGetComponent); + recorder.GetEffectiveDevDependencyValue(component.Component.Id).Should().BeTrue(); + } + + [TestMethod] + public async Task WithBinlog_ExplicitDevDependencyFalse_OverridesPackageToNotDev() + { + // On a normal project, IsDevelopmentDependency=false means "not dev", even when heuristics + // (framework component / autoReferenced) would otherwise classify it as dev. + var info = CreateProjectInfo(); + AddPackageReference(info, "Newtonsoft.Json", isDevelopmentDependency: false); + var assetsJson = SimpleAssetsJson("Newtonsoft.Json", "13.0.1"); + + var (result, recorder) = await ExecuteWithBinlogAsync([info], assetsJson); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + var component = recorder.GetDetectedComponents().Single(c => c.Component is NuGetComponent); + recorder.GetEffectiveDevDependencyValue(component.Component.Id).Should().BeFalse(); + } + + [TestMethod] + public async Task WithBinlog_TestProject_OverridesPerPackageFalse() + { + // Project-level classification (IsTestProject) always wins. + // IsDevelopmentDependency=false on a package is ignored because the project IS a test project. + var info = CreateProjectInfo(); + info.IsTestProject = true; + AddPackageReference(info, "Newtonsoft.Json", isDevelopmentDependency: false); + var assetsJson = SimpleAssetsJson("Newtonsoft.Json", "13.0.1"); + + var (result, recorder) = await ExecuteWithBinlogAsync([info], assetsJson); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + var component = recorder.GetDetectedComponents().Single(c => c.Component is NuGetComponent); + recorder.GetEffectiveDevDependencyValue(component.Component.Id).Should().BeTrue(); + } + + [TestMethod] + public async Task WithBinlog_TransitiveDepsOfDevPackage_InheritDevStatus() + { + // When a top-level package has IsDevelopmentDependency=true, the override callback + // (_ => true) applies transitively to the entire sub-graph. + var info = CreateProjectInfo(); + AddPackageReference(info, "Microsoft.Extensions.Logging", isDevelopmentDependency: true); + var assetsJson = TransitiveAssetsJson(); + + var (result, recorder) = await ExecuteWithBinlogAsync([info], assetsJson); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = recorder.GetDetectedComponents().Where(c => c.Component is NuGetComponent).ToList(); + components.Should().HaveCount(2); + + var abstractions = components.First(c => ((NuGetComponent)c.Component).Name == "Microsoft.Extensions.Logging.Abstractions"); + recorder.GetEffectiveDevDependencyValue(abstractions.Component.Id).Should().BeTrue(); + } + + [TestMethod] + public async Task WithBinlog_PackageDownload_DefaultIsDevDependency() + { + var info = CreateProjectInfo(); + var assetsJson = PackageDownloadAssetsJson(); + + var (result, recorder) = await ExecuteWithBinlogAsync([info], assetsJson); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + var download = recorder.GetDetectedComponents() + .First(c => ((NuGetComponent)c.Component).Name == "Microsoft.Net.Compilers.Toolset"); + recorder.GetEffectiveDevDependencyValue(download.Component.Id).Should().BeTrue(); + } + + [TestMethod] + public async Task WithBinlog_PackageDownload_ExplicitFalse_NotDevDependency() + { + var info = CreateProjectInfo(); + AddPackageDownload(info, "Microsoft.Net.Compilers.Toolset", isDevelopmentDependency: false); + var assetsJson = PackageDownloadAssetsJson(); + + var (result, recorder) = await ExecuteWithBinlogAsync([info], assetsJson); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + var download = recorder.GetDetectedComponents() + .First(c => ((NuGetComponent)c.Component).Name == "Microsoft.Net.Compilers.Toolset"); + recorder.GetEffectiveDevDependencyValue(download.Component.Id).Should().BeFalse(); + } + + [TestMethod] + public async Task WithBinlog_RegistersDotNetComponent() + { + var info = CreateProjectInfo(); + info.NETCoreSdkVersion = "8.0.100"; + info.OutputType = "Library"; + var assetsJson = SimpleAssetsJson("Newtonsoft.Json", "13.0.1"); + + var (result, recorder) = await ExecuteWithBinlogAsync([info], assetsJson); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + var dotNetComponents = recorder.GetDetectedComponents() + .Where(c => c.Component is DotNetComponent) + .ToList(); + dotNetComponents.Should().HaveCount(1); + + var dotNet = (DotNetComponent)dotNetComponents[0].Component; + dotNet.SdkVersion.Should().Be("8.0.100"); + dotNet.TargetFramework.Should().Be("net8.0"); + dotNet.ProjectType.Should().Be("library"); + } + + [TestMethod] + public async Task WithBinlog_SelfContained_RegistersApplicationSelfContained() + { + var info = CreateProjectInfo(); + info.NETCoreSdkVersion = "8.0.100"; + info.OutputType = "Exe"; + info.SelfContained = true; + var assetsJson = SimpleAssetsJson("Newtonsoft.Json", "13.0.1"); + + var (result, recorder) = await ExecuteWithBinlogAsync([info], assetsJson); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + var dotNet = recorder.GetDetectedComponents() + .Where(c => c.Component is DotNetComponent) + .Select(c => (DotNetComponent)c.Component) + .Single(); + dotNet.ProjectType.Should().Be("application-selfcontained"); + } + + [TestMethod] + public async Task WithBinlog_PublishAot_RegistersApplicationSelfContained() + { + var info = CreateProjectInfo(); + info.NETCoreSdkVersion = "8.0.100"; + info.OutputType = "Exe"; + info.PublishAot = true; + var assetsJson = SimpleAssetsJson("Newtonsoft.Json", "13.0.1"); + + var (result, recorder) = await ExecuteWithBinlogAsync([info], assetsJson); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + var dotNet = recorder.GetDetectedComponents() + .Where(c => c.Component is DotNetComponent) + .Select(c => (DotNetComponent)c.Component) + .Single(); + dotNet.ProjectType.Should().Be("application-selfcontained"); + } + + [TestMethod] + public async Task WithBinlog_LibrarySelfContained_RegistersLibrarySelfContained() + { + var info = CreateProjectInfo(); + info.NETCoreSdkVersion = "8.0.100"; + info.OutputType = "Library"; + info.SelfContained = true; + var assetsJson = SimpleAssetsJson("Newtonsoft.Json", "13.0.1"); + + var (result, recorder) = await ExecuteWithBinlogAsync([info], assetsJson); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + var dotNet = recorder.GetDetectedComponents() + .Where(c => c.Component is DotNetComponent) + .Select(c => (DotNetComponent)c.Component) + .Single(); + dotNet.ProjectType.Should().Be("library-selfcontained"); + } + + [TestMethod] + public async Task WithBinlog_NotSelfContained_RegistersPlainApplication() + { + var info = CreateProjectInfo(); + info.NETCoreSdkVersion = "8.0.100"; + info.OutputType = "Exe"; + info.SelfContained = false; + info.PublishAot = false; + var assetsJson = SimpleAssetsJson("Newtonsoft.Json", "13.0.1"); + + var (result, recorder) = await ExecuteWithBinlogAsync([info], assetsJson); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + var dotNet = recorder.GetDetectedComponents() + .Where(c => c.Component is DotNetComponent) + .Select(c => (DotNetComponent)c.Component) + .Single(); + dotNet.ProjectType.Should().Be("application"); + } + + [TestMethod] + public async Task WithBinlog_MultiTarget_PerTfmDevDependency() + { + // Multi-target project: net8.0 inner build marks package as dev, net6.0 does not. + var outer = CreateProjectInfo(targetFramework: null); + outer.TrySetProperty("TargetFrameworks", "net8.0;net6.0"); + + var innerNet8 = new MSBuildProjectInfo { ProjectPath = ProjectPath, ProjectAssetsFile = AssetsFilePath, TargetFramework = "net8.0" }; + AddPackageReference(innerNet8, "Newtonsoft.Json", isDevelopmentDependency: true); + + var innerNet6 = new MSBuildProjectInfo { ProjectPath = ProjectPath, ProjectAssetsFile = AssetsFilePath, TargetFramework = "net6.0" }; + + outer.InnerBuilds.Add(innerNet8); + outer.InnerBuilds.Add(innerNet6); + + var assetsJson = MultiTargetAssetsJson(); + + var (result, recorder) = await ExecuteWithBinlogAsync([outer], assetsJson); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + + // Package appears in both TFMs. + // net8.0: devDependencyOverride = true → registered as dev + // net6.0: devDependencyOverride = null → uses heuristic → normal package → NOT dev + // GetEffectiveDevDependencyValue ANDs across registrations → false wins + var component = recorder.GetDetectedComponents() + .Single(c => c.Component is NuGetComponent && ((NuGetComponent)c.Component).Name == "Newtonsoft.Json"); + recorder.GetEffectiveDevDependencyValue(component.Component.Id).Should().BeFalse(); } [TestMethod] - public async Task ScanDirectoryAsync_WithProjectReference_ExcludesProjectDependencies() + public async Task WithBinlog_NoBinlogMatch_FallsBackToStandardProcessing() { - var projectAssetsJson = @"{ - ""version"": 3, - ""targets"": { - ""net8.0"": { - ""Newtonsoft.Json/13.0.1"": { - ""type"": ""package"", - ""compile"": { - ""lib/net8.0/Newtonsoft.Json.dll"": {} - }, - ""runtime"": { - ""lib/net8.0/Newtonsoft.Json.dll"": {} + // Binlog contains info for a DIFFERENT project; assets file project path doesn't match. + var otherInfo = CreateProjectInfo( + projectPath: @"C:\other\OtherProject.csproj", + assetsFilePath: @"C:\other\obj\project.assets.json"); + otherInfo.IsTestProject = true; + + var assetsJson = SimpleAssetsJson("Newtonsoft.Json", "13.0.1"); + + var (result, recorder) = await ExecuteWithBinlogAsync([otherInfo], assetsJson); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + var component = recorder.GetDetectedComponents().Single(c => c.Component is NuGetComponent); + + // Falls back to standard processing – no dev-dependency override + recorder.GetEffectiveDevDependencyValue(component.Component.Id).Should().BeFalse(); + } + + // ================================================================ + // Helpers – project info construction + // ================================================================ + private static MSBuildProjectInfo CreateProjectInfo( + string projectPath = ProjectPath, + string assetsFilePath = AssetsFilePath, + string? targetFramework = "net8.0") + { + return new MSBuildProjectInfo + { + ProjectPath = projectPath, + ProjectAssetsFile = assetsFilePath, + TargetFramework = targetFramework, + }; + } + + private static void AddPackageReference(MSBuildProjectInfo info, string packageName, bool isDevelopmentDependency) + { + var item = CreateTaskItemMock(packageName, isDevelopmentDependency); + info.PackageReference[packageName] = item; + } + + private static void AddPackageDownload(MSBuildProjectInfo info, string packageName, bool isDevelopmentDependency) + { + var item = CreateTaskItemMock(packageName, isDevelopmentDependency); + info.PackageDownload[packageName] = item; + } + + private static ITaskItem CreateTaskItemMock(string itemSpec, bool isDevelopmentDependency) + { + var mock = new Mock(); + mock.SetupGet(x => x.ItemSpec).Returns(itemSpec); + mock.Setup(x => x.GetMetadata("IsDevelopmentDependency")) + .Returns(isDevelopmentDependency ? "true" : "false"); + return mock.Object; + } + + // ================================================================ + // Helpers – detector execution with mocked IBinLogProcessor + // ================================================================ + private static async Task<(IndividualDetectorScanResult Result, IComponentRecorder Recorder)> ExecuteWithBinlogAsync( + IReadOnlyList projectInfos, + string assetsJson, + string binlogPath = BinlogFilePath, + string assetsLocation = AssetsFilePath) + { + var binLogProcessorMock = new Mock(); + binLogProcessorMock + .Setup(x => x.ExtractProjectInfo(binlogPath)) + .Returns(projectInfos); + + var walkerMock = new Mock(); + var streamFactoryMock = new Mock(); + var fileUtilityMock = new Mock(); + fileUtilityMock.Setup(x => x.Exists(It.IsAny())).Returns(true); + var loggerMock = new Mock>(); + + var detector = new MSBuildBinaryLogComponentDetector( + streamFactoryMock.Object, + walkerMock.Object, + fileUtilityMock.Object, + binLogProcessorMock.Object, + loggerMock.Object); + + var recorder = new ComponentRecorder(); + + var requests = new[] + { + CreateProcessRequest(recorder, binlogPath, "fake-binlog-content"), + CreateProcessRequest(recorder, assetsLocation, assetsJson), + }; + + walkerMock + .Setup(x => x.GetFilteredComponentStreamObservable( + It.IsAny(), + It.IsAny>(), + It.IsAny())) + .Returns(requests.ToObservable()); + + var scanRequest = new ScanRequest( + new DirectoryInfo(Path.GetTempPath()), + null, + null, + new Dictionary(), + null, + recorder, + sourceFileRoot: new DirectoryInfo(Path.GetTempPath())); + + var result = await detector.ExecuteDetectorAsync(scanRequest); + return (result, recorder); + } + + private static ProcessRequest CreateProcessRequest(IComponentRecorder recorder, string location, string content) + { + var stream = new MemoryStream(Encoding.UTF8.GetBytes(content)); + var mockStream = new Mock(); + mockStream.SetupGet(x => x.Stream).Returns(stream); + mockStream.SetupGet(x => x.Location).Returns(location); + mockStream.SetupGet(x => x.Pattern).Returns(Path.GetFileName(location)); + + return new ProcessRequest + { + SingleFileComponentRecorder = recorder.CreateSingleFileComponentRecorder(location), + ComponentStream = mockStream.Object, + }; + } + + /// + /// Creates a minimal valid project.assets.json with a single package. + /// + private static string SimpleAssetsJson(string packageName, string version) => $$""" + { + "version": 3, + "targets": { + "net8.0": { + "{{packageName}}/{{version}}": { + "type": "package", + "compile": { "lib/net8.0/{{packageName}}.dll": {} }, + "runtime": { "lib/net8.0/{{packageName}}.dll": {} } + } + } + }, + "libraries": { + "{{packageName}}/{{version}}": { + "sha512": "fakehash", + "type": "package", + "path": "{{packageName.ToUpperInvariant()}}/{{version}}", + "files": [ "lib/net8.0/{{packageName}}.dll" ] + } + }, + "projectFileDependencyGroups": { + "net8.0": [ "{{packageName}} >= {{version}}" ] + }, + "packageFolders": { "C:\\Users\\test\\.nuget\\packages\\": {} }, + "project": { + "version": "1.0.0", + "restore": { + "projectName": "TestProject", + "projectPath": "C:\\test\\TestProject.csproj", + "outputPath": "C:\\test\\obj" + }, + "frameworks": { + "net8.0": { + "targetAlias": "net8.0", + "dependencies": { + "{{packageName}}": { "target": "Package", "version": "[{{version}}, )" } } + } + } + } + } + """; + + /// + /// Assets JSON with a top-level package that has a transitive dependency. + /// Microsoft.Extensions.Logging → Microsoft.Extensions.Logging.Abstractions. + /// + private static string TransitiveAssetsJson() => """ + { + "version": 3, + "targets": { + "net8.0": { + "Microsoft.Extensions.Logging/8.0.0": { + "type": "package", + "dependencies": { "Microsoft.Extensions.Logging.Abstractions": "8.0.0" }, + "compile": { "lib/net8.0/Microsoft.Extensions.Logging.dll": {} }, + "runtime": { "lib/net8.0/Microsoft.Extensions.Logging.dll": {} } }, - ""MyOtherProject/1.0.0"": { - ""type"": ""project"" + "Microsoft.Extensions.Logging.Abstractions/8.0.0": { + "type": "package", + "compile": { "lib/net8.0/Microsoft.Extensions.Logging.Abstractions.dll": {} }, + "runtime": { "lib/net8.0/Microsoft.Extensions.Logging.Abstractions.dll": {} } } } }, - ""libraries"": { - ""Newtonsoft.Json/13.0.1"": { - ""sha512"": ""test"", - ""type"": ""package"", - ""path"": ""newtonsoft.json/13.0.1"", - ""files"": [ - ""lib/net8.0/Newtonsoft.Json.dll"" - ] + "libraries": { + "Microsoft.Extensions.Logging/8.0.0": { + "sha512": "fakehash", "type": "package", + "path": "microsoft.extensions.logging/8.0.0", + "files": [ "lib/net8.0/Microsoft.Extensions.Logging.dll" ] }, - ""MyOtherProject/1.0.0"": { - ""type"": ""project"", - ""path"": ""../MyOtherProject/MyOtherProject.csproj"", - ""msbuildProject"": ""../MyOtherProject/MyOtherProject.csproj"" + "Microsoft.Extensions.Logging.Abstractions/8.0.0": { + "sha512": "fakehash", "type": "package", + "path": "microsoft.extensions.logging.abstractions/8.0.0", + "files": [ "lib/net8.0/Microsoft.Extensions.Logging.Abstractions.dll" ] } }, - ""projectFileDependencyGroups"": { - ""net8.0"": [ - ""Newtonsoft.Json >= 13.0.1"" - ] - }, - ""packageFolders"": { - ""C:\\Users\\test\\.nuget\\packages\\"": {} + "projectFileDependencyGroups": { + "net8.0": [ "Microsoft.Extensions.Logging >= 8.0.0" ] }, - ""project"": { - ""version"": ""1.0.0"", - ""restore"": { - ""projectName"": ""TestProject"", - ""projectPath"": ""C:\\test\\TestProject.csproj"", - ""outputPath"": ""C:\\test\\obj"" + "packageFolders": { "C:\\Users\\test\\.nuget\\packages\\": {} }, + "project": { + "version": "1.0.0", + "restore": { + "projectName": "TestProject", + "projectPath": "C:\\test\\TestProject.csproj", + "outputPath": "C:\\test\\obj" }, - ""frameworks"": { - ""net8.0"": { - ""targetAlias"": ""net8.0"", - ""dependencies"": { - ""Newtonsoft.Json"": { - ""target"": ""Package"", - ""version"": ""[13.0.1, )"" - } + "frameworks": { + "net8.0": { + "targetAlias": "net8.0", + "dependencies": { + "Microsoft.Extensions.Logging": { "target": "Package", "version": "[8.0.0, )" } } } } } - }"; - - var (scanResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("project.assets.json", projectAssetsJson) - .ExecuteDetectorAsync(); + } + """; - scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); - var detectedComponents = componentRecorder.GetDetectedComponents(); + /// + /// Assets JSON with a NuGet package and a project reference. + /// + private static string ProjectReferenceAssetsJson() => """ + { + "version": 3, + "targets": { + "net8.0": { + "Newtonsoft.Json/13.0.1": { + "type": "package", + "compile": { "lib/net8.0/Newtonsoft.Json.dll": {} }, + "runtime": { "lib/net8.0/Newtonsoft.Json.dll": {} } + }, + "MyOtherProject/1.0.0": { "type": "project" } + } + }, + "libraries": { + "Newtonsoft.Json/13.0.1": { + "sha512": "fakehash", "type": "package", + "path": "newtonsoft.json/13.0.1", + "files": [ "lib/net8.0/Newtonsoft.Json.dll" ] + }, + "MyOtherProject/1.0.0": { + "type": "project", + "path": "../MyOtherProject/MyOtherProject.csproj", + "msbuildProject": "../MyOtherProject/MyOtherProject.csproj" + } + }, + "projectFileDependencyGroups": { + "net8.0": [ "Newtonsoft.Json >= 13.0.1" ] + }, + "packageFolders": { "C:\\Users\\test\\.nuget\\packages\\": {} }, + "project": { + "version": "1.0.0", + "restore": { + "projectName": "TestProject", + "projectPath": "C:\\test\\TestProject.csproj", + "outputPath": "C:\\test\\obj" + }, + "frameworks": { + "net8.0": { + "targetAlias": "net8.0", + "dependencies": { + "Newtonsoft.Json": { "target": "Package", "version": "[13.0.1, )" } + } + } + } + } + } + """; - // Should only detect the NuGet package, not the project reference - detectedComponents.Should().HaveCount(1); - var component = detectedComponents.First().Component as NuGetComponent; - component!.Name.Should().Be("Newtonsoft.Json"); - } + /// + /// Assets JSON with a PackageDownload dependency. + /// + private static string PackageDownloadAssetsJson() => """ + { + "version": 3, + "targets": { "net8.0": {} }, + "libraries": {}, + "projectFileDependencyGroups": { "net8.0": [] }, + "packageFolders": { "C:\\Users\\test\\.nuget\\packages\\": {} }, + "project": { + "version": "1.0.0", + "restore": { + "projectName": "TestProject", + "projectPath": "C:\\test\\TestProject.csproj", + "outputPath": "C:\\test\\obj" + }, + "frameworks": { + "net8.0": { + "targetAlias": "net8.0", + "dependencies": {}, + "downloadDependencies": [ + { "name": "Microsoft.Net.Compilers.Toolset", "version": "[4.8.0, 4.8.0]" } + ] + } + } + } + } + """; - [TestMethod] - public async Task ScanDirectoryAsync_WithDevelopmentDependency_MarksAsDev() - { - // A package with only compile/build assets and no runtime assemblies should be marked as a dev dependency - var projectAssetsJson = @"{ - ""version"": 3, - ""targets"": { - ""net8.0"": { - ""StyleCop.Analyzers/1.2.0-beta.556"": { - ""type"": ""package"", - ""compile"": { - ""lib/netstandard2.0/_._"": {} - }, - ""runtime"": { - ""lib/netstandard2.0/_._"": {} - }, - ""build"": { - ""build/StyleCop.Analyzers.props"": {} - } + /// + /// Assets JSON with two target frameworks (net8.0 and net6.0), each containing the same package. + /// + private static string MultiTargetAssetsJson() => """ + { + "version": 3, + "targets": { + "net8.0": { + "Newtonsoft.Json/13.0.1": { + "type": "package", + "compile": { "lib/net8.0/Newtonsoft.Json.dll": {} }, + "runtime": { "lib/net8.0/Newtonsoft.Json.dll": {} } + } + }, + "net6.0": { + "Newtonsoft.Json/13.0.1": { + "type": "package", + "compile": { "lib/net6.0/Newtonsoft.Json.dll": {} }, + "runtime": { "lib/net6.0/Newtonsoft.Json.dll": {} } } } }, - ""libraries"": { - ""StyleCop.Analyzers/1.2.0-beta.556"": { - ""sha512"": ""test"", - ""type"": ""package"", - ""path"": ""stylecop.analyzers/1.2.0-beta.556"", - ""files"": [ - ""analyzers/dotnet/cs/StyleCop.Analyzers.dll"", - ""lib/netstandard2.0/_._"" - ] + "libraries": { + "Newtonsoft.Json/13.0.1": { + "sha512": "fakehash", "type": "package", + "path": "newtonsoft.json/13.0.1", + "files": [ "lib/net8.0/Newtonsoft.Json.dll", "lib/net6.0/Newtonsoft.Json.dll" ] } }, - ""projectFileDependencyGroups"": { - ""net8.0"": [ - ""StyleCop.Analyzers >= 1.2.0-beta.556"" - ] - }, - ""packageFolders"": { - ""C:\\Users\\test\\.nuget\\packages\\"": {} + "projectFileDependencyGroups": { + "net8.0": [ "Newtonsoft.Json >= 13.0.1" ], + "net6.0": [ "Newtonsoft.Json >= 13.0.1" ] }, - ""project"": { - ""version"": ""1.0.0"", - ""restore"": { - ""projectName"": ""TestProject"", - ""projectPath"": ""C:\\test\\TestProject.csproj"", - ""outputPath"": ""C:\\test\\obj"" + "packageFolders": { "C:\\Users\\test\\.nuget\\packages\\": {} }, + "project": { + "version": "1.0.0", + "restore": { + "projectName": "TestProject", + "projectPath": "C:\\test\\TestProject.csproj", + "outputPath": "C:\\test\\obj" }, - ""frameworks"": { - ""net8.0"": { - ""targetAlias"": ""net8.0"", - ""dependencies"": { - ""StyleCop.Analyzers"": { - ""target"": ""Package"", - ""version"": ""[1.2.0-beta.556, )"" - } + "frameworks": { + "net8.0": { + "targetAlias": "net8.0", + "dependencies": { + "Newtonsoft.Json": { "target": "Package", "version": "[13.0.1, )" } + } + }, + "net6.0": { + "targetAlias": "net6.0", + "dependencies": { + "Newtonsoft.Json": { "target": "Package", "version": "[13.0.1, )" } } } } } - }"; - - var (scanResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("project.assets.json", projectAssetsJson) - .ExecuteDetectorAsync(); - - scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); - var detectedComponents = componentRecorder.GetDetectedComponents(); - detectedComponents.Should().HaveCount(1); - - var component = detectedComponents.First(); - - // Analyzers are detected as development dependencies because they have analyzers in files - // but their runtime assets are placeholders - componentRecorder.GetEffectiveDevDependencyValue(component.Component.Id).Should().NotBeNull(); - } + } + """; } From d530ef1781ef219a1b2067d4995b42e5ceebb0e9 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Tue, 10 Mar 2026 14:27:26 -0700 Subject: [PATCH 04/26] Fix environment variable detection --- .../nuget/BinLogProcessor.cs | 39 +++++++++++++++---- .../nuget/MSBuildProjectInfo.cs | 7 ++++ 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs index 5b649f690..26a0e31df 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs @@ -43,9 +43,15 @@ public IReadOnlyList ExtractProjectInfo(string binlogPath) if (e?.BuildEventContext?.EvaluationId >= 0 && e is ProjectEvaluationFinishedEventArgs projectEvalArgs) { - var projectInfo = new MSBuildProjectInfo(); + // Reuse existing project info if one was created during evaluation + // (e.g., from EnvironmentVariableReadEventArgs or PropertyInitialValueSetEventArgs) + if (!projectInfoByEvaluationId.TryGetValue(e.BuildEventContext.EvaluationId, out var projectInfo)) + { + projectInfo = new MSBuildProjectInfo(); + projectInfoByEvaluationId[e.BuildEventContext.EvaluationId] = projectInfo; + } + this.PopulateFromEvaluation(projectEvalArgs, projectInfo); - projectInfoByEvaluationId[e.BuildEventContext.EvaluationId] = projectInfo; } }; @@ -263,6 +269,13 @@ private void HandleBuildEvent( projectInfo.TrySetProperty(propertyInitialValueSet.PropertyName, propertyInitialValueSet.PropertyValue); break; + // Environment variable reads during evaluation - MSBuild promotes env vars to properties + case EnvironmentVariableReadEventArgs envVarRead when + !string.IsNullOrEmpty(envVarRead.EnvironmentVariableName) && + MSBuildProjectInfo.IsPropertyOfInterest(envVarRead.EnvironmentVariableName): + projectInfo.TrySetProperty(envVarRead.EnvironmentVariableName, envVarRead.Message ?? string.Empty); + break; + // Task parameter events which can contain item arrays for add/remove/update case TaskParameterEventArgs taskParameter when MSBuildProjectInfo.IsItemTypeOfInterest(taskParameter.ItemType) && @@ -286,18 +299,28 @@ private bool TryGetProjectInfo( { projectInfo = null!; - if (args?.BuildEventContext?.ProjectInstanceId == null || args.BuildEventContext.ProjectInstanceId < 0) + // Try ProjectInstanceId first (available during build/target execution) + if (args?.BuildEventContext?.ProjectInstanceId >= 0 && + projectInstanceToEvaluationMap.TryGetValue(args.BuildEventContext.ProjectInstanceId, out var evaluationId) && + projectInfoByEvaluationId.TryGetValue(evaluationId, out projectInfo!)) { - return false; + return true; } - if (!projectInstanceToEvaluationMap.TryGetValue(args.BuildEventContext.ProjectInstanceId, out var evaluationId) || - !projectInfoByEvaluationId.TryGetValue(evaluationId, out projectInfo!)) + // Fall back to EvaluationId (available during evaluation, before ProjectStarted) + if (args?.BuildEventContext?.EvaluationId >= 0) { - return false; + if (!projectInfoByEvaluationId.TryGetValue(args.BuildEventContext.EvaluationId, out projectInfo!)) + { + // Create lazily for evaluation-time events that fire before ProjectEvaluationFinished + projectInfo = new MSBuildProjectInfo(); + projectInfoByEvaluationId[args.BuildEventContext.EvaluationId] = projectInfo; + } + + return true; } - return true; + return false; } /// diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildProjectInfo.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildProjectInfo.cs index 13d4d0ce7..25a7da6a9 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildProjectInfo.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildProjectInfo.cs @@ -148,6 +148,13 @@ internal class MSBuildProjectInfo /// True if the item type is of interest; otherwise, false. public static bool IsItemTypeOfInterest(string itemType) => ItemDictionaries.ContainsKey(itemType); + /// + /// Determines whether the specified property name is one that this class captures. + /// + /// The MSBuild property name. + /// True if the property is of interest; otherwise, false. + public static bool IsPropertyOfInterest(string propertyName) => PropertySetters.ContainsKey(propertyName); + /// /// Sets a property value if it is one of the properties of interest. /// From fb29c49ff6c1be53f6f2aca1fc0cba7aff9c1b0f Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Wed, 11 Mar 2026 08:30:25 -0700 Subject: [PATCH 05/26] Refactor dotnet/nuget/binlog detectors to share as much as possible --- .../dotnet/DotNetComponentDetector.cs | 331 +----------------- .../dotnet/DotNetProjectInfoProvider.cs | 325 +++++++++++++++++ .../nuget/LockFileUtilities.cs | 136 ++++++- .../MSBuildBinaryLogComponentDetector.cs | 157 +++------ ...ectModelProjectCentricComponentDetector.cs | 217 +----------- .../MSBuildBinaryLogComponentDetectorTests.cs | 27 +- 6 files changed, 545 insertions(+), 648 deletions(-) create mode 100644 src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetProjectInfoProvider.cs diff --git a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs index 871dba663..6d92ae056 100644 --- a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetComponentDetector.cs @@ -1,16 +1,8 @@ namespace Microsoft.ComponentDetection.Detectors.DotNet; -using System; -using System.Collections.Concurrent; using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.IO; -using System.Linq; -using System.Reflection.PortableExecutable; -using System.Text.Json; using System.Threading; using System.Threading.Tasks; -using global::NuGet.Frameworks; using global::NuGet.ProjectModel; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; @@ -19,18 +11,8 @@ namespace Microsoft.ComponentDetection.Detectors.DotNet; public class DotNetComponentDetector : FileComponentDetector { - private const string GlobalJsonFileName = "global.json"; - private readonly ICommandLineInvocationService commandLineInvocationService; - private readonly IDirectoryUtilityService directoryUtilityService; - private readonly IFileUtilityService fileUtilityService; - private readonly IPathUtilityService pathUtilityService; + private readonly DotNetProjectInfoProvider projectInfoProvider; private readonly LockFileFormat lockFileFormat = new(); - private readonly ConcurrentDictionary sdkVersionCache = []; - private readonly JsonDocumentOptions jsonDocumentOptions = - new() { CommentHandling = JsonCommentHandling.Skip, AllowTrailingCommas = true }; - - private string? sourceDirectory; - private string? sourceFileRootDirectory; public DotNetComponentDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, @@ -41,13 +23,15 @@ public DotNetComponentDetector( IObservableDirectoryWalkerFactory walkerFactory, ILogger logger) { - this.commandLineInvocationService = commandLineInvocationService; this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; - this.directoryUtilityService = directoryUtilityService; - this.fileUtilityService = fileUtilityService; - this.pathUtilityService = pathUtilityService; this.Scanner = walkerFactory; this.Logger = logger; + this.projectInfoProvider = new DotNetProjectInfoProvider( + commandLineInvocationService, + directoryUtilityService, + fileUtilityService, + pathUtilityService, + logger); } public override string Id => "DotNet"; @@ -60,93 +44,9 @@ public DotNetComponentDetector( public override IEnumerable Categories => ["DotNet"]; - private static string TrimAllEndingDirectorySeparators(string path) - { - string last; - - do - { - last = path; - path = Path.TrimEndingDirectorySeparator(last); - } - while (!ReferenceEquals(last, path)); - - return path; - } - - [return: NotNullIfNotNull(nameof(path))] - private string? NormalizeDirectory(string? path) => string.IsNullOrEmpty(path) ? path : TrimAllEndingDirectorySeparators(this.pathUtilityService.NormalizePath(path)); - - /// - /// Given a path under sourceDirectory, and the same path in another filesystem, - /// determine what path could be replaced with sourceDirectory. - /// - /// Some directory path under sourceDirectory, including sourceDirectory. - /// Path to the same directory as but in a different root. - /// Portion of that corresponds to root, or null if it can not be rebased. - private string? GetRootRebasePath(string sourceDirectoryBasedPath, string? rebasePath) - { - if (string.IsNullOrEmpty(rebasePath) || string.IsNullOrEmpty(this.sourceDirectory) || string.IsNullOrEmpty(sourceDirectoryBasedPath)) - { - return null; - } - - // sourceDirectory is normalized, normalize others - sourceDirectoryBasedPath = this.NormalizeDirectory(sourceDirectoryBasedPath); - rebasePath = this.NormalizeDirectory(rebasePath); - - // nothing to do if the paths are the same - if (rebasePath.Equals(sourceDirectoryBasedPath, StringComparison.Ordinal)) - { - return null; - } - - // find the relative path under sourceDirectory. - var sourceDirectoryRelativePath = this.NormalizeDirectory(Path.GetRelativePath(this.sourceDirectory!, sourceDirectoryBasedPath)); - - this.Logger.LogDebug("Attempting to rebase {RebasePath} to {SourceDirectoryBasedPath} using relative {SourceDirectoryRelativePath}", rebasePath, sourceDirectoryBasedPath, sourceDirectoryRelativePath); - - // if the rebase path has the same relative portion, then we have a replacement. - if (rebasePath.EndsWith(sourceDirectoryRelativePath)) - { - return rebasePath[..^sourceDirectoryRelativePath.Length]; - } - - // The path didn't have a common relative path, it might have been copied from a completely different location since it was built. - // We cannot rebase the paths. - return null; - } - - private async Task RunDotNetVersionAsync(string workingDirectoryPath, CancellationToken cancellationToken) - { - var workingDirectory = new DirectoryInfo(workingDirectoryPath); - - try - { - var process = await this.commandLineInvocationService.ExecuteCommandAsync("dotnet", ["dotnet.exe"], workingDirectory, cancellationToken, "--version").ConfigureAwait(false); - - if (process.ExitCode != 0) - { - // debug only - it could be that dotnet is not actually on the path and specified directly by the build scripts. - this.Logger.LogDebug("Failed to invoke 'dotnet --version'. Return: {Return} StdErr: {StdErr} StdOut: {StdOut}.", process.ExitCode, process.StdErr, process.StdOut); - return null; - } - - return process.StdOut.Trim(); - } - catch (InvalidOperationException ioe) - { - // debug only - it could be that dotnet is not actually on the path and specified directly by the build scripts. - this.Logger.LogDebug("Failed to invoke 'dotnet --version'. {Message}", ioe.Message); - return null; - } - } - public override Task ExecuteDetectorAsync(ScanRequest request, CancellationToken cancellationToken = default) { - this.sourceDirectory = this.NormalizeDirectory(request.SourceDirectory.FullName); - this.sourceFileRootDirectory = this.NormalizeDirectory(request.SourceFileRoot?.FullName); - + this.projectInfoProvider.Initialize(request.SourceDirectory.FullName, request.SourceFileRoot?.FullName); return base.ExecuteDetectorAsync(request, cancellationToken); } @@ -163,215 +63,10 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID return; } - var projectAssetsDirectory = this.pathUtilityService.GetParentDirectory(processRequest.ComponentStream.Location); - var projectPath = lockFile.PackageSpec.RestoreMetadata.ProjectPath; - var projectOutputPath = lockFile.PackageSpec.RestoreMetadata.OutputPath; - - // The output path should match the location that the assets file, if it doesn't we could be analyzing paths - // on a different filesystem root than they were created. - // Attempt to rebase paths based on the difference between this file's location and the output path. - var rebasePath = this.GetRootRebasePath(projectAssetsDirectory, projectOutputPath); - - if (rebasePath is not null) - { - projectPath = Path.Combine(this.sourceDirectory!, Path.GetRelativePath(rebasePath, projectPath)); - projectOutputPath = Path.Combine(this.sourceDirectory!, Path.GetRelativePath(rebasePath, projectOutputPath)); - } - - if (!this.fileUtilityService.Exists(projectPath)) - { - // Could be the assets file was not actually from this build - this.Logger.LogWarning("Project path {ProjectPath} specified by {ProjectAssetsPath} does not exist.", projectPath, processRequest.ComponentStream.Location); - } - - var projectDirectory = this.pathUtilityService.GetParentDirectory(projectPath); - var sdkVersion = await this.GetSdkVersionAsync(projectDirectory, cancellationToken); - - var projectName = lockFile.PackageSpec.RestoreMetadata.ProjectName; - - if (!this.directoryUtilityService.Exists(projectOutputPath)) - { - this.Logger.LogWarning("Project output path {ProjectOutputPath} specified by {ProjectAssetsPath} does not exist.", projectOutputPath, processRequest.ComponentStream.Location); - - // default to use the location of the assets file. - projectOutputPath = projectAssetsDirectory; - } - - var targetType = this.GetProjectType(projectOutputPath, projectName, cancellationToken); - - var componentReporter = this.ComponentRecorder.CreateSingleFileComponentRecorder(projectPath); - foreach (var target in lockFile.Targets ?? []) - { - var targetFramework = target.TargetFramework; - var isSelfContained = this.IsSelfContained(lockFile.PackageSpec, targetFramework, target); - var targetTypeWithSelfContained = this.GetTargetTypeWithSelfContained(targetType, isSelfContained); - - componentReporter.RegisterUsage(new DetectedComponent(new DotNetComponent(sdkVersion, targetFramework?.GetShortFolderName(), targetTypeWithSelfContained))); - } - } - - private string? GetProjectType(string projectOutputPath, string projectName, CancellationToken cancellationToken) - { - if (this.directoryUtilityService.Exists(projectOutputPath) && - projectName is not null && - projectName.IndexOfAny(Path.GetInvalidFileNameChars()) == -1) - { - try - { - // look for the compiled output, first as dll then as exe. - var candidates = this.directoryUtilityService.EnumerateFiles(projectOutputPath, projectName + ".dll", SearchOption.AllDirectories) - .Concat(this.directoryUtilityService.EnumerateFiles(projectOutputPath, projectName + ".exe", SearchOption.AllDirectories)); - foreach (var candidate in candidates) - { - try - { - return this.IsApplication(candidate) ? "application" : "library"; - } - catch (Exception e) - { - this.Logger.LogWarning("Failed to open output assembly {AssemblyPath} error {Message}.", candidate, e.Message); - } - } - } - catch (IOException e) - { - this.Logger.LogWarning("Failed to enumerate output directory {OutputPath} error {Message}.", projectOutputPath, e.Message); - } - } - - return null; - } - - private bool IsApplication(string assemblyPath) - { - using var peReader = new PEReader(this.fileUtilityService.MakeFileStream(assemblyPath)); - - // despite the name `IsExe` this is actually based of the CoffHeader Characteristics - return peReader.PEHeaders.IsExe; - } - - private bool IsSelfContained(PackageSpec packageSpec, NuGetFramework? targetFramework, LockFileTarget target) - { - // PublishAot projects reference Microsoft.DotNet.ILCompiler, which implies - // native AOT compilation and therefore a self-contained deployment. - if (target?.Libraries != null && - target.Libraries.Any(lib => "Microsoft.DotNet.ILCompiler".Equals(lib.Name, StringComparison.OrdinalIgnoreCase))) - { - return true; - } - - if (packageSpec?.TargetFrameworks == null || targetFramework == null) - { - return false; - } - - var targetFrameworkInfo = packageSpec.TargetFrameworks.FirstOrDefault(tf => tf.FrameworkName == targetFramework); - if (targetFrameworkInfo == null) - { - return false; - } - - var frameworkReferences = targetFrameworkInfo.FrameworkReferences; - var packageDownloads = targetFrameworkInfo.DownloadDependencies; - - if (frameworkReferences == null || frameworkReferences.Count == 0 || packageDownloads.IsDefaultOrEmpty) - { - return false; - } - - foreach (var frameworkRef in frameworkReferences) - { - var frameworkName = frameworkRef.Name; - var hasRuntimeDownload = packageDownloads.Any(pd => pd.Name.StartsWith($"{frameworkName}.Runtime", StringComparison.OrdinalIgnoreCase)); - - if (hasRuntimeDownload) - { - return true; - } - } - - return false; - } - - private string? GetTargetTypeWithSelfContained(string? targetType, bool isSelfContained) - { - if (string.IsNullOrWhiteSpace(targetType)) - { - return targetType; - } - - return isSelfContained ? $"{targetType}-selfcontained" : targetType; - } - - /// - /// Recursively get the sdk version from the project directory or parent directories. - /// - /// Directory to start the search. - /// Cancellation token to halt the search. - /// Sdk version found, or null if no version can be detected. - private async Task GetSdkVersionAsync(string? projectDirectory, CancellationToken cancellationToken) - { - // normalize since we need to use as a key - projectDirectory = this.NormalizeDirectory(projectDirectory); - - if (string.IsNullOrWhiteSpace(projectDirectory)) - { - // not expected - return null; - } - - if (this.sdkVersionCache.TryGetValue(projectDirectory, out var sdkVersion)) - { - return sdkVersion; - } - - var parentDirectory = this.pathUtilityService.GetParentDirectory(projectDirectory); - var globalJsonPath = Path.Combine(projectDirectory, GlobalJsonFileName); - - if (this.fileUtilityService.Exists(globalJsonPath)) - { - sdkVersion = await this.RunDotNetVersionAsync(projectDirectory, cancellationToken); - - if (string.IsNullOrWhiteSpace(sdkVersion)) - { - var globalJson = await JsonDocument.ParseAsync(this.fileUtilityService.MakeFileStream(globalJsonPath), cancellationToken: cancellationToken, options: this.jsonDocumentOptions).ConfigureAwait(false); - if (globalJson.RootElement.TryGetProperty("sdk", out var sdk)) - { - if (sdk.TryGetProperty("version", out var version)) - { - sdkVersion = version.GetString(); - } - } - } - - if (!string.IsNullOrWhiteSpace(sdkVersion)) - { - var globalJsonComponent = new DetectedComponent(new DotNetComponent(sdkVersion)); - var recorder = this.ComponentRecorder.CreateSingleFileComponentRecorder(globalJsonPath); - recorder.RegisterUsage(globalJsonComponent, isExplicitReferencedDependency: true); - return sdkVersion; - } - - // global.json may be malformed, or the sdk version may not be specified. - } - - if (projectDirectory.Equals(this.sourceDirectory, StringComparison.OrdinalIgnoreCase) || - projectDirectory.Equals(this.sourceFileRootDirectory, StringComparison.OrdinalIgnoreCase) || - string.IsNullOrEmpty(parentDirectory) || - projectDirectory.Equals(parentDirectory, StringComparison.OrdinalIgnoreCase)) - { - // if we are at the source directory, source file root, or have reached a root directory, run `dotnet --version` - // this could fail if dotnet is not on the path, or if the global.json is malformed - sdkVersion = await this.RunDotNetVersionAsync(projectDirectory, cancellationToken); - } - else - { - // recurse up the directory tree - sdkVersion = await this.GetSdkVersionAsync(parentDirectory, cancellationToken); - } - - this.sdkVersionCache[projectDirectory] = sdkVersion; - - return sdkVersion; + await this.projectInfoProvider.RegisterDotNetComponentsAsync( + lockFile, + processRequest.ComponentStream.Location, + this.ComponentRecorder, + cancellationToken); } } diff --git a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetProjectInfoProvider.cs b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetProjectInfoProvider.cs new file mode 100644 index 000000000..61836b726 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetProjectInfoProvider.cs @@ -0,0 +1,325 @@ +namespace Microsoft.ComponentDetection.Detectors.DotNet; + +using System; +using System.Collections.Concurrent; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using System.Reflection.PortableExecutable; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using global::NuGet.ProjectModel; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; +using NuGetLockFileUtilities = Microsoft.ComponentDetection.Detectors.NuGet.LockFileUtilities; + +/// +/// Resolves DotNet project and SDK information from the environment. +/// Handles SDK version resolution, project type detection, path rebasing, +/// and DotNet component registration. Used by both DotNetComponentDetector +/// and MSBuildBinaryLogComponentDetector. +/// +internal class DotNetProjectInfoProvider +{ + private const string GlobalJsonFileName = "global.json"; + + private readonly ICommandLineInvocationService commandLineInvocationService; + private readonly IDirectoryUtilityService directoryUtilityService; + private readonly IFileUtilityService fileUtilityService; + private readonly IPathUtilityService pathUtilityService; + private readonly ILogger logger; + private readonly ConcurrentDictionary sdkVersionCache = []; + private readonly JsonDocumentOptions jsonDocumentOptions = + new() { CommentHandling = JsonCommentHandling.Skip, AllowTrailingCommas = true }; + + private string? sourceDirectory; + private string? sourceFileRootDirectory; + + public DotNetProjectInfoProvider( + ICommandLineInvocationService commandLineInvocationService, + IDirectoryUtilityService directoryUtilityService, + IFileUtilityService fileUtilityService, + IPathUtilityService pathUtilityService, + ILogger logger) + { + this.commandLineInvocationService = commandLineInvocationService; + this.directoryUtilityService = directoryUtilityService; + this.fileUtilityService = fileUtilityService; + this.pathUtilityService = pathUtilityService; + this.logger = logger; + } + + /// + /// Initializes source directory paths for path rebasing. Call once per scan. + /// + public void Initialize(string? sourceDirectory, string? sourceFileRootDirectory) + { + this.sourceDirectory = this.NormalizeDirectory(sourceDirectory); + this.sourceFileRootDirectory = this.NormalizeDirectory(sourceFileRootDirectory); + } + + /// + /// Registers DotNet components from a lock file, determining SDK version and project type. + /// This is the complete DotNet component registration logic shared between DotNetComponentDetector + /// and MSBuildBinaryLogComponentDetector's fallback path. + /// + /// The lock file to analyze. + /// The location of the project.assets.json file. + /// The component recorder to register components with. + /// Cancellation token. + /// A task representing the asynchronous operation. + public async Task RegisterDotNetComponentsAsync( + LockFile lockFile, + string assetsFileLocation, + IComponentRecorder componentRecorder, + CancellationToken cancellationToken) + { + if (lockFile.PackageSpec?.RestoreMetadata is null) + { + return; + } + + var projectAssetsDirectory = this.pathUtilityService.GetParentDirectory(assetsFileLocation); + var projectPath = lockFile.PackageSpec.RestoreMetadata.ProjectPath; + var projectOutputPath = lockFile.PackageSpec.RestoreMetadata.OutputPath; + + // The output path should match the location of the assets file, if it doesn't we could be analyzing paths + // on a different filesystem root than they were created. + // Attempt to rebase paths based on the difference between this file's location and the output path. + var rebasePath = this.GetRootRebasePath(projectAssetsDirectory, projectOutputPath); + + if (rebasePath is not null) + { + projectPath = Path.Combine(this.sourceDirectory!, Path.GetRelativePath(rebasePath, projectPath)); + projectOutputPath = Path.Combine(this.sourceDirectory!, Path.GetRelativePath(rebasePath, projectOutputPath)); + } + + if (!this.fileUtilityService.Exists(projectPath)) + { + // Could be the assets file was not actually from this build + this.logger.LogWarning("Project path {ProjectPath} specified by {ProjectAssetsPath} does not exist.", projectPath, assetsFileLocation); + } + + var projectDirectory = this.pathUtilityService.GetParentDirectory(projectPath); + var sdkVersion = await this.GetSdkVersionAsync(projectDirectory, componentRecorder, cancellationToken); + + var projectName = lockFile.PackageSpec.RestoreMetadata.ProjectName; + + if (!this.directoryUtilityService.Exists(projectOutputPath)) + { + this.logger.LogWarning("Project output path {ProjectOutputPath} specified by {ProjectAssetsPath} does not exist.", projectOutputPath, assetsFileLocation); + + // default to use the location of the assets file. + projectOutputPath = projectAssetsDirectory; + } + + var targetType = this.GetProjectType(projectOutputPath, projectName); + + var singleFileComponentRecorder = componentRecorder.CreateSingleFileComponentRecorder(projectPath); + foreach (var target in lockFile.Targets ?? []) + { + var targetFramework = target.TargetFramework; + var isSelfContained = NuGetLockFileUtilities.IsSelfContainedFromLockFile(lockFile.PackageSpec, targetFramework, target); + var targetTypeWithSelfContained = NuGetLockFileUtilities.GetTargetTypeWithSelfContained(targetType, isSelfContained); + + singleFileComponentRecorder.RegisterUsage(new DetectedComponent(new DotNetComponent(sdkVersion, targetFramework?.GetShortFolderName(), targetTypeWithSelfContained))); + } + } + + private static string TrimAllEndingDirectorySeparators(string path) + { + string last; + + do + { + last = path; + path = Path.TrimEndingDirectorySeparator(last); + } + while (!ReferenceEquals(last, path)); + + return path; + } + + [return: NotNullIfNotNull(nameof(path))] + private string? NormalizeDirectory(string? path) => string.IsNullOrEmpty(path) ? path : TrimAllEndingDirectorySeparators(this.pathUtilityService.NormalizePath(path)); + + /// + /// Given a path under sourceDirectory, and the same path in another filesystem, + /// determine what path could be replaced with sourceDirectory. + /// + /// Some directory path under sourceDirectory, including sourceDirectory. + /// Path to the same directory as but in a different root. + /// Portion of that corresponds to root, or null if it can not be rebased. + internal string? GetRootRebasePath(string sourceDirectoryBasedPath, string? rebasePath) + { + if (string.IsNullOrEmpty(rebasePath) || string.IsNullOrEmpty(this.sourceDirectory) || string.IsNullOrEmpty(sourceDirectoryBasedPath)) + { + return null; + } + + // sourceDirectory is normalized, normalize others + sourceDirectoryBasedPath = this.NormalizeDirectory(sourceDirectoryBasedPath); + rebasePath = this.NormalizeDirectory(rebasePath); + + // nothing to do if the paths are the same + if (rebasePath.Equals(sourceDirectoryBasedPath, StringComparison.Ordinal)) + { + return null; + } + + // find the relative path under sourceDirectory. + var sourceDirectoryRelativePath = this.NormalizeDirectory(Path.GetRelativePath(this.sourceDirectory!, sourceDirectoryBasedPath)); + + this.logger.LogDebug("Attempting to rebase {RebasePath} to {SourceDirectoryBasedPath} using relative {SourceDirectoryRelativePath}", rebasePath, sourceDirectoryBasedPath, sourceDirectoryRelativePath); + + // if the rebase path has the same relative portion, then we have a replacement. + if (rebasePath.EndsWith(sourceDirectoryRelativePath)) + { + return rebasePath[..^sourceDirectoryRelativePath.Length]; + } + + // The path didn't have a common relative path, it might have been copied from a completely different location since it was built. + // We cannot rebase the paths. + return null; + } + + internal string? GetProjectType(string projectOutputPath, string projectName) + { + if (this.directoryUtilityService.Exists(projectOutputPath) && + projectName is not null && + projectName.IndexOfAny(Path.GetInvalidFileNameChars()) == -1) + { + try + { + // look for the compiled output, first as dll then as exe. + var candidates = this.directoryUtilityService.EnumerateFiles(projectOutputPath, projectName + ".dll", SearchOption.AllDirectories) + .Concat(this.directoryUtilityService.EnumerateFiles(projectOutputPath, projectName + ".exe", SearchOption.AllDirectories)); + foreach (var candidate in candidates) + { + try + { + return this.IsApplication(candidate) ? "application" : "library"; + } + catch (Exception e) + { + this.logger.LogWarning("Failed to open output assembly {AssemblyPath} error {Message}.", candidate, e.Message); + } + } + } + catch (IOException e) + { + this.logger.LogWarning("Failed to enumerate output directory {OutputPath} error {Message}.", projectOutputPath, e.Message); + } + } + + return null; + } + + private bool IsApplication(string assemblyPath) + { + using var peReader = new PEReader(this.fileUtilityService.MakeFileStream(assemblyPath)); + + // despite the name `IsExe` this is actually based of the CoffHeader Characteristics + return peReader.PEHeaders.IsExe; + } + + /// + /// Recursively get the sdk version from the project directory or parent directories. + /// + /// Directory to start the search. + /// Component recorder for registering global.json components. + /// Cancellation token to halt the search. + /// Sdk version found, or null if no version can be detected. + internal async Task GetSdkVersionAsync(string? projectDirectory, IComponentRecorder componentRecorder, CancellationToken cancellationToken) + { + // normalize since we need to use as a key + projectDirectory = this.NormalizeDirectory(projectDirectory); + + if (string.IsNullOrWhiteSpace(projectDirectory)) + { + // not expected + return null; + } + + if (this.sdkVersionCache.TryGetValue(projectDirectory, out var sdkVersion)) + { + return sdkVersion; + } + + var parentDirectory = this.pathUtilityService.GetParentDirectory(projectDirectory); + var globalJsonPath = Path.Combine(projectDirectory, GlobalJsonFileName); + + if (this.fileUtilityService.Exists(globalJsonPath)) + { + sdkVersion = await this.RunDotNetVersionAsync(projectDirectory, cancellationToken); + + if (string.IsNullOrWhiteSpace(sdkVersion)) + { + var globalJson = await JsonDocument.ParseAsync(this.fileUtilityService.MakeFileStream(globalJsonPath), cancellationToken: cancellationToken, options: this.jsonDocumentOptions).ConfigureAwait(false); + if (globalJson.RootElement.TryGetProperty("sdk", out var sdk)) + { + if (sdk.TryGetProperty("version", out var version)) + { + sdkVersion = version.GetString(); + } + } + } + + if (!string.IsNullOrWhiteSpace(sdkVersion)) + { + var globalJsonComponent = new DetectedComponent(new DotNetComponent(sdkVersion)); + var recorder = componentRecorder.CreateSingleFileComponentRecorder(globalJsonPath); + recorder.RegisterUsage(globalJsonComponent, isExplicitReferencedDependency: true); + return sdkVersion; + } + + // global.json may be malformed, or the sdk version may not be specified. + } + + if (projectDirectory.Equals(this.sourceDirectory, StringComparison.OrdinalIgnoreCase) || + projectDirectory.Equals(this.sourceFileRootDirectory, StringComparison.OrdinalIgnoreCase) || + string.IsNullOrEmpty(parentDirectory) || + projectDirectory.Equals(parentDirectory, StringComparison.OrdinalIgnoreCase)) + { + // if we are at the source directory, source file root, or have reached a root directory, run `dotnet --version` + // this could fail if dotnet is not on the path, or if the global.json is malformed + sdkVersion = await this.RunDotNetVersionAsync(projectDirectory, cancellationToken); + } + else + { + // recurse up the directory tree + sdkVersion = await this.GetSdkVersionAsync(parentDirectory, componentRecorder, cancellationToken); + } + + this.sdkVersionCache[projectDirectory] = sdkVersion; + + return sdkVersion; + } + + private async Task RunDotNetVersionAsync(string workingDirectoryPath, CancellationToken cancellationToken) + { + var workingDirectory = new DirectoryInfo(workingDirectoryPath); + + try + { + var process = await this.commandLineInvocationService.ExecuteCommandAsync("dotnet", ["dotnet.exe"], workingDirectory, cancellationToken, "--version").ConfigureAwait(false); + + if (process.ExitCode != 0) + { + // debug only - it could be that dotnet is not actually on the path and specified directly by the build scripts. + this.logger.LogDebug("Failed to invoke 'dotnet --version'. Return: {Return} StdErr: {StdErr} StdOut: {StdOut}.", process.ExitCode, process.StdErr, process.StdOut); + return null; + } + + return process.StdOut.Trim(); + } + catch (InvalidOperationException ioe) + { + // debug only - it could be that dotnet is not actually on the path and specified directly by the build scripts. + this.logger.LogDebug("Failed to invoke 'dotnet --version'. {Message}", ioe.Message); + return null; + } + } +} diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/LockFileUtilities.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/LockFileUtilities.cs index 433579277..7a682556e 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/LockFileUtilities.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/LockFileUtilities.cs @@ -12,6 +12,9 @@ namespace Microsoft.ComponentDetection.Detectors.NuGet; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.Extensions.Logging; +// LockFileUtilities also includes self-contained detection helpers shared by +// DotNetComponentDetector and MSBuildBinaryLogComponentDetector. + /// /// Shared utility methods for processing NuGet lock files (project.assets.json). /// Used by both NuGetProjectModelProjectCentricComponentDetector and MSBuildBinaryLogComponentDetector. @@ -176,7 +179,7 @@ static bool IsAPlaceholderItem(LockFileItem item) => { matchingLibrary = matchingLibraryNames.First(); var versionString = versionRange != null ? versionRange.ToNormalizedString() : version?.ToString(); - logger?.LogDebug( + logger?.LogWarning( "Couldn't satisfy lookup for {Version}. Falling back to first found component for {MatchingLibraryName}, resolving to version {MatchingLibraryVersion}.", versionString, matchingLibrary.Name, @@ -290,4 +293,135 @@ public static void RegisterPackageDownloads( } } } + + /// + /// Resolves the top-level (explicit) dependencies from a lock file into actual library entries + /// and builds the set of component IDs for those dependencies. + /// + /// The lock file to analyze. + /// Optional logger for warning messages. + /// A tuple of the resolved library list and their component ID set. + internal static (List Libraries, HashSet ComponentIds) ResolveExplicitDependencies( + LockFile lockFile, + ILogger? logger = null) + { + var libraries = new List(); + foreach (var lib in GetTopLevelLibraries(lockFile)) + { + var resolved = GetLibraryComponentWithDependencyLookup(lockFile.Libraries, lib.Name, lib.Version, lib.VersionRange, logger); + if (resolved != null) + { + libraries.Add(resolved); + } + else + { + logger?.LogWarning( + "Could not resolve top-level dependency {DependencyName}. The project.assets.json may be malformed.", + lib.Name); + } + } + + var componentIds = libraries + .Select(x => new NuGetComponent(x.Name, x.Version.ToNormalizedString()).Id) + .ToHashSet(); + + return (libraries, componentIds); + } + + /// + /// Processes a lock file using the standard project-centric approach: + /// resolves explicit dependencies, walks the dependency graph for each target, + /// and registers PackageDownload dependencies. + /// + /// The lock file to process. + /// The component recorder to register with. + /// Optional logger for warning messages. + internal static void ProcessLockFile( + LockFile lockFile, + ISingleFileComponentRecorder singleFileComponentRecorder, + ILogger? logger = null) + { + var (explicitReferencedDependencies, explicitlyReferencedComponentIds) = ResolveExplicitDependencies(lockFile, logger); + + foreach (var target in lockFile.Targets) + { + var frameworkReferences = GetFrameworkReferences(lockFile, target); + var frameworkPackages = FrameworkPackages.GetFrameworkPackages(target.TargetFramework, frameworkReferences, target); + + bool IsFrameworkOrDevelopmentDependency(LockFileTargetLibrary library) => + frameworkPackages.Any(fp => fp.IsAFrameworkComponent(library.Name, library.Version)) || + IsADevelopmentDependency(library, lockFile); + + foreach (var library in explicitReferencedDependencies.Select(x => target.GetTargetLibrary(x.Name)).Where(x => x != null)) + { + NavigateAndRegister(target, explicitlyReferencedComponentIds, singleFileComponentRecorder, library!, null, IsFrameworkOrDevelopmentDependency); + } + } + + RegisterPackageDownloads(singleFileComponentRecorder, lockFile); + } + + /// + /// Determines if a project is self-contained by inspecting the lock file. + /// Checks for Microsoft.DotNet.ILCompiler in target libraries (indicates PublishAot) + /// and runtime download dependencies matching framework references (indicates SelfContained). + /// + /// The package spec from the lock file. + /// The target framework to check. + /// The lock file target containing library information. + /// True if the project appears to be self-contained. + public static bool IsSelfContainedFromLockFile(PackageSpec? packageSpec, NuGetFramework? targetFramework, LockFileTarget target) + { + // PublishAot projects reference Microsoft.DotNet.ILCompiler + if (target?.Libraries != null && + target.Libraries.Any(lib => "Microsoft.DotNet.ILCompiler".Equals(lib.Name, StringComparison.OrdinalIgnoreCase))) + { + return true; + } + + if (packageSpec?.TargetFrameworks == null || targetFramework == null) + { + return false; + } + + var targetFrameworkInfo = packageSpec.TargetFrameworks.FirstOrDefault(tf => tf.FrameworkName == targetFramework); + if (targetFrameworkInfo == null) + { + return false; + } + + var frameworkReferences = targetFrameworkInfo.FrameworkReferences; + var packageDownloads = targetFrameworkInfo.DownloadDependencies; + + if (frameworkReferences == null || frameworkReferences.Count == 0 || packageDownloads.IsDefaultOrEmpty) + { + return false; + } + + foreach (var frameworkRef in frameworkReferences) + { + if (packageDownloads.Any(pd => pd.Name.StartsWith($"{frameworkRef.Name}.Runtime", StringComparison.OrdinalIgnoreCase))) + { + return true; + } + } + + return false; + } + + /// + /// Appends "-selfcontained" suffix to the project type when the project is self-contained. + /// + /// The base target type (e.g., "application" or "library"). + /// Whether the project is self-contained. + /// The target type with optional "-selfcontained" suffix. + public static string? GetTargetTypeWithSelfContained(string? targetType, bool isSelfContained) + { + if (string.IsNullOrWhiteSpace(targetType)) + { + return targetType; + } + + return isSelfContained ? $"{targetType}-selfcontained" : targetType; + } } diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs index 835e63c3c..026e9b827 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs @@ -15,6 +15,7 @@ namespace Microsoft.ComponentDetection.Detectors.NuGet; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.ComponentDetection.Detectors.DotNet; using Microsoft.Extensions.Logging; using Task = System.Threading.Tasks.Task; @@ -53,6 +54,7 @@ public class MSBuildBinaryLogComponentDetector : FileComponentDetector, IExperim { private readonly IBinLogProcessor binLogProcessor; private readonly IFileUtilityService fileUtilityService; + private readonly DotNetProjectInfoProvider projectInfoProvider; private readonly LockFileFormat lockFileFormat = new(); // Track which assets files have been processed to avoid duplicate processing @@ -69,12 +71,18 @@ public class MSBuildBinaryLogComponentDetector : FileComponentDetector, IExperim /// /// Factory for creating component streams. /// Factory for directory walking. + /// Service for command line invocation. + /// Service for directory operations. /// Service for file operations. + /// Service for path operations. /// Logger for diagnostic messages. public MSBuildBinaryLogComponentDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, IObservableDirectoryWalkerFactory walkerFactory, + ICommandLineInvocationService commandLineInvocationService, + IDirectoryUtilityService directoryUtilityService, IFileUtilityService fileUtilityService, + IPathUtilityService pathUtilityService, ILogger logger) { this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; @@ -82,6 +90,12 @@ public MSBuildBinaryLogComponentDetector( this.binLogProcessor = new BinLogProcessor(logger); this.fileUtilityService = fileUtilityService; this.Logger = logger; + this.projectInfoProvider = new DotNetProjectInfoProvider( + commandLineInvocationService, + directoryUtilityService, + fileUtilityService, + pathUtilityService, + logger); } /// @@ -91,7 +105,10 @@ public MSBuildBinaryLogComponentDetector( internal MSBuildBinaryLogComponentDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, IObservableDirectoryWalkerFactory walkerFactory, + ICommandLineInvocationService commandLineInvocationService, + IDirectoryUtilityService directoryUtilityService, IFileUtilityService fileUtilityService, + IPathUtilityService pathUtilityService, IBinLogProcessor binLogProcessor, ILogger logger) { @@ -100,6 +117,12 @@ internal MSBuildBinaryLogComponentDetector( this.binLogProcessor = binLogProcessor; this.fileUtilityService = fileUtilityService; this.Logger = logger; + this.projectInfoProvider = new DotNetProjectInfoProvider( + commandLineInvocationService, + directoryUtilityService, + fileUtilityService, + pathUtilityService, + logger); } /// @@ -117,6 +140,13 @@ internal MSBuildBinaryLogComponentDetector( /// public override int Version { get; } = 1; + /// + public override Task ExecuteDetectorAsync(ScanRequest request, CancellationToken cancellationToken = default) + { + this.projectInfoProvider.Initialize(request.SourceDirectory.FullName, request.SourceFileRoot?.FullName); + return base.ExecuteDetectorAsync(request, cancellationToken); + } + /// protected override async Task> OnPrepareDetectionAsync( IObservable processRequests, @@ -147,7 +177,7 @@ protected override async Task> OnPrepareDetectionAsy } /// - protected override Task OnFileFoundAsync( + protected override async Task OnFileFoundAsync( ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) @@ -160,10 +190,8 @@ protected override Task OnFileFoundAsync( } else if (processRequest.ComponentStream.Location.EndsWith("project.assets.json", StringComparison.OrdinalIgnoreCase)) { - this.ProcessAssetsFile(processRequest); + await this.ProcessAssetsFileAsync(processRequest, cancellationToken); } - - return Task.CompletedTask; } /// @@ -206,64 +234,6 @@ private static bool IsDevelopmentOnlyProject(MSBuildProjectInfo projectInfo) => private static bool IsSelfContainedFromProjectInfo(MSBuildProjectInfo projectInfo) => projectInfo.SelfContained == true || projectInfo.PublishAot == true; - /// - /// Determines if a project is self-contained by inspecting the lock file. - /// This is the same heuristic used by DotNetComponentDetector: - /// 1. Check for Microsoft.DotNet.ILCompiler in target libraries (indicates PublishAot). - /// 2. Check for runtime download dependencies matching framework references (indicates SelfContained). - /// - private static bool IsSelfContainedFromLockFile(PackageSpec? packageSpec, NuGetFramework? targetFramework, LockFileTarget target) - { - // PublishAot projects reference Microsoft.DotNet.ILCompiler - if (target.Libraries.Any(lib => "Microsoft.DotNet.ILCompiler".Equals(lib.Name, StringComparison.OrdinalIgnoreCase))) - { - return true; - } - - if (packageSpec?.TargetFrameworks == null || targetFramework == null) - { - return false; - } - - var targetFrameworkInfo = packageSpec.TargetFrameworks.FirstOrDefault(tf => tf.FrameworkName == targetFramework); - if (targetFrameworkInfo == null) - { - return false; - } - - var frameworkReferences = targetFrameworkInfo.FrameworkReferences; - var packageDownloads = targetFrameworkInfo.DownloadDependencies; - - if (frameworkReferences == null || frameworkReferences.Count == 0 || packageDownloads.IsDefaultOrEmpty) - { - return false; - } - - foreach (var frameworkRef in frameworkReferences) - { - if (packageDownloads.Any(pd => pd.Name.StartsWith($"{frameworkRef.Name}.Runtime", StringComparison.OrdinalIgnoreCase))) - { - return true; - } - } - - return false; - } - - /// - /// Appends "-selfcontained" suffix to the project type when the project is self-contained. - /// Matches DotNetComponentDetector's GetTargetTypeWithSelfContained behavior. - /// - private static string? GetTargetTypeWithSelfContained(string? targetType, bool isSelfContained) - { - if (string.IsNullOrWhiteSpace(targetType)) - { - return targetType; - } - - return isSelfContained ? $"{targetType}-selfcontained" : targetType; - } - private void ProcessBinlogFile(ProcessRequest processRequest) { var binlogPath = processRequest.ComponentStream.Location; @@ -390,8 +360,8 @@ private void RegisterDotNetComponent(MSBuildProjectInfo projectInfo, LockFile? l foreach (var target in lockFile.Targets) { var isSelfContained = isSelfContainedFromBinlog || - IsSelfContainedFromLockFile(lockFile.PackageSpec, target.TargetFramework, target); - var projectType = GetTargetTypeWithSelfContained(targetType, isSelfContained); + LockFileUtilities.IsSelfContainedFromLockFile(lockFile.PackageSpec, target.TargetFramework, target); + var projectType = LockFileUtilities.GetTargetTypeWithSelfContained(targetType, isSelfContained); var frameworkName = target.TargetFramework?.GetShortFolderName(); singleFileComponentRecorder.RegisterUsage( @@ -406,7 +376,7 @@ private void RegisterDotNetComponent(MSBuildProjectInfo projectInfo, LockFile? l } // Binlog-only path: no lock file available or no targets in lock file - var projectTypeFromBinlog = GetTargetTypeWithSelfContained(targetType, isSelfContainedFromBinlog); + var projectTypeFromBinlog = LockFileUtilities.GetTargetTypeWithSelfContained(targetType, isSelfContainedFromBinlog); // Get target frameworks from binlog properties var targetFrameworks = new List(); @@ -436,7 +406,7 @@ private void RegisterDotNetComponent(MSBuildProjectInfo projectInfo, LockFile? l } } - private void ProcessAssetsFile(ProcessRequest processRequest) + private async Task ProcessAssetsFileAsync(ProcessRequest processRequest, CancellationToken cancellationToken) { var assetsFilePath = processRequest.ComponentStream.Location; @@ -473,11 +443,11 @@ private void ProcessAssetsFile(ProcessRequest processRequest) else { // Fallback to standard processing without binlog info - // This matches NuGetProjectModelProjectCentricComponentDetector's behavior exactly + // This matches NuGetProjectModelProjectCentricComponentDetector + DotNetComponentDetector behavior this.Logger.LogDebug( "No binlog information found for assets file: {AssetsFile}. Using fallback processing.", assetsFilePath); - this.ProcessLockFileFallback(lockFile, assetsFilePath); + await this.ProcessLockFileFallbackAsync(lockFile, assetsFilePath, cancellationToken); } } catch (Exception ex) @@ -522,14 +492,7 @@ private void ProcessAssetsFile(ProcessRequest processRequest) /// private void ProcessLockFileWithProjectInfo(LockFile lockFile, MSBuildProjectInfo projectInfo) { - var explicitReferencedDependencies = LockFileUtilities.GetTopLevelLibraries(lockFile) - .Select(x => LockFileUtilities.GetLibraryComponentWithDependencyLookup(lockFile.Libraries, x.Name, x.Version, x.VersionRange, this.Logger)) - .Where(x => x != null) - .ToList(); - - var explicitlyReferencedComponentIds = explicitReferencedDependencies - .Select(x => new NuGetComponent(x!.Name, x.Version.ToNormalizedString()).Id) - .ToHashSet(); + var (explicitReferencedDependencies, explicitlyReferencedComponentIds) = LockFileUtilities.ResolveExplicitDependencies(lockFile, this.Logger); // Use project path from RestoreMetadata (consistent with NuGetProjectModelProjectCentricComponentDetector) var singleFileComponentRecorder = this.ComponentRecorder.CreateSingleFileComponentRecorder( @@ -565,7 +528,7 @@ bool IsFrameworkOrDevDependency(LockFileTargetLibrary library) => foreach (var dependency in explicitReferencedDependencies) { - var library = target.GetTargetLibrary(dependency!.Name); + var library = target.GetTargetLibrary(dependency.Name); if (library?.Name == null) { continue; @@ -639,44 +602,14 @@ private bool IsPackageDownloadDevDependency(string packageName, NuGetFramework? /// This method exactly matches NuGetProjectModelProjectCentricComponentDetector's behavior /// to ensure no loss of information when binlog data is not available. /// - private void ProcessLockFileFallback(LockFile lockFile, string location) + private async Task ProcessLockFileFallbackAsync(LockFile lockFile, string location, CancellationToken cancellationToken) { - var explicitReferencedDependencies = LockFileUtilities.GetTopLevelLibraries(lockFile) - .Select(x => LockFileUtilities.GetLibraryComponentWithDependencyLookup(lockFile.Libraries, x.Name, x.Version, x.VersionRange, this.Logger)) - .Where(x => x != null) - .ToList(); - - var explicitlyReferencedComponentIds = explicitReferencedDependencies - .Select(x => new NuGetComponent(x!.Name, x.Version.ToNormalizedString()).Id) - .ToHashSet(); - - // Use project path from RestoreMetadata (consistent with NuGetProjectModelProjectCentricComponentDetector) var projectPath = lockFile.PackageSpec?.RestoreMetadata?.ProjectPath ?? location; var singleFileComponentRecorder = this.ComponentRecorder.CreateSingleFileComponentRecorder(projectPath); + LockFileUtilities.ProcessLockFile(lockFile, singleFileComponentRecorder, this.Logger); - foreach (var target in lockFile.Targets) - { - var frameworkReferences = LockFileUtilities.GetFrameworkReferences(lockFile, target); - var frameworkPackages = FrameworkPackages.GetFrameworkPackages(target.TargetFramework, frameworkReferences, target); - - // Same logic as NuGetProjectModelProjectCentricComponentDetector.IsFrameworkOrDevelopmentDependency - bool IsFrameworkOrDevDependency(LockFileTargetLibrary library) => - frameworkPackages.Any(fp => fp.IsAFrameworkComponent(library.Name, library.Version)) || - LockFileUtilities.IsADevelopmentDependency(library, lockFile); - - foreach (var library in explicitReferencedDependencies.Select(x => target.GetTargetLibrary(x!.Name)).Where(x => x != null)) - { - LockFileUtilities.NavigateAndRegister( - target, - explicitlyReferencedComponentIds, - singleFileComponentRecorder, - library!, - null, - IsFrameworkOrDevDependency); - } - } - - // Register PackageDownload dependencies (same as NuGetProjectModelProjectCentricComponentDetector) - LockFileUtilities.RegisterPackageDownloads(singleFileComponentRecorder, lockFile); + // Register DotNet components (SDK version, target framework, project type) + // This matches DotNetComponentDetector's behavior for the fallback path + await this.projectInfoProvider.RegisterDotNetComponentsAsync(lockFile, location, this.ComponentRecorder, cancellationToken); } } diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetProjectModelProjectCentricComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetProjectModelProjectCentricComponentDetector.cs index 068283dd4..0d4e507b7 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetProjectModelProjectCentricComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetProjectModelProjectCentricComponentDetector.cs @@ -3,13 +3,9 @@ namespace Microsoft.ComponentDetection.Detectors.NuGet; using System; using System.Collections.Generic; -using System.IO; -using System.Linq; using System.Threading; using System.Threading.Tasks; -using global::NuGet.Packaging.Core; using global::NuGet.ProjectModel; -using global::NuGet.Versioning; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; @@ -17,8 +13,6 @@ namespace Microsoft.ComponentDetection.Detectors.NuGet; public class NuGetProjectModelProjectCentricComponentDetector : FileComponentDetector { - public const string ProjectDependencyType = "project"; - private readonly IFileUtilityService fileUtilityService; public NuGetProjectModelProjectCentricComponentDetector( @@ -43,46 +37,6 @@ public NuGetProjectModelProjectCentricComponentDetector( public override int Version { get; } = 2; - private static string[] GetFrameworkReferences(LockFile lockFile, LockFileTarget target) - { - var frameworkInformation = lockFile.PackageSpec.TargetFrameworks.FirstOrDefault(x => x.FrameworkName.Equals(target.TargetFramework)); - - if (frameworkInformation == null) - { - return []; - } - - // add directly referenced frameworks - var results = frameworkInformation.FrameworkReferences.Select(x => x.Name); - - // add transitive framework references - results = results.Concat(target.Libraries.SelectMany(l => l.FrameworkReferences)); - - return results.Distinct().ToArray(); - } - - private static bool IsADevelopmentDependency(LockFileTargetLibrary library, LockFile lockFile) - { - // a placeholder item is an empty file that doesn't exist with name _._ meant to indicate an empty folder in a nuget package, but also used by NuGet when a package's assets are excluded. - static bool IsAPlaceholderItem(LockFileItem item) => Path.GetFileName(item.Path).Equals(PackagingCoreConstants.EmptyFolder, StringComparison.OrdinalIgnoreCase); - - // All(IsAPlaceholderItem) checks if the collection is empty or all items are placeholders. - return library.RuntimeAssemblies.All(IsAPlaceholderItem) && - library.RuntimeTargets.All(IsAPlaceholderItem) && - library.ResourceAssemblies.All(IsAPlaceholderItem) && - library.NativeLibraries.All(IsAPlaceholderItem) && - library.ContentFiles.All(IsAPlaceholderItem) && - library.Build.All(IsAPlaceholderItem) && - library.BuildMultiTargeting.All(IsAPlaceholderItem) && - - // The SDK looks at the library for analyzers using the following hueristic: - // https://github.com/dotnet/sdk/blob/d7fe6e66d8f67dc93c5c294a75f42a2924889196/src/Tasks/Microsoft.NET.Build.Tasks/NuGetUtils.NuGet.cs#L43 - (!lockFile.GetLibrary(library.Name, library.Version)?.Files - .Any(file => file.StartsWith("analyzers", StringComparison.Ordinal) - && file.EndsWith(".dll", StringComparison.OrdinalIgnoreCase) - && !file.EndsWith(".resources.dll", StringComparison.OrdinalIgnoreCase)) ?? false); - } - protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) { try @@ -97,34 +51,9 @@ protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDiction return Task.CompletedTask; } - var explicitReferencedDependencies = this.GetTopLevelLibraries(lockFile) - .Select(x => this.GetLibraryComponentWithDependencyLookup(lockFile.Libraries, x.Name, x.Version, x.VersionRange)) - .ToList(); - var explicitlyReferencedComponentIds = - explicitReferencedDependencies - .Select(x => new NuGetComponent(x.Name, x.Version.ToNormalizedString()).Id) - .ToHashSet(); - // Since we report projects as the location, we ignore the passed in single file recorder. var singleFileComponentRecorder = this.ComponentRecorder.CreateSingleFileComponentRecorder(lockFile.PackageSpec.RestoreMetadata.ProjectPath); - foreach (var target in lockFile.Targets) - { - var frameworkReferences = GetFrameworkReferences(lockFile, target); - var frameworkPackages = FrameworkPackages.GetFrameworkPackages(target.TargetFramework, frameworkReferences, target); - bool IsFrameworkOrDevelopmentDependency(LockFileTargetLibrary library) => - frameworkPackages.Any(fp => fp.IsAFrameworkComponent(library.Name, library.Version)) || - IsADevelopmentDependency(library, lockFile); - - // This call to GetTargetLibrary is not guarded, because if this can't be resolved then something is fundamentally broken (e.g. an explicit dependency reference not being in the list of libraries) - // issue: we treat top level dependencies for all targets as top level for each target, but some may not be top level for other targets, or may not even be present for other targets. - foreach (var library in explicitReferencedDependencies.Select(x => target.GetTargetLibrary(x.Name)).Where(x => x != null)) - { - this.NavigateAndRegister(target, explicitlyReferencedComponentIds, singleFileComponentRecorder, library, null, IsFrameworkOrDevelopmentDependency); - } - } - - // Register PackageDownload - this.RegisterPackageDownloads(singleFileComponentRecorder, lockFile); + LockFileUtilities.ProcessLockFile(lockFile, singleFileComponentRecorder, this.Logger); } catch (Exception e) { @@ -134,148 +63,4 @@ bool IsFrameworkOrDevelopmentDependency(LockFileTargetLibrary library) => return Task.CompletedTask; } - - private void NavigateAndRegister( - LockFileTarget target, - HashSet explicitlyReferencedComponentIds, - ISingleFileComponentRecorder singleFileComponentRecorder, - LockFileTargetLibrary library, - string parentComponentId, - Func isDevelopmentDependency, - HashSet visited = null) - { - if (library.Type == ProjectDependencyType) - { - return; - } - - visited ??= []; - - var libraryComponent = new DetectedComponent(new NuGetComponent(library.Name, library.Version.ToNormalizedString())); - - // Possibly adding target framework to single file recorder - singleFileComponentRecorder.RegisterUsage( - libraryComponent, - explicitlyReferencedComponentIds.Contains(libraryComponent.Component.Id), - parentComponentId, - isDevelopmentDependency: isDevelopmentDependency(library), - targetFramework: target.TargetFramework?.GetShortFolderName()); - - foreach (var dependency in library.Dependencies) - { - if (visited.Contains(dependency.Id)) - { - continue; - } - - var targetLibrary = target.GetTargetLibrary(dependency.Id); - - // There are project.assets.json files that don't have a dependency library in the libraries set. - if (targetLibrary != null) - { - visited.Add(dependency.Id); - this.NavigateAndRegister(target, explicitlyReferencedComponentIds, singleFileComponentRecorder, targetLibrary, libraryComponent.Component.Id, isDevelopmentDependency, visited); - } - } - } - - private void RegisterPackageDownloads(ISingleFileComponentRecorder singleFileComponentRecorder, LockFile lockFile) - { - foreach (var framework in lockFile.PackageSpec.TargetFrameworks) - { - foreach (var packageDownload in framework.DownloadDependencies) - { - if (packageDownload?.Name is null || packageDownload?.VersionRange?.MinVersion is null) - { - continue; - } - - var libraryComponent = new DetectedComponent(new NuGetComponent(packageDownload.Name, packageDownload.VersionRange.MinVersion.ToNormalizedString())); - - // PackageDownload is always a development dependency since it's usage does not make it part of the application - singleFileComponentRecorder.RegisterUsage( - libraryComponent, - isExplicitReferencedDependency: true, - parentComponentId: null, - isDevelopmentDependency: true, - targetFramework: framework.FrameworkName?.GetShortFolderName()); - } - } - } - - private List<(string Name, Version Version, VersionRange VersionRange)> GetTopLevelLibraries(LockFile lockFile) - { - // First, populate libraries from the TargetFrameworks section -- This is the base level authoritative list of nuget packages a project has dependencies on. - var toBeFilled = new List<(string Name, Version Version, VersionRange VersionRange)>(); - - foreach (var framework in lockFile.PackageSpec.TargetFrameworks) - { - foreach (var dependency in framework.Dependencies) - { - toBeFilled.Add((dependency.Name, Version: null, dependency.LibraryRange.VersionRange)); - } - } - - // Next, we need to resolve project references -- This is a little funky, because project references are only stored via path in - // project.assets.json, so we first build a list of all paths and then compare what is top level to them to resolve their - // associated library. - var projectDirectory = Path.GetDirectoryName(lockFile.PackageSpec.RestoreMetadata.ProjectPath); - var librariesWithAbsolutePath = - lockFile.Libraries.Where(x => x.Type == ProjectDependencyType) - .Select(x => (library: x, absoluteProjectPath: Path.GetFullPath(Path.Combine(projectDirectory, x.Path)))) - .ToDictionary(x => x.absoluteProjectPath, x => x.library); - - foreach (var restoreMetadataTargetFramework in lockFile.PackageSpec.RestoreMetadata.TargetFrameworks) - { - foreach (var projectReference in restoreMetadataTargetFramework.ProjectReferences) - { - if (librariesWithAbsolutePath.TryGetValue(Path.GetFullPath(projectReference.ProjectPath), out var library)) - { - toBeFilled.Add((library.Name, library.Version.Version, null)); - } - } - } - - return toBeFilled; - } - - // Looks up a library in project.assets.json given a version (preferred) or version range (have to in some cases due to how project.assets.json stores things) - private LockFileLibrary GetLibraryComponentWithDependencyLookup(IList libraries, string dependencyId, Version version, VersionRange versionRange) - { - if ((version == null && versionRange == null) || (version != null && versionRange != null)) - { - throw new ArgumentException($"Either {nameof(version)} or {nameof(versionRange)} must be specified, but not both."); - } - - var matchingLibraryNames = libraries.Where(x => string.Equals(x.Name, dependencyId, StringComparison.OrdinalIgnoreCase)).ToList(); - - if (matchingLibraryNames.Count == 0) - { - throw new InvalidOperationException("Project.assets.json is malformed, no library could be found matching: " + dependencyId); - } - - LockFileLibrary matchingLibrary; - if (version != null) - { - // .Version.Version ensures we get to a nuget normalized 4 part version - matchingLibrary = matchingLibraryNames.FirstOrDefault(x => x.Version.Version.Equals(version)); - } - else - { - matchingLibrary = matchingLibraryNames.FirstOrDefault(x => versionRange.Satisfies(x.Version)); - } - - if (matchingLibrary == null) - { - matchingLibrary = matchingLibraryNames.First(); - var versionString = versionRange != null ? versionRange.ToNormalizedString() : version.ToString(); - this.Logger.LogWarning( - "Couldn't satisfy lookup for {Version}. Falling back to first found component for {MatchingLibraryName}, resolving to version {MatchingLibraryVersion}.", - versionString, - matchingLibrary.Name, - matchingLibrary.Version); - } - - return matchingLibrary; - } } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs index 9893d1f3c..f78669cc5 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs @@ -5,6 +5,7 @@ namespace Microsoft.ComponentDetection.Detectors.Tests.NuGet; using System.Linq; using System.Reactive.Linq; using System.Text; +using System.Threading; using System.Threading.Tasks; using AwesomeAssertions; using Microsoft.Build.Framework; @@ -32,13 +33,29 @@ public class MSBuildBinaryLogComponentDetectorTests : BaseDetectorTest commandLineInvocationServiceMock; private readonly Mock fileUtilityServiceMock; + private readonly Mock pathUtilityServiceMock; public MSBuildBinaryLogComponentDetectorTests() { + this.commandLineInvocationServiceMock = new Mock(); + this.commandLineInvocationServiceMock + .Setup(x => x.ExecuteCommandAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new CommandLineExecutionResult { ExitCode = 1 }); + this.fileUtilityServiceMock = new Mock(); this.fileUtilityServiceMock.Setup(x => x.Exists(It.IsAny())).Returns(true); - this.DetectorTestUtility.AddServiceMock(this.fileUtilityServiceMock); + this.fileUtilityServiceMock.Setup(x => x.Exists(It.Is(p => p.EndsWith("global.json")))).Returns(false); + + this.pathUtilityServiceMock = new Mock(); + this.pathUtilityServiceMock.Setup(x => x.NormalizePath(It.IsAny())).Returns(p => p); + this.pathUtilityServiceMock.Setup(x => x.GetParentDirectory(It.IsAny())).Returns(p => Path.GetDirectoryName(p) ?? string.Empty); + + this.DetectorTestUtility + .AddServiceMock(this.commandLineInvocationServiceMock) + .AddServiceMock(this.fileUtilityServiceMock) + .AddServiceMock(this.pathUtilityServiceMock); } // ================================================================ @@ -504,14 +521,22 @@ private static ITaskItem CreateTaskItemMock(string itemSpec, bool isDevelopmentD var walkerMock = new Mock(); var streamFactoryMock = new Mock(); + var commandLineInvocationMock = new Mock(); + var directoryUtilityMock = new Mock(); var fileUtilityMock = new Mock(); fileUtilityMock.Setup(x => x.Exists(It.IsAny())).Returns(true); + var pathUtilityMock = new Mock(); + pathUtilityMock.Setup(x => x.NormalizePath(It.IsAny())).Returns(p => p); + pathUtilityMock.Setup(x => x.GetParentDirectory(It.IsAny())).Returns(p => Path.GetDirectoryName(p) ?? string.Empty); var loggerMock = new Mock>(); var detector = new MSBuildBinaryLogComponentDetector( streamFactoryMock.Object, walkerMock.Object, + commandLineInvocationMock.Object, + directoryUtilityMock.Object, fileUtilityMock.Object, + pathUtilityMock.Object, binLogProcessorMock.Object, loggerMock.Object); From 9218aebf22d46b541ee0131c12b7ebb028bdc06b Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Wed, 11 Mar 2026 08:58:27 -0700 Subject: [PATCH 06/26] Fix testing of mocks against internal interfaces --- .../Microsoft.ComponentDetection.Detectors.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj b/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj index b152f4f3c..02f8d02ea 100644 --- a/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj +++ b/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj @@ -5,6 +5,7 @@ + From ad75ea811afe799d02c48a203d1eed0eb637f000 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Wed, 11 Mar 2026 11:08:33 -0700 Subject: [PATCH 07/26] Fix a couple failing tests --- .../nuget/MSBuildBinaryLogComponentDetectorTests.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs index f78669cc5..3e81cc91f 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs @@ -72,7 +72,7 @@ public async Task Fallback_SimpleAssetsFile_DetectsComponents() .ExecuteDetectorAsync(); result.ResultCode.Should().Be(ProcessingResultCode.Success); - var components = recorder.GetDetectedComponents(); + var components = recorder.GetDetectedComponents().Where(c => c.Component is NuGetComponent).ToList(); components.Should().HaveCount(1); var nuget = (NuGetComponent)components.Single().Component; @@ -90,7 +90,7 @@ public async Task Fallback_TransitiveDependencies_BuildsDependencyGraph() .ExecuteDetectorAsync(); result.ResultCode.Should().Be(ProcessingResultCode.Success); - var components = recorder.GetDetectedComponents(); + var components = recorder.GetDetectedComponents().Where(c => c.Component is NuGetComponent).ToList(); components.Should().HaveCount(2); var graphs = recorder.GetDependencyGraphsByLocation(); @@ -128,7 +128,7 @@ public async Task Fallback_ProjectReference_ExcludesProjectDependencies() .ExecuteDetectorAsync(); result.ResultCode.Should().Be(ProcessingResultCode.Success); - var components = recorder.GetDetectedComponents(); + var components = recorder.GetDetectedComponents().Where(c => c.Component is NuGetComponent).ToList(); components.Should().HaveCount(1); ((NuGetComponent)components.Single().Component).Name.Should().Be("Newtonsoft.Json"); } @@ -144,6 +144,7 @@ public async Task Fallback_PackageDownload_RegisteredAsDevDependency() result.ResultCode.Should().Be(ProcessingResultCode.Success); var download = recorder.GetDetectedComponents() + .Where(c => c.Component is NuGetComponent) .Single(c => ((NuGetComponent)c.Component).Name == "Microsoft.Net.Compilers.Toolset"); recorder.GetEffectiveDevDependencyValue(download.Component.Id).Should().BeTrue(); } From a2fa655d7c7ebbd6edeef671aa3ce7aa145507dd Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Wed, 11 Mar 2026 13:19:30 -0700 Subject: [PATCH 08/26] Address feedback --- .../nuget/BinLogProcessor.cs | 14 ++++++-- .../MSBuildBinaryLogComponentDetector.cs | 27 ++++++++++----- .../nuget/MSBuildProjectInfo.cs | 18 +++++++--- .../Configs/MSBuildBinaryLogExperiment.cs | 6 ++-- .../nuget/BinLogProcessorTests.cs | 34 +++++++++++++++++-- 5 files changed, 79 insertions(+), 20 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs index 26a0e31df..9369f1fcf 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs @@ -135,8 +135,18 @@ private void AddOrMergeProjectInfo( } else if (existing.IsOuterBuild && !projectInfo.IsOuterBuild && !string.IsNullOrEmpty(projectInfo.TargetFramework)) { - // Existing is outer, new is inner - add new as inner build - existing.InnerBuilds.Add(projectInfo); + // Existing is outer, new is inner - de-duplicate by TargetFramework + var matchingInner = existing.InnerBuilds.FirstOrDefault( + ib => string.Equals(ib.TargetFramework, projectInfo.TargetFramework, StringComparison.OrdinalIgnoreCase)); + if (matchingInner != null) + { + // Same TFM seen again (e.g., build + publish pass) - merge + matchingInner.MergeWith(projectInfo); + } + else + { + existing.InnerBuilds.Add(projectInfo); + } } else if (!existing.IsOuterBuild && !projectInfo.IsOuterBuild && !string.IsNullOrEmpty(projectInfo.TargetFramework)) { diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs index 026e9b827..8f439d7d8 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs @@ -283,17 +283,25 @@ private void ProcessBinlogFile(ProcessRequest processRequest) private void IndexProjectInfo(MSBuildProjectInfo projectInfo, List assetsFilesFound) { - // Store the project info for later use when processing assets files + // Store the project info for later use when processing assets files. + // Use AddOrUpdate+MergeWith so that multiple binlogs for the same project + // (e.g., build and publish passes) form a superset rather than keeping only the first. var projectPath = projectInfo.ProjectPath; if (!string.IsNullOrEmpty(projectPath)) { - this.projectInfoByProjectPath.TryAdd(projectPath, projectInfo); + this.projectInfoByProjectPath.AddOrUpdate( + projectPath, + _ => projectInfo, + (_, existing) => existing.MergeWith(projectInfo)); } // Also index by assets file path for direct lookup if (!string.IsNullOrEmpty(projectInfo.ProjectAssetsFile)) { - this.projectInfoByAssetsFile.TryAdd(projectInfo.ProjectAssetsFile, projectInfo); + this.projectInfoByAssetsFile.AddOrUpdate( + projectInfo.ProjectAssetsFile, + _ => projectInfo, + (_, existing) => existing.MergeWith(projectInfo)); assetsFilesFound.Add(projectInfo.ProjectAssetsFile); } } @@ -438,7 +446,7 @@ private async Task ProcessAssetsFileAsync(ProcessRequest processRequest, Cancell if (projectInfo != null) { // We have binlog info, use enhanced processing - this.ProcessLockFileWithProjectInfo(lockFile, projectInfo); + this.ProcessLockFileWithProjectInfo(lockFile, projectInfo, assetsFilePath); } else { @@ -490,13 +498,16 @@ private async Task ProcessAssetsFileAsync(ProcessRequest processRequest, Cancell /// all dependencies are marked as development dependencies. /// - Per-package IsDevelopmentDependency metadata overrides are applied transitively. /// - private void ProcessLockFileWithProjectInfo(LockFile lockFile, MSBuildProjectInfo projectInfo) + private void ProcessLockFileWithProjectInfo(LockFile lockFile, MSBuildProjectInfo projectInfo, string assetsFilePath) { var (explicitReferencedDependencies, explicitlyReferencedComponentIds) = LockFileUtilities.ResolveExplicitDependencies(lockFile, this.Logger); - // Use project path from RestoreMetadata (consistent with NuGetProjectModelProjectCentricComponentDetector) - var singleFileComponentRecorder = this.ComponentRecorder.CreateSingleFileComponentRecorder( - lockFile.PackageSpec?.RestoreMetadata?.ProjectPath ?? projectInfo.ProjectPath ?? string.Empty); + // Use project path from RestoreMetadata (consistent with NuGetProjectModelProjectCentricComponentDetector). + // If no project path is available, fall back to the assets file path to avoid collisions. + var recorderLocation = lockFile.PackageSpec?.RestoreMetadata?.ProjectPath + ?? projectInfo.ProjectPath + ?? assetsFilePath; + var singleFileComponentRecorder = this.ComponentRecorder.CreateSingleFileComponentRecorder(recorderLocation); // Get the project info for the target framework (use inner build if available) MSBuildProjectInfo GetProjectInfoForTarget(LockFileTarget target) diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildProjectInfo.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildProjectInfo.cs index 25a7da6a9..d368bf752 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildProjectInfo.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildProjectInfo.cs @@ -209,14 +209,19 @@ public bool TryRemoveItem(string itemType, string itemSpec) /// /// Merges another project info into this one, forming a superset. - /// Properties from override null values in this instance. - /// Boolean properties use logical OR (true wins). + /// This is used when the same project is seen multiple times (e.g., build + publish passes). + /// In practice, property values are not expected to differ across passes for the same project + /// and target framework — the merge fills in any values that were not set rather than overriding. + /// Boolean properties use logical OR (any true value is sufficient to classify the project). /// Items from are added if not already present. /// /// The other project info to merge from. - public void MergeWith(MSBuildProjectInfo other) + /// This instance for fluent chaining. + public MSBuildProjectInfo MergeWith(MSBuildProjectInfo other) { - // Merge boolean properties: true wins (if either says true, result is true) + // Merge boolean properties: true wins. For all classification booleans (IsTestProject, + // IsDevelopment, IsShipping, etc.), if any pass reports true it is sufficient to classify + // the project accordingly. These values are not expected to differ across passes. this.IsDevelopment = MergeBool(this.IsDevelopment, other.IsDevelopment); this.IsPackable = MergeBool(this.IsPackable, other.IsPackable); this.IsShipping = MergeBool(this.IsShipping, other.IsShipping); @@ -224,7 +229,8 @@ public void MergeWith(MSBuildProjectInfo other) this.PublishAot = MergeBool(this.PublishAot, other.PublishAot); this.SelfContained = MergeBool(this.SelfContained, other.SelfContained); - // Merge string properties: prefer non-null/non-empty + // Merge string properties: fill in unset values only. + // These are not expected to differ across passes for the same project/TFM. this.OutputType ??= other.OutputType; this.NETCoreSdkVersion ??= other.NETCoreSdkVersion; this.ProjectAssetsFile ??= other.ProjectAssetsFile; @@ -234,6 +240,8 @@ public void MergeWith(MSBuildProjectInfo other) // Merge items: add items from other that are not already present MergeItems(this.PackageReference, other.PackageReference); MergeItems(this.PackageDownload, other.PackageDownload); + + return this; } private static bool? MergeBool(bool? existing, bool? incoming) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/MSBuildBinaryLogExperiment.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/MSBuildBinaryLogExperiment.cs index 6b11f5611..e5f19ba4e 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/MSBuildBinaryLogExperiment.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/MSBuildBinaryLogExperiment.cs @@ -1,11 +1,13 @@ namespace Microsoft.ComponentDetection.Orchestrator.Experiments.Configs; using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Detectors.DotNet; using Microsoft.ComponentDetection.Detectors.NuGet; /// /// Experiment configuration for validating the -/// against the existing . +/// against the existing and +/// . /// public class MSBuildBinaryLogExperiment : IExperimentConfiguration { @@ -14,7 +16,7 @@ public class MSBuildBinaryLogExperiment : IExperimentConfiguration /// public bool IsInControlGroup(IComponentDetector componentDetector) => - componentDetector is NuGetProjectModelProjectCentricComponentDetector; + componentDetector is NuGetProjectModelProjectCentricComponentDetector or DotNetComponentDetector; /// public bool IsInExperimentGroup(IComponentDetector componentDetector) => diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs index b7e6c2813..493e2307a 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs @@ -29,9 +29,37 @@ public void TestInitialize() this.testDir = Path.Combine(Path.GetTempPath(), "BinLogProcessorTests", Guid.NewGuid().ToString("N")); Directory.CreateDirectory(this.testDir); - // Pin the SDK version so temp projects use the same SDK as the workspace - var globalJson = """{ "sdk": { "version": "8.0.100", "rollForward": "latestFeature" } }"""; - WriteFile(this.testDir, "global.json", globalJson); + // Copy the workspace global.json so temp projects use the same SDK as the repo. + // This avoids hardcoding a version that may not be installed in CI/dev machines. + var workspaceGlobalJson = FindWorkspaceGlobalJsonPath(); + if (workspaceGlobalJson != null) + { + File.Copy(workspaceGlobalJson, Path.Combine(this.testDir, "global.json")); + } + } + + private static string? FindWorkspaceGlobalJsonPath() + { + var currentDirectory = Directory.GetCurrentDirectory(); + + while (!string.IsNullOrEmpty(currentDirectory)) + { + var candidate = Path.Combine(currentDirectory, "global.json"); + if (File.Exists(candidate)) + { + return candidate; + } + + var parent = Directory.GetParent(currentDirectory)?.FullName; + if (parent == currentDirectory) + { + break; + } + + currentDirectory = parent; + } + + return null; } [TestCleanup] From 33b0533668d1e6d9168d7d822771427e93569838 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Wed, 11 Mar 2026 13:52:09 -0700 Subject: [PATCH 09/26] Reduce state to just assets to project info map --- .../MSBuildBinaryLogComponentDetector.cs | 76 ++----- .../MSBuildBinaryLogComponentDetectorTests.cs | 201 ++++++++++++++++++ 2 files changed, 223 insertions(+), 54 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs index 8f439d7d8..c307dff86 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs @@ -57,13 +57,23 @@ public class MSBuildBinaryLogComponentDetector : FileComponentDetector, IExperim private readonly DotNetProjectInfoProvider projectInfoProvider; private readonly LockFileFormat lockFileFormat = new(); - // Track which assets files have been processed to avoid duplicate processing - private readonly ConcurrentDictionary processedAssetsFiles = new(StringComparer.OrdinalIgnoreCase); - - // Store project information extracted from binlogs keyed by project path - private readonly ConcurrentDictionary projectInfoByProjectPath = new(StringComparer.OrdinalIgnoreCase); - - // Store project information extracted from binlogs keyed by assets file path + /// + /// Stores project information extracted from binlogs, keyed by assets file path. + /// + /// + /// All binlog files are processed before any assets files (guaranteed by ), + /// so by the time an assets file is processed this dictionary contains the merged superset of project + /// info from every binlog that referenced that project. This is intentional: a repository may produce + /// separate binlogs for different build passes (e.g., build vs. publish). Properties like + /// or are + /// typically only set in the publish pass, so merging ensures those properties are available when + /// processing the shared project.assets.json. + /// + /// Memory impact: each is roughly 15–16 KB (including inner builds + /// and PackageReference/PackageDownload dictionaries). A repository with 100K projects would use + /// approximately 1.5 GB for project info storage alone — significant but proportional to the + /// multi-GB binlog files that such a repository would produce. + /// private readonly ConcurrentDictionary projectInfoByAssetsFile = new(StringComparer.OrdinalIgnoreCase); /// @@ -257,15 +267,6 @@ private void ProcessBinlogFile(ProcessRequest processRequest) this.LogMissingAssetsWarnings(projectInfo); } - // Register DotNet components for projects that won't have lock files - // (projects with lock files get DotNet registration in ProcessLockFileWithProjectInfo - // where we can combine binlog and lock-file self-contained heuristics) - foreach (var projectInfo in projectInfos.Where( - pi => string.IsNullOrEmpty(pi.ProjectAssetsFile) || !this.fileUtilityService.Exists(pi.ProjectAssetsFile))) - { - this.RegisterDotNetComponent(projectInfo); - } - // Log summary warning if no assets files were found if (assetsFilesFound.Count == 0 && projectInfos.Count > 0) { @@ -283,19 +284,9 @@ private void ProcessBinlogFile(ProcessRequest processRequest) private void IndexProjectInfo(MSBuildProjectInfo projectInfo, List assetsFilesFound) { - // Store the project info for later use when processing assets files. + // Index by assets file path for lookup when processing lock files. // Use AddOrUpdate+MergeWith so that multiple binlogs for the same project // (e.g., build and publish passes) form a superset rather than keeping only the first. - var projectPath = projectInfo.ProjectPath; - if (!string.IsNullOrEmpty(projectPath)) - { - this.projectInfoByProjectPath.AddOrUpdate( - projectPath, - _ => projectInfo, - (_, existing) => existing.MergeWith(projectInfo)); - } - - // Also index by assets file path for direct lookup if (!string.IsNullOrEmpty(projectInfo.ProjectAssetsFile)) { this.projectInfoByAssetsFile.AddOrUpdate( @@ -418,13 +409,6 @@ private async Task ProcessAssetsFileAsync(ProcessRequest processRequest, Cancell { var assetsFilePath = processRequest.ComponentStream.Location; - // Check if this assets file was already processed - if (this.processedAssetsFiles.ContainsKey(assetsFilePath)) - { - this.Logger.LogDebug("Assets file already processed: {AssetsFile}", assetsFilePath); - return; - } - try { var lockFile = this.lockFileFormat.Read(processRequest.ComponentStream.Stream, assetsFilePath); @@ -438,10 +422,7 @@ private async Task ProcessAssetsFileAsync(ProcessRequest processRequest, Cancell } // Try to find matching binlog info - var projectInfo = this.FindProjectInfoForAssetsFile(assetsFilePath, lockFile); - - // Mark as processed - this.processedAssetsFiles.TryAdd(assetsFilePath, true); + var projectInfo = this.FindProjectInfoForAssetsFile(assetsFilePath); if (projectInfo != null) { @@ -464,23 +445,10 @@ private async Task ProcessAssetsFileAsync(ProcessRequest processRequest, Cancell } } - private MSBuildProjectInfo? FindProjectInfoForAssetsFile(string assetsFilePath, LockFile lockFile) + private MSBuildProjectInfo? FindProjectInfoForAssetsFile(string assetsFilePath) { - // Try to find by assets file path first - if (this.projectInfoByAssetsFile.TryGetValue(assetsFilePath, out var infoByAssets)) - { - return infoByAssets; - } - - // Try to find by project path from the lock file - var projectPath = lockFile.PackageSpec?.RestoreMetadata?.ProjectPath; - if (!string.IsNullOrEmpty(projectPath) && - this.projectInfoByProjectPath.TryGetValue(projectPath, out var infoByProject)) - { - return infoByProject; - } - - return null; + this.projectInfoByAssetsFile.TryGetValue(assetsFilePath, out var projectInfo); + return projectInfo; } /// diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs index 3e81cc91f..69bad1760 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs @@ -469,6 +469,138 @@ public async Task WithBinlog_NoBinlogMatch_FallsBackToStandardProcessing() recorder.GetEffectiveDevDependencyValue(component.Component.Id).Should().BeFalse(); } + // ================================================================ + // Multi-binlog merge tests – verify that project info from multiple + // binlogs (e.g., build + publish) is merged before assets processing. + // This is intentional: a repo may produce separate binlogs for build + // and publish passes. The detector merges them via AddOrUpdate+MergeWith + // so that properties like SelfContained (only set during publish) are + // available when processing the shared project.assets.json. + // ================================================================ + [TestMethod] + public async Task MultipleBinlogs_BuildThenPublishSelfContained_MergedAsSelfContained() + { + // First binlog: normal build — SelfContained is not set + var buildInfo = CreateProjectInfo(); + buildInfo.NETCoreSdkVersion = "8.0.100"; + buildInfo.OutputType = "Exe"; + + // Second binlog: publish self-contained — SelfContained = true + var publishInfo = CreateProjectInfo(); + publishInfo.NETCoreSdkVersion = "8.0.100"; + publishInfo.OutputType = "Exe"; + publishInfo.SelfContained = true; + + var assetsJson = SimpleAssetsJson("Newtonsoft.Json", "13.0.1"); + + var binlogs = new Dictionary> + { + [@"C:\test\build.binlog"] = [buildInfo], + [@"C:\test\publish.binlog"] = [publishInfo], + }; + + var (result, recorder) = await ExecuteWithMultipleBinlogsAsync(binlogs, assetsJson); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + + // The merged project info should have SelfContained=true from the publish binlog + var dotNet = recorder.GetDetectedComponents() + .Where(c => c.Component is DotNetComponent) + .Select(c => (DotNetComponent)c.Component) + .Single(); + dotNet.ProjectType.Should().Be("application-selfcontained"); + } + + [TestMethod] + public async Task MultipleBinlogs_BuildThenPublishNotSelfContained_MergedAsNotSelfContained() + { + // Both binlogs: normal build — SelfContained is not set in either + var buildInfo = CreateProjectInfo(); + buildInfo.NETCoreSdkVersion = "8.0.100"; + buildInfo.OutputType = "Exe"; + + var publishInfo = CreateProjectInfo(); + publishInfo.NETCoreSdkVersion = "8.0.100"; + publishInfo.OutputType = "Exe"; + + var assetsJson = SimpleAssetsJson("Newtonsoft.Json", "13.0.1"); + + var binlogs = new Dictionary> + { + [@"C:\test\build.binlog"] = [buildInfo], + [@"C:\test\publish.binlog"] = [publishInfo], + }; + + var (result, recorder) = await ExecuteWithMultipleBinlogsAsync(binlogs, assetsJson); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + + var dotNet = recorder.GetDetectedComponents() + .Where(c => c.Component is DotNetComponent) + .Select(c => (DotNetComponent)c.Component) + .Single(); + dotNet.ProjectType.Should().Be("application"); + } + + [TestMethod] + public async Task MultipleBinlogs_BuildThenPublishAot_MergedAsSelfContained() + { + // First binlog: normal build + var buildInfo = CreateProjectInfo(); + buildInfo.NETCoreSdkVersion = "8.0.100"; + buildInfo.OutputType = "Exe"; + + // Second binlog: publish with AOT + var publishInfo = CreateProjectInfo(); + publishInfo.NETCoreSdkVersion = "8.0.100"; + publishInfo.OutputType = "Exe"; + publishInfo.PublishAot = true; + + var assetsJson = SimpleAssetsJson("Newtonsoft.Json", "13.0.1"); + + var binlogs = new Dictionary> + { + [@"C:\test\build.binlog"] = [buildInfo], + [@"C:\test\publish.binlog"] = [publishInfo], + }; + + var (result, recorder) = await ExecuteWithMultipleBinlogsAsync(binlogs, assetsJson); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + + var dotNet = recorder.GetDetectedComponents() + .Where(c => c.Component is DotNetComponent) + .Select(c => (DotNetComponent)c.Component) + .Single(); + dotNet.ProjectType.Should().Be("application-selfcontained"); + } + + [TestMethod] + public async Task MultipleBinlogs_MergesTestProjectFlag_AllDepsAreDev() + { + // First binlog: normal build — IsTestProject not set + var buildInfo = CreateProjectInfo(); + + // Second binlog: test invocation — IsTestProject = true + var testInfo = CreateProjectInfo(); + testInfo.IsTestProject = true; + + var assetsJson = SimpleAssetsJson("Newtonsoft.Json", "13.0.1"); + + var binlogs = new Dictionary> + { + [@"C:\test\build.binlog"] = [buildInfo], + [@"C:\test\test.binlog"] = [testInfo], + }; + + var (result, recorder) = await ExecuteWithMultipleBinlogsAsync(binlogs, assetsJson); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + + var component = recorder.GetDetectedComponents().Single(c => c.Component is NuGetComponent); + recorder.GetEffectiveDevDependencyValue(component.Component.Id).Should().BeTrue(); + } + // ================================================================ // Helpers – project info construction // ================================================================ @@ -569,6 +701,75 @@ private static ITaskItem CreateTaskItemMock(string itemSpec, bool isDevelopmentD return (result, recorder); } + /// + /// Executes the detector with multiple binlog files, each returning its own set of project infos. + /// This exercises the merge path: when two binlogs reference the same project (same assets file), + /// their MSBuildProjectInfo instances are merged via AddOrUpdate+MergeWith before the assets file + /// is processed. For example, a normal build binlog + a publish-self-contained binlog should + /// produce a merged MSBuildProjectInfo with SelfContained=true. + /// + private static async Task<(IndividualDetectorScanResult Result, IComponentRecorder Recorder)> ExecuteWithMultipleBinlogsAsync( + Dictionary> binlogProjectInfos, + string assetsJson, + string assetsLocation = AssetsFilePath) + { + var binLogProcessorMock = new Mock(); + foreach (var (binlogPath, projectInfos) in binlogProjectInfos) + { + binLogProcessorMock + .Setup(x => x.ExtractProjectInfo(binlogPath)) + .Returns(projectInfos); + } + + var walkerMock = new Mock(); + var streamFactoryMock = new Mock(); + var commandLineInvocationMock = new Mock(); + var directoryUtilityMock = new Mock(); + var fileUtilityMock = new Mock(); + fileUtilityMock.Setup(x => x.Exists(It.IsAny())).Returns(true); + var pathUtilityMock = new Mock(); + pathUtilityMock.Setup(x => x.NormalizePath(It.IsAny())).Returns(p => p); + pathUtilityMock.Setup(x => x.GetParentDirectory(It.IsAny())).Returns(p => Path.GetDirectoryName(p) ?? string.Empty); + var loggerMock = new Mock>(); + + var detector = new MSBuildBinaryLogComponentDetector( + streamFactoryMock.Object, + walkerMock.Object, + commandLineInvocationMock.Object, + directoryUtilityMock.Object, + fileUtilityMock.Object, + pathUtilityMock.Object, + binLogProcessorMock.Object, + loggerMock.Object); + + var recorder = new ComponentRecorder(); + + // Build process requests: one per binlog file, then the assets file + var requests = binlogProjectInfos.Keys + .Select(binlogPath => CreateProcessRequest(recorder, binlogPath, "fake-binlog-content")) + .Append(CreateProcessRequest(recorder, assetsLocation, assetsJson)) + .ToArray(); + + walkerMock + .Setup(x => x.GetFilteredComponentStreamObservable( + It.IsAny(), + It.IsAny>(), + It.IsAny())) + .Returns(requests.ToObservable()); + + var scanRequest = new ScanRequest( + new DirectoryInfo(Path.GetTempPath()), + null, + null, + new Dictionary(), + null, + recorder, + sourceFileRoot: new DirectoryInfo(Path.GetTempPath())); + + var result = await detector.ExecuteDetectorAsync(scanRequest); + return (result, recorder); + } + private static ProcessRequest CreateProcessRequest(IComponentRecorder recorder, string location, string content) { var stream = new MemoryStream(Encoding.UTF8.GetBytes(content)); From f5e632ba590caa3496cff63efb42e7931ed99a16 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Wed, 11 Mar 2026 16:55:47 -0700 Subject: [PATCH 10/26] Rebase binlog paths --- .../dotnet/DotNetProjectInfoProvider.cs | 48 +----- .../dotnet/PathRebasingUtility.cs | 150 ++++++++++++++++++ .../nuget/BinLogProcessor.cs | 112 ++++++++++--- .../nuget/IBinLogProcessor.cs | 14 +- .../MSBuildBinaryLogComponentDetector.cs | 14 +- .../nuget/MSBuildProjectInfo.cs | 94 +++++++---- .../MSBuildBinaryLogComponentDetectorTests.cs | 4 +- 7 files changed, 341 insertions(+), 95 deletions(-) create mode 100644 src/Microsoft.ComponentDetection.Detectors/dotnet/PathRebasingUtility.cs diff --git a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetProjectInfoProvider.cs b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetProjectInfoProvider.cs index 61836b726..e94d01d31 100644 --- a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetProjectInfoProvider.cs +++ b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetProjectInfoProvider.cs @@ -128,22 +128,8 @@ public async Task RegisterDotNetComponentsAsync( } } - private static string TrimAllEndingDirectorySeparators(string path) - { - string last; - - do - { - last = path; - path = Path.TrimEndingDirectorySeparator(last); - } - while (!ReferenceEquals(last, path)); - - return path; - } - [return: NotNullIfNotNull(nameof(path))] - private string? NormalizeDirectory(string? path) => string.IsNullOrEmpty(path) ? path : TrimAllEndingDirectorySeparators(this.pathUtilityService.NormalizePath(path)); + private string? NormalizeDirectory(string? path) => PathRebasingUtility.NormalizeDirectory(path); /// /// Given a path under sourceDirectory, and the same path in another filesystem, @@ -154,35 +140,17 @@ private static string TrimAllEndingDirectorySeparators(string path) /// Portion of that corresponds to root, or null if it can not be rebased. internal string? GetRootRebasePath(string sourceDirectoryBasedPath, string? rebasePath) { - if (string.IsNullOrEmpty(rebasePath) || string.IsNullOrEmpty(this.sourceDirectory) || string.IsNullOrEmpty(sourceDirectoryBasedPath)) - { - return null; - } - - // sourceDirectory is normalized, normalize others - sourceDirectoryBasedPath = this.NormalizeDirectory(sourceDirectoryBasedPath); - rebasePath = this.NormalizeDirectory(rebasePath); + var result = PathRebasingUtility.GetRebaseRoot(this.sourceDirectory, sourceDirectoryBasedPath, rebasePath); - // nothing to do if the paths are the same - if (rebasePath.Equals(sourceDirectoryBasedPath, StringComparison.Ordinal)) + if (result != null) { - return null; + this.logger.LogDebug( + "Rebasing paths from {RebasePath} to {SourceDirectoryBasedPath}", + rebasePath, + sourceDirectoryBasedPath); } - // find the relative path under sourceDirectory. - var sourceDirectoryRelativePath = this.NormalizeDirectory(Path.GetRelativePath(this.sourceDirectory!, sourceDirectoryBasedPath)); - - this.logger.LogDebug("Attempting to rebase {RebasePath} to {SourceDirectoryBasedPath} using relative {SourceDirectoryRelativePath}", rebasePath, sourceDirectoryBasedPath, sourceDirectoryRelativePath); - - // if the rebase path has the same relative portion, then we have a replacement. - if (rebasePath.EndsWith(sourceDirectoryRelativePath)) - { - return rebasePath[..^sourceDirectoryRelativePath.Length]; - } - - // The path didn't have a common relative path, it might have been copied from a completely different location since it was built. - // We cannot rebase the paths. - return null; + return result; } internal string? GetProjectType(string projectOutputPath, string projectName) diff --git a/src/Microsoft.ComponentDetection.Detectors/dotnet/PathRebasingUtility.cs b/src/Microsoft.ComponentDetection.Detectors/dotnet/PathRebasingUtility.cs new file mode 100644 index 000000000..81d0f0fc2 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/dotnet/PathRebasingUtility.cs @@ -0,0 +1,150 @@ +namespace Microsoft.ComponentDetection.Detectors.DotNet; + +using System; +using System.Collections.Generic; +using System.IO; + +/// +/// Utility for rebasing absolute paths from one filesystem root to another. +/// +/// +/// When component detection scans build outputs produced on a different machine (e.g., a CI agent), +/// the absolute paths recorded in artifacts like binlogs and project.assets.json will not match +/// the paths on the scanning machine. This utility detects and compensates for that by finding +/// the common relative suffix between two representations of the same directory and deriving +/// the root prefix that needs to be substituted. +/// +internal static class PathRebasingUtility +{ + /// + /// Normalizes a path by replacing backslashes with forward slashes. + /// Windows accepts / as a separator, so forward-slash normalization works everywhere. + /// + /// The path with all backslashes replaced by forward slashes. + internal static string NormalizePath(string path) => path.Replace('\\', Path.AltDirectorySeparatorChar); + + /// + /// Normalizes a directory path: forward slashes, no trailing separator. + /// Returns null/empty passthrough for null/empty input. + /// + /// The normalized directory path, or null/empty for null/empty input. + internal static string? NormalizeDirectory(string? path) => + string.IsNullOrEmpty(path) ? path : TrimAllEndingDirectorySeparators(NormalizePath(path)); + + /// + /// Given a path known to be under on the scanning machine, + /// and the same directory as it appears in a build artifact (binlog, lock file, etc.), + /// determine the root prefix in the artifact that corresponds to . + /// + /// The scanning machine's source directory (already normalized). + /// Path to a directory under on the scanning machine. + /// Path to the same directory as it appears in the build artifact. + /// + /// The root prefix of that can be replaced with , + /// or null if the paths cannot be rebased (same root, or no common relative suffix). + /// + internal static string? GetRebaseRoot(string? sourceDirectory, string sourceDirectoryBasedPath, string? artifactPath) + { + if (string.IsNullOrEmpty(artifactPath) || string.IsNullOrEmpty(sourceDirectory) || string.IsNullOrEmpty(sourceDirectoryBasedPath)) + { + return null; + } + + sourceDirectoryBasedPath = NormalizeDirectory(sourceDirectoryBasedPath)!; + artifactPath = NormalizeDirectory(artifactPath)!; + + // Nothing to do if the paths are the same (no rebasing needed). + if (artifactPath.Equals(sourceDirectoryBasedPath, StringComparison.Ordinal)) + { + return null; + } + + // Find the relative path under sourceDirectory. + var sourceDirectoryRelativePath = NormalizeDirectory(Path.GetRelativePath(sourceDirectory, sourceDirectoryBasedPath))!; + + // If the artifact path has the same relative portion, extract the root prefix. + if (artifactPath.EndsWith(sourceDirectoryRelativePath, StringComparison.Ordinal)) + { + return artifactPath[..^sourceDirectoryRelativePath.Length]; + } + + // The path didn't have a common relative suffix — it might have been copied from + // a completely different location since it was built. We cannot rebase. + return null; + } + + /// + /// Rebases an absolute path from one root to another. + /// + /// The absolute path to rebase (from the build artifact). + /// The root prefix from the build artifact (as returned by ). + /// The root on the scanning machine (typically sourceDirectory). + /// The rebased path under . + internal static string RebasePath(string path, string originalRoot, string newRoot) + { + var normalizedPath = NormalizeDirectory(path)!; + var normalizedOriginal = NormalizeDirectory(originalRoot)!; + var normalizedNew = NormalizeDirectory(newRoot)!; + var relative = Path.GetRelativePath(normalizedOriginal, normalizedPath); + return NormalizePath(Path.Combine(normalizedNew, relative)); + } + + /// + /// Searches a dictionary for a key that matches the given scan-machine path after rebasing. + /// Computes the relative path of under + /// and looks for a dictionary key whose normalized form ends with the same relative suffix. + /// + /// The dictionary value type. + /// The dictionary keyed by build-machine paths. + /// The scanning machine's source directory (normalized). + /// The path on the scanning machine to look up. + /// + /// If a match is found, outputs the build-machine root prefix that can be used with + /// to convert other build-machine paths. null if no match is found. + /// + /// The matched value, or default if no match is found. + internal static TValue? FindByRelativePath( + IEnumerable> dictionary, + string sourceDirectory, + string scanMachinePath, + out string? rebaseRoot) + { + rebaseRoot = null; + + var normalizedScanPath = NormalizePath(scanMachinePath); + var relativePath = NormalizePath(Path.GetRelativePath(sourceDirectory, normalizedScanPath)); + + // If the path isn't under sourceDirectory, we can't match by suffix. + if (relativePath.StartsWith("..", StringComparison.Ordinal)) + { + return default; + } + + foreach (var kvp in dictionary) + { + var normalizedKey = NormalizePath(kvp.Key); + if (normalizedKey.EndsWith(relativePath, StringComparison.OrdinalIgnoreCase)) + { + // Derive the build-machine root from this match. + rebaseRoot = normalizedKey[..^relativePath.Length]; + return kvp.Value; + } + } + + return default; + } + + private static string TrimAllEndingDirectorySeparators(string path) + { + string last; + + do + { + last = path; + path = Path.TrimEndingDirectorySeparator(last); + } + while (!ReferenceEquals(last, path)); + + return path; + } +} diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs index 9369f1fcf..e2ccdfd35 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs @@ -3,9 +3,11 @@ namespace Microsoft.ComponentDetection.Detectors.NuGet; using System; using System.Collections; using System.Collections.Generic; +using System.IO; using System.Linq; using Microsoft.Build.Framework; using Microsoft.Build.Logging.StructuredLogger; +using Microsoft.ComponentDetection.Detectors.DotNet; using Microsoft.Extensions.Logging; /// @@ -22,11 +24,19 @@ internal class BinLogProcessor : IBinLogProcessor public BinLogProcessor(Microsoft.Extensions.Logging.ILogger logger) => this.logger = logger; /// - public IReadOnlyList ExtractProjectInfo(string binlogPath) + public IReadOnlyList ExtractProjectInfo(string binlogPath, string? sourceDirectory = null) { // Maps project path to the primary MSBuildProjectInfo for that project var projectInfoByPath = new Dictionary(StringComparer.OrdinalIgnoreCase); + // Pre-compute source directory info for path rebasing. + // When the binlog was built on a different machine, BinLogFilePath (recorded at the start + // of the log) lets us derive the root substitution. The rebasePath function is set once + // the BinLogFilePath message is seen and is applied inline to all path-valued properties. + var normalizedSourceDir = PathRebasingUtility.NormalizeDirectory(sourceDirectory); + var binlogDir = PathRebasingUtility.NormalizeDirectory(Path.GetDirectoryName(binlogPath)); + Func? rebasePath = null; + try { var reader = new BinLogReader(); @@ -51,7 +61,7 @@ public IReadOnlyList ExtractProjectInfo(string binlogPath) projectInfoByEvaluationId[e.BuildEventContext.EvaluationId] = projectInfo; } - this.PopulateFromEvaluation(projectEvalArgs, projectInfo); + this.PopulateFromEvaluation(projectEvalArgs, projectInfo, rebasePath); } }; @@ -67,7 +77,37 @@ public IReadOnlyList ExtractProjectInfo(string binlogPath) if (!string.IsNullOrEmpty(e.ProjectFile) && projectInfoByEvaluationId.TryGetValue(e.BuildEventContext.EvaluationId, out var projectInfo)) { - projectInfo.ProjectPath = e.ProjectFile; + projectInfo.ProjectPath = rebasePath != null ? rebasePath(e.ProjectFile) : e.ProjectFile; + } + } + }; + + // Hook into message events to capture BinLogFilePath from the initial build messages. + // MSBuild's BinaryLogger writes "BinLogFilePath=" as a BuildMessageEventArgs + // with SenderName "BinaryLogger" at the start of the log. This arrives before any + // evaluation or project events, so we can compute the rebase function here and apply + // it to all subsequent path-valued properties. + // https://github.com/dotnet/msbuild/blob/7d73e8e9074fe9a4420e38cd22d45645b28a11f7/src/Build/Logging/BinaryLogger/BinaryLogger.cs#L473 + reader.MessageRaised += (sender, e) => + { + if (rebasePath == null && + binlogDir != null && + normalizedSourceDir != null && + e is BuildMessageEventArgs msg && + msg.SenderName == "BinaryLogger" && + msg.Message != null && + msg.Message.StartsWith("BinLogFilePath=", StringComparison.Ordinal)) + { + var originalBinLogFilePath = msg.Message["BinLogFilePath=".Length..]; + var originalBinlogDir = PathRebasingUtility.NormalizeDirectory(Path.GetDirectoryName(originalBinLogFilePath)); + var rebaseRoot = PathRebasingUtility.GetRebaseRoot(normalizedSourceDir, binlogDir, originalBinlogDir); + if (rebaseRoot != null) + { + this.logger.LogDebug( + "Rebasing binlog paths from build-machine root '{RebaseRoot}' to scan-machine root '{SourceDirectory}'", + rebaseRoot, + normalizedSourceDir); + rebasePath = path => PathRebasingUtility.RebasePath(path, rebaseRoot, normalizedSourceDir!); } } }; @@ -75,7 +115,7 @@ public IReadOnlyList ExtractProjectInfo(string binlogPath) // Hook into any event to capture property reassignments and item changes during build reader.AnyEventRaised += (sender, e) => { - this.HandleBuildEvent(e, projectInstanceToEvaluationMap, projectInfoByEvaluationId); + this.HandleBuildEvent(e, projectInstanceToEvaluationMap, projectInfoByEvaluationId, rebasePath); }; // Hook into project finished to collect final project info and establish hierarchy @@ -194,7 +234,7 @@ private void AddOrMergeProjectInfo( /// /// Populates project info from evaluation results (properties and items). /// - private void PopulateFromEvaluation(ProjectEvaluationFinishedEventArgs projectEvalArgs, MSBuildProjectInfo projectInfo) + private void PopulateFromEvaluation(ProjectEvaluationFinishedEventArgs projectEvalArgs, MSBuildProjectInfo projectInfo, Func? rebasePath) { // Extract properties if (projectEvalArgs?.Properties != null) @@ -205,7 +245,7 @@ private void PopulateFromEvaluation(ProjectEvaluationFinishedEventArgs projectEv { foreach (var kvp in propertiesDict) { - projectInfo.TrySetProperty(kvp.Key, kvp.Value); + this.SetPropertyWithRebase(projectInfo, kvp.Key, kvp.Value, rebasePath); } } else @@ -229,7 +269,7 @@ private void PopulateFromEvaluation(ProjectEvaluationFinishedEventArgs projectEv if (!string.IsNullOrEmpty(key)) { - projectInfo.TrySetProperty(key, value ?? string.Empty); + this.SetPropertyWithRebase(projectInfo, key, value ?? string.Empty, rebasePath); } } } @@ -245,9 +285,15 @@ private void PopulateFromEvaluation(ProjectEvaluationFinishedEventArgs projectEv { if (itemEntry is DictionaryEntry entry && entry.Key is string itemType && - MSBuildProjectInfo.IsItemTypeOfInterest(itemType) && + MSBuildProjectInfo.IsItemTypeOfInterest(itemType, out var isPath) && entry.Value is ITaskItem taskItem) { + if (isPath && rebasePath != null) + { + // Rebase the item spec if it's a path + taskItem.ItemSpec = rebasePath(taskItem.ItemSpec); + } + projectInfo.TryAddOrUpdateItem(itemType, taskItem); } } @@ -260,7 +306,8 @@ entry.Key is string itemType && private void HandleBuildEvent( BuildEventArgs? args, Dictionary projectInstanceToEvaluationMap, - Dictionary projectInfoByEvaluationId) + Dictionary projectInfoByEvaluationId, + Func? rebasePath) { if (!this.TryGetProjectInfo(args, projectInstanceToEvaluationMap, projectInfoByEvaluationId, out var projectInfo)) { @@ -271,26 +318,24 @@ private void HandleBuildEvent( { // Property reassignments (when a property value changes during the build) case PropertyReassignmentEventArgs propertyReassignment: - projectInfo.TrySetProperty(propertyReassignment.PropertyName, propertyReassignment.NewValue); + this.SetPropertyWithRebase(projectInfo, propertyReassignment.PropertyName, propertyReassignment.NewValue, rebasePath); break; // Initial property value set events case PropertyInitialValueSetEventArgs propertyInitialValueSet: - projectInfo.TrySetProperty(propertyInitialValueSet.PropertyName, propertyInitialValueSet.PropertyValue); + this.SetPropertyWithRebase(projectInfo, propertyInitialValueSet.PropertyName, propertyInitialValueSet.PropertyValue, rebasePath); break; // Environment variable reads during evaluation - MSBuild promotes env vars to properties case EnvironmentVariableReadEventArgs envVarRead when - !string.IsNullOrEmpty(envVarRead.EnvironmentVariableName) && - MSBuildProjectInfo.IsPropertyOfInterest(envVarRead.EnvironmentVariableName): - projectInfo.TrySetProperty(envVarRead.EnvironmentVariableName, envVarRead.Message ?? string.Empty); + !string.IsNullOrEmpty(envVarRead.EnvironmentVariableName): + this.SetPropertyWithRebase(projectInfo, envVarRead.EnvironmentVariableName, envVarRead.Message ?? string.Empty, rebasePath); break; // Task parameter events which can contain item arrays for add/remove/update case TaskParameterEventArgs taskParameter when - MSBuildProjectInfo.IsItemTypeOfInterest(taskParameter.ItemType) && taskParameter.Items is IList taskItems: - this.ProcessTaskParameterItems(taskParameter.Kind, taskParameter.ItemType, taskItems, projectInfo); + this.ProcessTaskParameterItems(taskParameter.Kind, taskParameter.ItemType, taskItems, projectInfo, rebasePath); break; default: @@ -340,13 +385,20 @@ private void ProcessTaskParameterItems( TaskParameterMessageKind kind, string itemType, IList items, - MSBuildProjectInfo projectInfo) + MSBuildProjectInfo projectInfo, + Func? rebasePath) { + if (!MSBuildProjectInfo.IsItemTypeOfInterest(itemType, out var isPath)) + { + return; + } + if (kind == TaskParameterMessageKind.RemoveItem) { foreach (var item in items) { - projectInfo.TryRemoveItem(itemType, item.ItemSpec); + var itemSpec = isPath && rebasePath != null ? rebasePath(item.ItemSpec) : item.ItemSpec; + projectInfo.TryRemoveItem(itemType, itemSpec); } } else if (kind == TaskParameterMessageKind.TaskInput || @@ -355,10 +407,34 @@ private void ProcessTaskParameterItems( { foreach (var item in items) { + if (isPath && rebasePath != null) + { + // Rebase the item spec if it's a path + item.ItemSpec = rebasePath(item.ItemSpec); + } + projectInfo.TryAddOrUpdateItem(itemType, item); } } // SkippedTargetInputs and SkippedTargetOutputs are informational and don't modify items } + + /// + /// Sets a property on a project info, rebasing the value first if it is a path property. + /// + private void SetPropertyWithRebase(MSBuildProjectInfo projectInfo, string propertyName, string value, Func? rebasePath) + { + if (!MSBuildProjectInfo.IsPropertyOfInterest(propertyName, out var isPath)) + { + return; + } + + if (isPath && rebasePath != null && !string.IsNullOrEmpty(value)) + { + value = rebasePath(value); + } + + projectInfo.TrySetProperty(propertyName, value); + } } diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/IBinLogProcessor.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/IBinLogProcessor.cs index 44a1733f0..2bbdea69d 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/IBinLogProcessor.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/IBinLogProcessor.cs @@ -9,8 +9,16 @@ internal interface IBinLogProcessor { /// /// Extracts project information from a binary log file. + /// All absolute paths in the returned objects + /// (e.g., , ) + /// are rebased to be relative to when the binlog was produced + /// on a different machine. /// - /// Path to the binary log file. - /// Collection of project information extracted from the binlog. - IReadOnlyList ExtractProjectInfo(string binlogPath); + /// Path to the binary log file on the scanning machine. + /// + /// The source directory on the scanning machine, used to rebase paths when the binlog + /// was produced on a different machine. May be null to skip rebasing. + /// + /// Collection of project information extracted from the binlog, with paths rebased. + IReadOnlyList ExtractProjectInfo(string binlogPath, string? sourceDirectory = null); } diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs index c307dff86..25b9dd70c 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs @@ -76,6 +76,9 @@ public class MSBuildBinaryLogComponentDetector : FileComponentDetector, IExperim /// private readonly ConcurrentDictionary projectInfoByAssetsFile = new(StringComparer.OrdinalIgnoreCase); + // Source directory passed to BinLogProcessor for path rebasing. + private string? sourceDirectory; + /// /// Initializes a new instance of the class. /// @@ -153,6 +156,7 @@ internal MSBuildBinaryLogComponentDetector( /// public override Task ExecuteDetectorAsync(ScanRequest request, CancellationToken cancellationToken = default) { + this.sourceDirectory = request.SourceDirectory.FullName; this.projectInfoProvider.Initialize(request.SourceDirectory.FullName, request.SourceFileRoot?.FullName); return base.ExecuteDetectorAsync(request, cancellationToken); } @@ -253,7 +257,7 @@ private void ProcessBinlogFile(ProcessRequest processRequest) { this.Logger.LogDebug("Processing binlog file: {BinlogPath}", binlogPath); - var projectInfos = this.binLogProcessor.ExtractProjectInfo(binlogPath); + var projectInfos = this.binLogProcessor.ExtractProjectInfo(binlogPath, this.sourceDirectory); if (projectInfos.Count == 0) { @@ -445,6 +449,10 @@ private async Task ProcessAssetsFileAsync(ProcessRequest processRequest, Cancell } } + /// + /// Finds the associated with the given assets file path. + /// Paths are already rebased to the scanning machine by . + /// private MSBuildProjectInfo? FindProjectInfoForAssetsFile(string assetsFilePath) { this.projectInfoByAssetsFile.TryGetValue(assetsFilePath, out var projectInfo); @@ -471,7 +479,9 @@ private void ProcessLockFileWithProjectInfo(LockFile lockFile, MSBuildProjectInf var (explicitReferencedDependencies, explicitlyReferencedComponentIds) = LockFileUtilities.ResolveExplicitDependencies(lockFile, this.Logger); // Use project path from RestoreMetadata (consistent with NuGetProjectModelProjectCentricComponentDetector). - // If no project path is available, fall back to the assets file path to avoid collisions. + // BinLogProcessor has already rebased projectInfo.ProjectPath to the scanning machine. + // RestoreMetadata.ProjectPath comes from the lock file which is on the same machine as the assets file. + // Fall back to the assets file path to avoid collisions when no project path is available. var recorderLocation = lockFile.PackageSpec?.RestoreMetadata?.ProjectPath ?? projectInfo.ProjectPath ?? assetsFilePath; diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildProjectInfo.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildProjectInfo.cs index d368bf752..e4cc89379 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildProjectInfo.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildProjectInfo.cs @@ -11,36 +11,38 @@ namespace Microsoft.ComponentDetection.Detectors.NuGet; internal class MSBuildProjectInfo { /// - /// Maps MSBuild property names to their setter actions. + /// Maps MSBuild property names to their metadata. /// - private static readonly Dictionary> PropertySetters = new(StringComparer.OrdinalIgnoreCase) + private static readonly Dictionary Properties = new(StringComparer.OrdinalIgnoreCase) { - [nameof(IsDevelopment)] = (info, value) => info.IsDevelopment = string.Equals(value, "true", StringComparison.OrdinalIgnoreCase), - [nameof(IsPackable)] = (info, value) => info.IsPackable = string.Equals(value, "true", StringComparison.OrdinalIgnoreCase), - [nameof(IsShipping)] = (info, value) => info.IsShipping = string.Equals(value, "true", StringComparison.OrdinalIgnoreCase), - [nameof(IsTestProject)] = (info, value) => info.IsTestProject = string.Equals(value, "true", StringComparison.OrdinalIgnoreCase), - [nameof(NETCoreSdkVersion)] = (info, value) => info.NETCoreSdkVersion = value, - [nameof(OutputType)] = (info, value) => info.OutputType = value, - [nameof(PublishAot)] = (info, value) => info.PublishAot = string.Equals(value, "true", StringComparison.OrdinalIgnoreCase), - [nameof(ProjectAssetsFile)] = (info, value) => - { - if (!string.IsNullOrEmpty(value)) + [nameof(IsDevelopment)] = new((info, value) => info.IsDevelopment = string.Equals(value, "true", StringComparison.OrdinalIgnoreCase)), + [nameof(IsPackable)] = new((info, value) => info.IsPackable = string.Equals(value, "true", StringComparison.OrdinalIgnoreCase)), + [nameof(IsShipping)] = new((info, value) => info.IsShipping = string.Equals(value, "true", StringComparison.OrdinalIgnoreCase)), + [nameof(IsTestProject)] = new((info, value) => info.IsTestProject = string.Equals(value, "true", StringComparison.OrdinalIgnoreCase)), + [nameof(NETCoreSdkVersion)] = new((info, value) => info.NETCoreSdkVersion = value), + [nameof(OutputType)] = new((info, value) => info.OutputType = value), + [nameof(PublishAot)] = new((info, value) => info.PublishAot = string.Equals(value, "true", StringComparison.OrdinalIgnoreCase)), + [nameof(ProjectAssetsFile)] = new( + (info, value) => { - info.ProjectAssetsFile = value; - } - }, - [nameof(SelfContained)] = (info, value) => info.SelfContained = string.Equals(value, "true", StringComparison.OrdinalIgnoreCase), - [nameof(TargetFramework)] = (info, value) => info.TargetFramework = value, - [nameof(TargetFrameworks)] = (info, value) => info.TargetFrameworks = value, + if (!string.IsNullOrEmpty(value)) + { + info.ProjectAssetsFile = value; + } + }, + IsPath: true), + [nameof(SelfContained)] = new((info, value) => info.SelfContained = string.Equals(value, "true", StringComparison.OrdinalIgnoreCase)), + [nameof(TargetFramework)] = new((info, value) => info.TargetFramework = value), + [nameof(TargetFrameworks)] = new((info, value) => info.TargetFrameworks = value), }; /// - /// Maps MSBuild item type names to their dictionary accessor. + /// Maps MSBuild item type names to their metadata. /// - private static readonly Dictionary>> ItemDictionaries = new(StringComparer.OrdinalIgnoreCase) + private static readonly Dictionary Items = new(StringComparer.OrdinalIgnoreCase) { - [nameof(PackageReference)] = info => info.PackageReference, - [nameof(PackageDownload)] = info => info.PackageDownload, + [nameof(PackageReference)] = new(info => info.PackageReference), + [nameof(PackageDownload)] = new(info => info.PackageDownload), }; /// @@ -145,15 +147,37 @@ internal class MSBuildProjectInfo /// Determines whether the specified item type is one that this class captures. /// /// The MSBuild item type. + /// When true, the item's ItemSpec is a filesystem path that may need rebasing. /// True if the item type is of interest; otherwise, false. - public static bool IsItemTypeOfInterest(string itemType) => ItemDictionaries.ContainsKey(itemType); + public static bool IsItemTypeOfInterest(string itemType, out bool isPath) + { + if (Items.TryGetValue(itemType, out var info)) + { + isPath = info.IsPath; + return true; + } + + isPath = false; + return false; + } /// /// Determines whether the specified property name is one that this class captures. /// /// The MSBuild property name. + /// When true, the property value is a filesystem path that may need rebasing. /// True if the property is of interest; otherwise, false. - public static bool IsPropertyOfInterest(string propertyName) => PropertySetters.ContainsKey(propertyName); + public static bool IsPropertyOfInterest(string propertyName, out bool isPath) + { + if (Properties.TryGetValue(propertyName, out var info)) + { + isPath = info.IsPath; + return true; + } + + isPath = false; + return false; + } /// /// Sets a property value if it is one of the properties of interest. @@ -163,9 +187,9 @@ internal class MSBuildProjectInfo /// True if the property was set; otherwise, false. public bool TrySetProperty(string propertyName, string value) { - if (PropertySetters.TryGetValue(propertyName, out var setter)) + if (Properties.TryGetValue(propertyName, out var info)) { - setter(this, value); + info.Setter(this, value); return true; } @@ -180,12 +204,12 @@ public bool TrySetProperty(string propertyName, string value) /// True if the item was added or updated; otherwise, false. public bool TryAddOrUpdateItem(string itemType, ITaskItem item) { - if (item == null || !ItemDictionaries.TryGetValue(itemType, out var getDictionary)) + if (item == null || !Items.TryGetValue(itemType, out var itemInfo)) { return false; } - var dictionary = getDictionary(this); + var dictionary = itemInfo.GetDictionary(this); dictionary[item.ItemSpec] = item; return true; } @@ -198,12 +222,12 @@ public bool TryAddOrUpdateItem(string itemType, ITaskItem item) /// True if the item was removed; otherwise, false. public bool TryRemoveItem(string itemType, string itemSpec) { - if (!ItemDictionaries.TryGetValue(itemType, out var getDictionary)) + if (!Items.TryGetValue(itemType, out var info)) { return false; } - var dictionary = getDictionary(this); + var dictionary = info.GetDictionary(this); return dictionary.Remove(itemSpec); } @@ -262,4 +286,14 @@ private static void MergeItems(IDictionary target, IDictionar target.TryAdd(kvp.Key, kvp.Value); } } + + /// + /// Metadata for a tracked MSBuild property: its setter and whether its value is a filesystem path. + /// + private record PropertyInfo(Action Setter, bool IsPath = false); + + /// + /// Metadata for a tracked MSBuild item type: its dictionary accessor and whether its ItemSpec is a filesystem path. + /// + private record ItemInfo(Func> GetDictionary, bool IsPath = false); } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs index 69bad1760..9ac2d2a29 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs @@ -649,7 +649,7 @@ private static ITaskItem CreateTaskItemMock(string itemSpec, bool isDevelopmentD { var binLogProcessorMock = new Mock(); binLogProcessorMock - .Setup(x => x.ExtractProjectInfo(binlogPath)) + .Setup(x => x.ExtractProjectInfo(binlogPath, It.IsAny())) .Returns(projectInfos); var walkerMock = new Mock(); @@ -717,7 +717,7 @@ private static ITaskItem CreateTaskItemMock(string itemSpec, bool isDevelopmentD foreach (var (binlogPath, projectInfos) in binlogProjectInfos) { binLogProcessorMock - .Setup(x => x.ExtractProjectInfo(binlogPath)) + .Setup(x => x.ExtractProjectInfo(binlogPath, It.IsAny())) .Returns(projectInfos); } From edaa6c0586cc7b4fedb64850e5cfef9ecd01000e Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Wed, 11 Mar 2026 18:11:33 -0700 Subject: [PATCH 11/26] Add tests for PathRebasingUtility --- .../dotnet/PathRebasingUtility.cs | 19 +- .../dotnet/PathRebasingUtilityTests.cs | 318 ++++++++++++++++++ 2 files changed, 336 insertions(+), 1 deletion(-) create mode 100644 test/Microsoft.ComponentDetection.Detectors.Tests/dotnet/PathRebasingUtilityTests.cs diff --git a/src/Microsoft.ComponentDetection.Detectors/dotnet/PathRebasingUtility.cs b/src/Microsoft.ComponentDetection.Detectors/dotnet/PathRebasingUtility.cs index 81d0f0fc2..69221e3b0 100644 --- a/src/Microsoft.ComponentDetection.Detectors/dotnet/PathRebasingUtility.cs +++ b/src/Microsoft.ComponentDetection.Detectors/dotnet/PathRebasingUtility.cs @@ -79,13 +79,30 @@ internal static class PathRebasingUtility /// The absolute path to rebase (from the build artifact). /// The root prefix from the build artifact (as returned by ). /// The root on the scanning machine (typically sourceDirectory). - /// The rebased path under . + /// + /// The rebased path under , or the normalized input + /// unchanged when it is not rooted or not under . + /// internal static string RebasePath(string path, string originalRoot, string newRoot) { var normalizedPath = NormalizeDirectory(path)!; + + if (!Path.IsPathRooted(normalizedPath)) + { + return normalizedPath; + } + var normalizedOriginal = NormalizeDirectory(originalRoot)!; var normalizedNew = NormalizeDirectory(newRoot)!; var relative = Path.GetRelativePath(normalizedOriginal, normalizedPath); + + // If the path is outside the original root the relative result will start + // with ".." or remain rooted (Windows cross-drive). Return unchanged. + if (Path.IsPathRooted(relative) || relative.StartsWith("..", StringComparison.Ordinal)) + { + return normalizedPath; + } + return NormalizePath(Path.Combine(normalizedNew, relative)); } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/dotnet/PathRebasingUtilityTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/dotnet/PathRebasingUtilityTests.cs new file mode 100644 index 000000000..106c0bf09 --- /dev/null +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/dotnet/PathRebasingUtilityTests.cs @@ -0,0 +1,318 @@ +namespace Microsoft.ComponentDetection.Detectors.Tests.DotNet; + +using System.Collections.Generic; +using System.Runtime.InteropServices; +using AwesomeAssertions; +using Microsoft.ComponentDetection.Detectors.DotNet; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[TestClass] +public class PathRebasingUtilityTests +{ + private static readonly string RootDir = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "C:" : string.Empty; + + // A second root to simulate the build machine having a different drive or prefix. + private static readonly string AltRootDir = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "D:" : "/alt"; + + [TestMethod] + public void NormalizePath_ReplacesBackslashesWithForwardSlashes() + { + PathRebasingUtility.NormalizePath(@"C:\path\to\file").Should().Be("C:/path/to/file"); + } + + [TestMethod] + public void NormalizePath_ForwardSlashesUnchanged() + { + PathRebasingUtility.NormalizePath("C:/path/to/file").Should().Be("C:/path/to/file"); + } + + [TestMethod] + public void NormalizePath_MixedSlashes() + { + PathRebasingUtility.NormalizePath(@"C:\path/to\file").Should().Be("C:/path/to/file"); + } + + [TestMethod] + public void NormalizeDirectory_NullReturnsNull() + { + PathRebasingUtility.NormalizeDirectory(null).Should().BeNull(); + } + + [TestMethod] + public void NormalizeDirectory_EmptyReturnsEmpty() + { + PathRebasingUtility.NormalizeDirectory(string.Empty).Should().BeEmpty(); + } + + [TestMethod] + public void NormalizeDirectory_TrimsTrailingSeparators() + { + PathRebasingUtility.NormalizeDirectory(@"C:\path\to\dir\").Should().Be("C:/path/to/dir"); + } + + [TestMethod] + public void NormalizeDirectory_TrimsMultipleTrailingSeparators() + { + PathRebasingUtility.NormalizeDirectory(@"C:\path\to\dir\/\/").Should().Be("C:/path/to/dir"); + } + + [TestMethod] + public void NormalizeDirectory_NoTrailingSeparator() + { + PathRebasingUtility.NormalizeDirectory(@"C:\path\to\dir").Should().Be("C:/path/to/dir"); + } + + [TestMethod] + public void GetRebaseRoot_BasicRebase_ReturnsBuildMachineRoot() + { + // Scan machine: C:/src/repo/path/to/project + // Build machine: D:/a/_work/1/s/path/to/project + var sourceDir = $"{RootDir}/src/repo"; + var sourceBasedPath = $"{RootDir}/src/repo/path/to/project"; + var artifactPath = $"{AltRootDir}/a/_work/1/s/path/to/project"; + + var result = PathRebasingUtility.GetRebaseRoot(sourceDir, sourceBasedPath, artifactPath); + + result.Should().Be($"{AltRootDir}/a/_work/1/s/"); + } + + [TestMethod] + public void GetRebaseRoot_SamePaths_ReturnsNull() + { + var path = $"{RootDir}/src/repo/path/to/project"; + + var result = PathRebasingUtility.GetRebaseRoot($"{RootDir}/src/repo", path, path); + + result.Should().BeNull(); + } + + [TestMethod] + public void GetRebaseRoot_NoCommonSuffix_ReturnsNull() + { + // Artifact path doesn't share any relative suffix with the source-based path. + var sourceDir = $"{RootDir}/src/repo"; + var sourceBasedPath = $"{RootDir}/src/repo/path/to/project"; + var artifactPath = $"{AltRootDir}/completely/different/layout"; + + var result = PathRebasingUtility.GetRebaseRoot(sourceDir, sourceBasedPath, artifactPath); + + result.Should().BeNull(); + } + + [TestMethod] + public void GetRebaseRoot_NullSourceDirectory_ReturnsNull() + { + PathRebasingUtility.GetRebaseRoot(null, "/some/path", "/other/path").Should().BeNull(); + } + + [TestMethod] + public void GetRebaseRoot_EmptySourceDirectory_ReturnsNull() + { + PathRebasingUtility.GetRebaseRoot(string.Empty, "/some/path", "/other/path").Should().BeNull(); + } + + [TestMethod] + public void GetRebaseRoot_NullArtifactPath_ReturnsNull() + { + PathRebasingUtility.GetRebaseRoot("/src", "/src/path", null).Should().BeNull(); + } + + [TestMethod] + public void GetRebaseRoot_EmptyArtifactPath_ReturnsNull() + { + PathRebasingUtility.GetRebaseRoot("/src", "/src/path", string.Empty).Should().BeNull(); + } + + [TestMethod] + public void GetRebaseRoot_BackslashPaths_NormalizedBeforeComparison() + { + var sourceDir = $"{RootDir}/src/repo"; + var sourceBasedPath = $@"{RootDir}\src\repo\path\to\project"; + var artifactPath = $@"{AltRootDir}\a\_work\1\s\path\to\project"; + + var result = PathRebasingUtility.GetRebaseRoot(sourceDir, sourceBasedPath, artifactPath); + + result.Should().Be($"{AltRootDir}/a/_work/1/s/"); + } + + [TestMethod] + public void GetRebaseRoot_TrailingSeparatorsAreNormalized() + { + var sourceDir = $"{RootDir}/src/repo"; + var sourceBasedPath = $"{RootDir}/src/repo/path/to/project/"; + var artifactPath = $"{AltRootDir}/agent/path/to/project/"; + + var result = PathRebasingUtility.GetRebaseRoot(sourceDir, sourceBasedPath, artifactPath); + + result.Should().Be($"{AltRootDir}/agent/"); + } + + [TestMethod] + public void RebasePath_BasicRebase() + { + var originalRoot = $"{AltRootDir}/a/_work/1/s/"; + var newRoot = $"{RootDir}/src/repo"; + var path = $"{AltRootDir}/a/_work/1/s/path/to/project/obj"; + + var result = PathRebasingUtility.RebasePath(path, originalRoot, newRoot); + + result.Should().Be($"{RootDir}/src/repo/path/to/project/obj"); + } + + [TestMethod] + public void RebasePath_BackslashInput_NormalizedOutput() + { + var originalRoot = $@"{AltRootDir}\a\_work\1\s\"; + var newRoot = $@"{RootDir}\src\repo"; + var path = $@"{AltRootDir}\a\_work\1\s\path\to\file.csproj"; + + var result = PathRebasingUtility.RebasePath(path, originalRoot, newRoot); + + result.Should().Be($"{RootDir}/src/repo/path/to/file.csproj"); + } + + [TestMethod] + public void RebasePath_RootOnlyPath_ReturnsNewRoot() + { + var originalRoot = $"{AltRootDir}/build/"; + var newRoot = $"{RootDir}/scan"; + var path = $"{AltRootDir}/build"; + + var result = PathRebasingUtility.RebasePath(path, originalRoot, newRoot); + + // Path equals originalRoot → relative is "." → Path.Combine preserves it. + result.Should().Be($"{RootDir}/scan/."); + } + + [TestMethod] + public void RebasePath_RoundTripsWithGetRebaseRoot() + { + var sourceDir = $"{RootDir}/src/repo"; + var sourceBasedPath = $"{RootDir}/src/repo/subdir/project"; + var artifactPath = $"{AltRootDir}/agent/s/subdir/project"; + + var rebaseRoot = PathRebasingUtility.GetRebaseRoot(sourceDir, sourceBasedPath, artifactPath); + rebaseRoot.Should().NotBeNull(); + + // Given a different path on the build machine, rebase it to the scan machine. + var buildMachinePath = $"{AltRootDir}/agent/s/subdir/project/obj/project.assets.json"; + var result = PathRebasingUtility.RebasePath(buildMachinePath, rebaseRoot!, sourceDir); + + result.Should().Be($"{RootDir}/src/repo/subdir/project/obj/project.assets.json"); + } + + [TestMethod] + public void RebasePath_RelativePath_ReturnedUnchanged() + { + var originalRoot = $"{RootDir}/build/root"; + var newRoot = $"{RootDir}/scan/root"; + var relativePath = "relative/path/file.csproj"; + + var result = PathRebasingUtility.RebasePath(relativePath, originalRoot, newRoot); + + // A non-rooted path cannot be rebased — returned unchanged. + result.Should().Be("relative/path/file.csproj"); + } + + [TestMethod] + public void RebasePath_PathOutsideOriginalRoot_ReturnedUnchanged() + { + // Path is on a completely different root from originalRoot. + var originalRoot = $"{AltRootDir}/a/_work/1/s"; + var newRoot = $"{RootDir}/src/repo"; + var outsidePath = $"{RootDir}/completely/different/file.csproj"; + + var result = PathRebasingUtility.RebasePath(outsidePath, originalRoot, newRoot); + + // Cannot be rebased — returned unchanged (normalized). + result.Should().Be($"{RootDir}/completely/different/file.csproj"); + } + + [TestMethod] + public void FindByRelativePath_MatchesBySuffix() + { + var sourceDir = $"{RootDir}/src/repo"; + var scanPath = $"{RootDir}/src/repo/path/to/project/obj/project.assets.json"; + + var dictionary = new Dictionary + { + { $"{AltRootDir}/agent/s/path/to/project/obj/project.assets.json", "matched-value" }, + }; + + var result = PathRebasingUtility.FindByRelativePath( + dictionary, sourceDir, scanPath, out var rebaseRoot); + + result.Should().Be("matched-value"); + rebaseRoot.Should().Be($"{AltRootDir}/agent/s/"); + } + + [TestMethod] + public void FindByRelativePath_NoMatch_ReturnsDefault() + { + var sourceDir = $"{RootDir}/src/repo"; + var scanPath = $"{RootDir}/src/repo/path/to/project/obj/project.assets.json"; + + var dictionary = new Dictionary + { + { $"{AltRootDir}/completely/different/layout.json", "value" }, + }; + + var result = PathRebasingUtility.FindByRelativePath( + dictionary, sourceDir, scanPath, out var rebaseRoot); + + result.Should().BeNull(); + rebaseRoot.Should().BeNull(); + } + + [TestMethod] + public void FindByRelativePath_PathOutsideSourceDir_ReturnsDefault() + { + // scanMachinePath is NOT under sourceDirectory (relative path starts with "..") + var sourceDir = $"{RootDir}/src/repo"; + var scanPath = $"{RootDir}/somewhere/else/file.json"; + + var dictionary = new Dictionary + { + { $"{AltRootDir}/agent/s/somewhere/else/file.json", "value" }, + }; + + var result = PathRebasingUtility.FindByRelativePath( + dictionary, sourceDir, scanPath, out var rebaseRoot); + + result.Should().BeNull(); + rebaseRoot.Should().BeNull(); + } + + [TestMethod] + public void FindByRelativePath_EmptyDictionary_ReturnsDefault() + { + var sourceDir = $"{RootDir}/src/repo"; + var scanPath = $"{RootDir}/src/repo/path/file.json"; + + var result = PathRebasingUtility.FindByRelativePath( + [], sourceDir, scanPath, out var rebaseRoot); + + result.Should().BeNull(); + rebaseRoot.Should().BeNull(); + } + + [TestMethod] + public void FindByRelativePath_MultipleEntries_MatchesCorrectOne() + { + var sourceDir = $"{RootDir}/src/repo"; + var scanPath = $"{RootDir}/src/repo/path/B/file.json"; + + var dictionary = new Dictionary + { + { $"{AltRootDir}/agent/path/A/file.json", "wrong" }, + { $"{AltRootDir}/agent/path/B/file.json", "correct" }, + { $"{AltRootDir}/agent/path/C/file.json", "wrong" }, + }; + + var result = PathRebasingUtility.FindByRelativePath( + dictionary, sourceDir, scanPath, out var rebaseRoot); + + result.Should().Be("correct"); + rebaseRoot.Should().Be($"{AltRootDir}/agent/"); + } +} From e84c773491bf5e7a644426acfd87c42a67c41a74 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Thu, 12 Mar 2026 17:13:24 -0700 Subject: [PATCH 12/26] Don't run Integration tests in local build --- .github/workflows/build.yml | 3 ++- CONTRIBUTING.md | 18 ++++++++++++++++-- test/Directory.Build.props | 2 ++ .../DotNetComponentDetectorTests.cs | 8 ++++++++ .../nuget/BinLogProcessorTests.cs | 2 ++ 5 files changed, 30 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d41315362..e0f9ae08c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,7 +43,8 @@ jobs: - name: Run tests # The '--' separator forwards options to the Microsoft Testing Platform runner. # After upgrading to .NET 10 SDK, these can be passed directly without '--'. - run: dotnet test --no-build --configuration Debug -- --coverage --coverage-output-format cobertura + # Override the default Integration-test exclusion from test/Directory.Build.props. + run: dotnet test --no-build --configuration Debug -p:TestingPlatformCommandLineArguments="--coverage --coverage-output-format cobertura" - name: Upload coverage reports to Codecov uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7e2a15933..56fc419fb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,6 +61,20 @@ Analysis rulesets are defined in [analyzers.ruleset](analyzers.ruleset) and vali ### Testing -L0s are defined in `Microsoft.ComponentDetection.*.Tests`. +**Unit tests (L0s)** are defined in `Microsoft.ComponentDetection.*.Tests` and should be fast, isolated, and free of external process calls. -Verification tests are run on the sample projects defined in [microsoft/componentdetection-verification](https://github.com/microsoft/componentdetection-verification). +**Integration tests** spawn real processes (e.g. `dotnet build`, `dotnet restore`) and are therefore slower and environment-dependent. Tag these with `[TestCategory("Integration")]` so they can be filtered during local development. + +By default, `dotnet test` **excludes** Integration tests (configured in `test/Directory.Build.props`). CI runs all tests including Integration (see `.github/workflows/build.yml`). + +To run only Integration tests locally: +```bash +dotnet test -- --filter "TestCategory=Integration" +``` + +To run everything (same as CI): +```bash +dotnet test -p:TestingPlatformCommandLineArguments="" +``` + +**Verification tests** are run on the sample projects defined in [microsoft/componentdetection-verification](https://github.com/microsoft/componentdetection-verification). diff --git a/test/Directory.Build.props b/test/Directory.Build.props index 584193311..641d03916 100644 --- a/test/Directory.Build.props +++ b/test/Directory.Build.props @@ -6,6 +6,8 @@ true true + + $(TestingPlatformCommandLineArguments) --filter TestCategory!=Integration diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs index e6483b995..20eac5430 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/DotNetComponentDetectorTests.cs @@ -862,7 +862,12 @@ public async Task TestDotNetDetectorRebasePaths(string additionalPathSegment) discoveredComponents.Where(component => component.Component.Id == "4.5.6 netstandard2.0 library - DotNet").Should().ContainSingle(); } + // The following tests call RestoreProjectAndGetAssetsPathAsync which spawns a real + // 'dotnet restore' process, making them slow and environment-dependent. They are + // tagged as Integration so they are excluded from default local test runs + // (see test/Directory.Build.props). [TestMethod] + [TestCategory("Integration")] public async Task TestDotNetDetectorSelfContainedWithSelfContainedProperty() { // Emit a self-contained .csproj, restore it, and use the real project.assets.json. @@ -934,6 +939,7 @@ public async Task TestDotNetDetectorSelfContainedWithSelfContainedProperty() } [TestMethod] + [TestCategory("Integration")] public async Task TestDotNetDetectorSelfContainedLibrary() { // A library can also be self-contained when it sets SelfContained + RuntimeIdentifier. @@ -990,6 +996,7 @@ public async Task TestDotNetDetectorSelfContainedLibrary() } [TestMethod] + [TestCategory("Integration")] public async Task TestDotNetDetectorSelfContainedWithPublishAot() { // PublishAot implies native AOT compilation (self-contained). @@ -1050,6 +1057,7 @@ public async Task TestDotNetDetectorSelfContainedWithPublishAot() } [TestMethod] + [TestCategory("Integration")] public async Task TestDotNetDetectorNotSelfContained() { // Framework-dependent app — no RuntimeIdentifier, no SelfContained. diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs index 493e2307a..dc1f202d5 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs @@ -13,6 +13,8 @@ namespace Microsoft.ComponentDetection.Detectors.Tests.NuGet; /// /// Integration tests for that build real MSBuild projects /// to produce binlog files, then parse them to verify extracted project information. +/// Excluded from default local test runs via TestCategory("Integration"); +/// see test/Directory.Build.props. /// [TestClass] [TestCategory("Integration")] From 53992f9eae326cea4f7846900fa5008989f70778 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Mon, 16 Mar 2026 12:54:52 -0700 Subject: [PATCH 13/26] Try to fix linux tests --- .../nuget/BinLogProcessorTests.cs | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs index dc1f202d5..bda2a781a 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs @@ -4,6 +4,7 @@ namespace Microsoft.ComponentDetection.Detectors.Tests.NuGet; using System.Diagnostics; using System.IO; using System.Linq; +using System.Runtime.InteropServices; using System.Threading.Tasks; using AwesomeAssertions; using Microsoft.ComponentDetection.Detectors.NuGet; @@ -25,6 +26,8 @@ public class BinLogProcessorTests public BinLogProcessorTests() => this.processor = new BinLogProcessor(NullLogger.Instance); + private static string CurrentRid => RuntimeInformation.RuntimeIdentifier; + [TestInitialize] public void TestInitialize() { @@ -502,13 +505,13 @@ public async Task SelfContainedProject_ExtractsSelfContained() var projectDir = Path.Combine(this.testDir, "SelfContained"); Directory.CreateDirectory(projectDir); - var content = """ + var content = $""" net8.0 Exe true - win-x64 + {CurrentRid} """; @@ -795,12 +798,12 @@ public async Task TraversalBuildAndPublish_MergesProperties() WriteMinimalProgram(appDir); // Orchestrator: restore, build, then restore with SelfContained (all in one binlog) - var orchestratorContent = """ + var orchestratorContent = $""" - - - + + + """; @@ -848,12 +851,12 @@ public async Task BuildThenPublishSelfContained_MergesSelfContained() // First build (not self-contained), then publish (self-contained), both into same binlog // We use an orchestrator project that invokes MSBuild twice - var orchestratorContent = """ + var orchestratorContent = $""" - + """; @@ -1422,12 +1425,12 @@ public async Task MultiTarget_BuildAndPublishSelfContained_MergesInnerBuilds() WriteMinimalProgram(appDir); // Orchestrator: restore, build, then restore with SelfContained (all in one binlog) - var orchestratorContent = """ + var orchestratorContent = $""" - - - + + + """; @@ -1564,7 +1567,7 @@ public async Task PublishAotWithGlobalSelfContained_BothCaptured() WriteMinimalProgram(projectDir); var binlogPath = Path.Combine(projectDir, "build.binlog"); - await RunDotNetAsync(projectDir, $"msbuild AotAndSC.csproj -t:Restore -bl:\"{binlogPath}\" /p:SelfContained=true /p:RuntimeIdentifier=win-x64"); + await RunDotNetAsync(projectDir, $"msbuild AotAndSC.csproj -t:Restore -bl:\"{binlogPath}\" /p:SelfContained=true /p:RuntimeIdentifier={CurrentRid}"); var results = this.processor.ExtractProjectInfo(binlogPath); From 2cc78bf7fb411cbc978324f52c4ffe8ce059eadb Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Mon, 16 Mar 2026 18:56:10 -0700 Subject: [PATCH 14/26] Add some test diagnostics --- .../nuget/BinLogProcessorTests.cs | 51 +++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs index bda2a781a..c6cfe4071 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs @@ -1,6 +1,7 @@ namespace Microsoft.ComponentDetection.Detectors.Tests.NuGet; using System; +using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; @@ -8,7 +9,7 @@ namespace Microsoft.ComponentDetection.Detectors.Tests.NuGet; using System.Threading.Tasks; using AwesomeAssertions; using Microsoft.ComponentDetection.Detectors.NuGet; -using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; /// @@ -21,10 +22,11 @@ namespace Microsoft.ComponentDetection.Detectors.Tests.NuGet; [TestCategory("Integration")] public class BinLogProcessorTests { + private readonly CapturingLogger logger = new(); private readonly BinLogProcessor processor; private string testDir = null!; - public BinLogProcessorTests() => this.processor = new BinLogProcessor(NullLogger.Instance); + public BinLogProcessorTests() => this.processor = new BinLogProcessor(this.logger); private static string CurrentRid => RuntimeInformation.RuntimeIdentifier; @@ -106,7 +108,14 @@ public async Task SingleTargetProject_ExtractsBasicProperties() var binlogPath = await BuildProjectAsync(projectDir, "SingleTarget.csproj"); var results = this.processor.ExtractProjectInfo(binlogPath); - results.Should().NotBeEmpty(); + // Build diagnostic context that appears in assertion failure messages + var binlogSize = new FileInfo(binlogPath).Length; + var projectPaths = string.Join(", ", results.Select(r => $"'{r.ProjectPath}'")); + var logMessages = this.logger.GetMessages(); + var diag = $"binlog='{binlogPath}' ({binlogSize} bytes), results.Count={results.Count}, " + + $"projectPaths=[{projectPaths}], logMessages=[{logMessages}]"; + + results.Should().NotBeEmpty(diag); var projectInfo = results.First(p => p.ProjectPath != null && @@ -1599,6 +1608,7 @@ private static async Task BuildProjectAsync(string projectDir, string pr throw new InvalidOperationException($"Build did not produce binlog at: {binlogPath}"); } + Console.WriteLine($"[DIAG] Built binlog: {binlogPath} ({new FileInfo(binlogPath).Length} bytes)"); return binlogPath; } @@ -1672,4 +1682,39 @@ private static async Task RunProcessWithEnvAsync( $"Process exited with code {process.ExitCode}.\nCommand: {fileName} {arguments}\nStdout:\n{stdout}\nStderr:\n{stderr}"); } } + + /// + /// Logger that captures messages so they can be included in assertion failure output. + /// + private sealed class CapturingLogger : ILogger + { + private readonly List messages = []; + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + var msg = $"[{logLevel}] {formatter(state, exception)}"; + if (exception != null) + { + msg += $"\n{exception}"; + } + + lock (this.messages) + { + this.messages.Add(msg); + } + } + + public string GetMessages() + { + lock (this.messages) + { + return this.messages.Count == 0 ? "(none)" : string.Join("; ", this.messages); + } + } + } } From 434b5e14b42582f3404decfeef9ce1559bbbe5f5 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Mon, 16 Mar 2026 20:44:21 -0700 Subject: [PATCH 15/26] More logging --- .../nuget/BinLogProcessor.cs | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs index e2ccdfd35..d18ede2fa 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs @@ -47,12 +47,24 @@ public IReadOnlyList ExtractProjectInfo(string binlogPath, s // Maps project instance ID to evaluation ID var projectInstanceToEvaluationMap = new Dictionary(); + // Diagnostic counters + var statusEventCount = 0; + var evalFinishedCount = 0; + var projectStartedCount = 0; + var projectStartedSkippedCount = 0; + var projectFinishedCount = 0; + var projectFinishedAddedCount = 0; + var anyEventCount = 0; + // Hook into status events to capture property evaluations reader.StatusEventRaised += (sender, e) => { + statusEventCount++; if (e?.BuildEventContext?.EvaluationId >= 0 && e is ProjectEvaluationFinishedEventArgs projectEvalArgs) { + evalFinishedCount++; + // Reuse existing project info if one was created during evaluation // (e.g., from EnvironmentVariableReadEventArgs or PropertyInitialValueSetEventArgs) if (!projectInfoByEvaluationId.TryGetValue(e.BuildEventContext.EvaluationId, out var projectInfo)) @@ -68,6 +80,7 @@ public IReadOnlyList ExtractProjectInfo(string binlogPath, s // Hook into project started to map project instance to evaluation and capture project path reader.ProjectStarted += (sender, e) => { + projectStartedCount++; if (e?.BuildEventContext?.EvaluationId >= 0 && e?.BuildEventContext?.ProjectInstanceId >= 0) { @@ -80,6 +93,15 @@ public IReadOnlyList ExtractProjectInfo(string binlogPath, s projectInfo.ProjectPath = rebasePath != null ? rebasePath(e.ProjectFile) : e.ProjectFile; } } + else + { + projectStartedSkippedCount++; + this.logger.LogDebug( + "ProjectStarted skipped: EvalId={EvalId}, InstanceId={InstanceId}, File={File}", + e?.BuildEventContext?.EvaluationId, + e?.BuildEventContext?.ProjectInstanceId, + e?.ProjectFile); + } }; // Hook into message events to capture BinLogFilePath from the initial build messages. @@ -115,22 +137,41 @@ e is BuildMessageEventArgs msg && // Hook into any event to capture property reassignments and item changes during build reader.AnyEventRaised += (sender, e) => { + anyEventCount++; this.HandleBuildEvent(e, projectInstanceToEvaluationMap, projectInfoByEvaluationId, rebasePath); }; // Hook into project finished to collect final project info and establish hierarchy reader.ProjectFinished += (sender, e) => { + projectFinishedCount++; if (e?.BuildEventContext?.ProjectInstanceId >= 0 && projectInstanceToEvaluationMap.TryGetValue(e.BuildEventContext.ProjectInstanceId, out var evaluationId) && projectInfoByEvaluationId.TryGetValue(evaluationId, out var projectInfo) && !string.IsNullOrEmpty(projectInfo.ProjectPath)) { + projectFinishedAddedCount++; this.AddOrMergeProjectInfo(projectInfo, projectInfoByPath); } }; reader.Replay(binlogPath); + + this.logger.LogDebug( + "Binlog replay complete: StatusEvents={StatusEvents}, EvalFinished={EvalFinished}, " + + "ProjectStarted={ProjectStarted} (skipped={ProjectStartedSkipped}), " + + "ProjectFinished={ProjectFinished} (added={ProjectFinishedAdded}), " + + "AnyEvents={AnyEvents}, EvalMapSize={EvalMapSize}, InstanceMapSize={InstanceMapSize}, Results={Results}", + statusEventCount, + evalFinishedCount, + projectStartedCount, + projectStartedSkippedCount, + projectFinishedCount, + projectFinishedAddedCount, + anyEventCount, + projectInfoByEvaluationId.Count, + projectInstanceToEvaluationMap.Count, + projectInfoByPath.Count); } catch (Exception ex) { From 007c76040051fc5a804ccccbc942ff8e674582e0 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Tue, 17 Mar 2026 07:10:42 -0700 Subject: [PATCH 16/26] More logging --- .../nuget/BinLogProcessor.cs | 21 +++++++++- .../nuget/BinLogProcessorTests.cs | 39 ++++++++++++++++++- 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs index d18ede2fa..46a2d9cca 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs @@ -55,11 +55,21 @@ public IReadOnlyList ExtractProjectInfo(string binlogPath, s var projectFinishedCount = 0; var projectFinishedAddedCount = 0; var anyEventCount = 0; + var firstEventTypes = new List(10); // Hook into status events to capture property evaluations reader.StatusEventRaised += (sender, e) => { statusEventCount++; + if (statusEventCount <= 3) + { + this.logger.LogDebug( + "StatusEvent[{Index}]: Type={Type}, EvalId={EvalId}", + statusEventCount, + e?.GetType().FullName, + e?.BuildEventContext?.EvaluationId); + } + if (e?.BuildEventContext?.EvaluationId >= 0 && e is ProjectEvaluationFinishedEventArgs projectEvalArgs) { @@ -138,6 +148,11 @@ e is BuildMessageEventArgs msg && reader.AnyEventRaised += (sender, e) => { anyEventCount++; + if (firstEventTypes.Count < 10) + { + firstEventTypes.Add(e?.GetType().Name ?? "null"); + } + this.HandleBuildEvent(e, projectInstanceToEvaluationMap, projectInfoByEvaluationId, rebasePath); }; @@ -161,7 +176,8 @@ e is BuildMessageEventArgs msg && "Binlog replay complete: StatusEvents={StatusEvents}, EvalFinished={EvalFinished}, " + "ProjectStarted={ProjectStarted} (skipped={ProjectStartedSkipped}), " + "ProjectFinished={ProjectFinished} (added={ProjectFinishedAdded}), " + - "AnyEvents={AnyEvents}, EvalMapSize={EvalMapSize}, InstanceMapSize={InstanceMapSize}, Results={Results}", + "AnyEvents={AnyEvents}, EvalMapSize={EvalMapSize}, InstanceMapSize={InstanceMapSize}, Results={Results}, " + + "FirstEventTypes=[{FirstEventTypes}]", statusEventCount, evalFinishedCount, projectStartedCount, @@ -171,7 +187,8 @@ e is BuildMessageEventArgs msg && anyEventCount, projectInfoByEvaluationId.Count, projectInstanceToEvaluationMap.Count, - projectInfoByPath.Count); + projectInfoByPath.Count, + string.Join(", ", firstEventTypes)); } catch (Exception ex) { diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs index c6cfe4071..da613bc40 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs @@ -43,6 +43,10 @@ public void TestInitialize() { File.Copy(workspaceGlobalJson, Path.Combine(this.testDir, "global.json")); } + + this.logger.Clear(); + var globalJsonStatus = workspaceGlobalJson ?? "NOT FOUND"; + this.logger.Log(LogLevel.Information, default, $"TestInit: cwd={Directory.GetCurrentDirectory()}, globalJson={globalJsonStatus}, testDir={this.testDir}", null, (s, _) => s); } private static string? FindWorkspaceGlobalJsonPath() @@ -1600,12 +1604,16 @@ private static void WriteMinimalProgram(string directory) private static async Task BuildProjectAsync(string projectDir, string projectFile) { + // Log SDK version for diagnostics (helps identify version mismatches on CI) + var sdkVersionResult = await RunProcessCaptureAsync(projectDir, "dotnet", "--version"); + Console.WriteLine($"[DIAG] dotnet --version in {projectDir}: {sdkVersionResult.Trim()}"); + var binlogPath = Path.Combine(projectDir, "build.binlog"); await RunDotNetAsync(projectDir, $"build \"{projectFile}\" -bl:\"{binlogPath}\" /p:UseAppHost=false"); if (!File.Exists(binlogPath)) { - throw new InvalidOperationException($"Build did not produce binlog at: {binlogPath}"); + throw new InvalidOperationException($"Build did not produce binlog at: {binlogPath}. SDK={sdkVersionResult.Trim()}"); } Console.WriteLine($"[DIAG] Built binlog: {binlogPath} ({new FileInfo(binlogPath).Length} bytes)"); @@ -1618,6 +1626,27 @@ private static async Task RunDotNetAsync(string workingDirectory, string argumen await RunProcessAsync(workingDirectory, "dotnet", arguments); } + private static async Task RunProcessCaptureAsync(string workingDirectory, string fileName, string arguments) + { + var psi = new ProcessStartInfo + { + FileName = fileName, + Arguments = arguments, + WorkingDirectory = workingDirectory, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + + using var process = Process.Start(psi) + ?? throw new InvalidOperationException($"Failed to start: {fileName} {arguments}"); + + var stdout = await process.StandardOutput.ReadToEndAsync(); + await process.WaitForExitAsync(); + return stdout; + } + private static async Task RunProcessAsync(string workingDirectory, string fileName, string arguments) { var psi = new ProcessStartInfo @@ -1716,5 +1745,13 @@ public string GetMessages() return this.messages.Count == 0 ? "(none)" : string.Join("; ", this.messages); } } + + public void Clear() + { + lock (this.messages) + { + this.messages.Clear(); + } + } } } From 4563877ceea76130bfdbba37706bd13992ea5d25 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Tue, 17 Mar 2026 10:31:21 -0700 Subject: [PATCH 17/26] Save log file --- .../nuget/BinLogProcessorTests.cs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs index da613bc40..ce7ac2e40 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs @@ -28,12 +28,20 @@ public class BinLogProcessorTests public BinLogProcessorTests() => this.processor = new BinLogProcessor(this.logger); + public TestContext TestContext { get; set; } = null!; + private static string CurrentRid => RuntimeInformation.RuntimeIdentifier; [TestInitialize] public void TestInitialize() { - this.testDir = Path.Combine(Path.GetTempPath(), "BinLogProcessorTests", Guid.NewGuid().ToString("N")); + // Use test name as directory so binlogs are easy to find after failures. + this.testDir = Path.Combine(Path.GetTempPath(), "BinLogProcessorTests", this.TestContext.TestName!); + if (Directory.Exists(this.testDir)) + { + Directory.Delete(this.testDir, recursive: true); + } + Directory.CreateDirectory(this.testDir); // Copy the workspace global.json so temp projects use the same SDK as the repo. @@ -76,6 +84,12 @@ public void TestInitialize() [TestCleanup] public void TestCleanup() { + if (this.TestContext.CurrentTestOutcome != UnitTestOutcome.Passed) + { + Console.WriteLine($"[DIAG] Test failed — preserving test artifacts at: {this.testDir}"); + return; + } + try { if (Directory.Exists(this.testDir)) From d9373da6d8f32afc6645d68df95ff1429aaa4abc Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Tue, 17 Mar 2026 11:18:28 -0700 Subject: [PATCH 18/26] Add exception handling to binlog reading --- .../nuget/BinLogProcessor.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs index 46a2d9cca..44612629b 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs @@ -41,6 +41,12 @@ public IReadOnlyList ExtractProjectInfo(string binlogPath, s { var reader = new BinLogReader(); + reader.OnException += ex => + this.logger.LogWarning(ex, "BinLogReader.OnException during replay"); + + reader.RecoverableReadError += args => + this.logger.LogDebug("BinLogReader.RecoverableReadError: {Message}", args.ErrorType); + // Maps evaluation ID to MSBuildProjectInfo being populated var projectInfoByEvaluationId = new Dictionary(); From 86ec06f7b84c5eed515f475ff8a71bc2e0830f27 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Tue, 17 Mar 2026 17:16:30 -0700 Subject: [PATCH 19/26] Workaround binlogger issue with older binlogs Add diagnostics as well. --- .../nuget/BinLogProcessor.cs | 86 ++++--------------- .../nuget/BinLogProcessorTests.cs | 77 ++++++++--------- 2 files changed, 52 insertions(+), 111 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs index 44612629b..5439d2111 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs @@ -39,13 +39,20 @@ public IReadOnlyList ExtractProjectInfo(string binlogPath, s try { + // Workaround: ensure StructuredLogger's resource strings are loaded + // before replay. Some binlogs lack the CurrentUICulture message that + // triggers Strings.Initialize(), leaving format strings null and causing + // Replay() to silently abort. Fixed upstream: + // https://github.com/KirillOsenkov/MSBuildStructuredLog/issues/936 + Strings.Initialize(); + var reader = new BinLogReader(); + // Surface errors that BinLogReader catches internally during Replay(). + // Without this handler, exceptions from Read() are silently swallowed + // and replay stops early, producing truncated results. reader.OnException += ex => - this.logger.LogWarning(ex, "BinLogReader.OnException during replay"); - - reader.RecoverableReadError += args => - this.logger.LogDebug("BinLogReader.RecoverableReadError: {Message}", args.ErrorType); + this.logger.LogWarning(ex, "BinLogReader encountered an error during replay of {BinlogPath}", binlogPath); // Maps evaluation ID to MSBuildProjectInfo being populated var projectInfoByEvaluationId = new Dictionary(); @@ -53,36 +60,12 @@ public IReadOnlyList ExtractProjectInfo(string binlogPath, s // Maps project instance ID to evaluation ID var projectInstanceToEvaluationMap = new Dictionary(); - // Diagnostic counters - var statusEventCount = 0; - var evalFinishedCount = 0; - var projectStartedCount = 0; - var projectStartedSkippedCount = 0; - var projectFinishedCount = 0; - var projectFinishedAddedCount = 0; - var anyEventCount = 0; - var firstEventTypes = new List(10); - // Hook into status events to capture property evaluations reader.StatusEventRaised += (sender, e) => { - statusEventCount++; - if (statusEventCount <= 3) - { - this.logger.LogDebug( - "StatusEvent[{Index}]: Type={Type}, EvalId={EvalId}", - statusEventCount, - e?.GetType().FullName, - e?.BuildEventContext?.EvaluationId); - } - if (e?.BuildEventContext?.EvaluationId >= 0 && e is ProjectEvaluationFinishedEventArgs projectEvalArgs) { - evalFinishedCount++; - - // Reuse existing project info if one was created during evaluation - // (e.g., from EnvironmentVariableReadEventArgs or PropertyInitialValueSetEventArgs) if (!projectInfoByEvaluationId.TryGetValue(e.BuildEventContext.EvaluationId, out var projectInfo)) { projectInfo = new MSBuildProjectInfo(); @@ -96,36 +79,22 @@ public IReadOnlyList ExtractProjectInfo(string binlogPath, s // Hook into project started to map project instance to evaluation and capture project path reader.ProjectStarted += (sender, e) => { - projectStartedCount++; if (e?.BuildEventContext?.EvaluationId >= 0 && e?.BuildEventContext?.ProjectInstanceId >= 0) { projectInstanceToEvaluationMap[e.BuildEventContext.ProjectInstanceId] = e.BuildEventContext.EvaluationId; - // Set the project path on the MSBuildProjectInfo if (!string.IsNullOrEmpty(e.ProjectFile) && projectInfoByEvaluationId.TryGetValue(e.BuildEventContext.EvaluationId, out var projectInfo)) { projectInfo.ProjectPath = rebasePath != null ? rebasePath(e.ProjectFile) : e.ProjectFile; } } - else - { - projectStartedSkippedCount++; - this.logger.LogDebug( - "ProjectStarted skipped: EvalId={EvalId}, InstanceId={InstanceId}, File={File}", - e?.BuildEventContext?.EvaluationId, - e?.BuildEventContext?.ProjectInstanceId, - e?.ProjectFile); - } }; // Hook into message events to capture BinLogFilePath from the initial build messages. // MSBuild's BinaryLogger writes "BinLogFilePath=" as a BuildMessageEventArgs - // with SenderName "BinaryLogger" at the start of the log. This arrives before any - // evaluation or project events, so we can compute the rebase function here and apply - // it to all subsequent path-valued properties. - // https://github.com/dotnet/msbuild/blob/7d73e8e9074fe9a4420e38cd22d45645b28a11f7/src/Build/Logging/BinaryLogger/BinaryLogger.cs#L473 + // with SenderName "BinaryLogger" at the start of the log. reader.MessageRaised += (sender, e) => { if (rebasePath == null && @@ -153,52 +122,27 @@ e is BuildMessageEventArgs msg && // Hook into any event to capture property reassignments and item changes during build reader.AnyEventRaised += (sender, e) => { - anyEventCount++; - if (firstEventTypes.Count < 10) - { - firstEventTypes.Add(e?.GetType().Name ?? "null"); - } - this.HandleBuildEvent(e, projectInstanceToEvaluationMap, projectInfoByEvaluationId, rebasePath); }; // Hook into project finished to collect final project info and establish hierarchy reader.ProjectFinished += (sender, e) => { - projectFinishedCount++; if (e?.BuildEventContext?.ProjectInstanceId >= 0 && projectInstanceToEvaluationMap.TryGetValue(e.BuildEventContext.ProjectInstanceId, out var evaluationId) && projectInfoByEvaluationId.TryGetValue(evaluationId, out var projectInfo) && !string.IsNullOrEmpty(projectInfo.ProjectPath)) { - projectFinishedAddedCount++; this.AddOrMergeProjectInfo(projectInfo, projectInfoByPath); } }; reader.Replay(binlogPath); - - this.logger.LogDebug( - "Binlog replay complete: StatusEvents={StatusEvents}, EvalFinished={EvalFinished}, " + - "ProjectStarted={ProjectStarted} (skipped={ProjectStartedSkipped}), " + - "ProjectFinished={ProjectFinished} (added={ProjectFinishedAdded}), " + - "AnyEvents={AnyEvents}, EvalMapSize={EvalMapSize}, InstanceMapSize={InstanceMapSize}, Results={Results}, " + - "FirstEventTypes=[{FirstEventTypes}]", - statusEventCount, - evalFinishedCount, - projectStartedCount, - projectStartedSkippedCount, - projectFinishedCount, - projectFinishedAddedCount, - anyEventCount, - projectInfoByEvaluationId.Count, - projectInstanceToEvaluationMap.Count, - projectInfoByPath.Count, - string.Join(", ", firstEventTypes)); } - catch (Exception ex) + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { - this.logger.LogWarning(ex, "Error parsing binlog: {BinlogPath}", binlogPath); + // Expected for missing, locked, or inaccessible binlog files. + this.logger.LogWarning(ex, "Could not read binlog: {BinlogPath}", binlogPath); } return [.. projectInfoByPath.Values]; diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs index ce7ac2e40..2e1a3337c 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs @@ -17,6 +17,14 @@ namespace Microsoft.ComponentDetection.Detectors.Tests.NuGet; /// to produce binlog files, then parse them to verify extracted project information. /// Excluded from default local test runs via TestCategory("Integration"); /// see test/Directory.Build.props. +/// +/// Diagnosing failures: When a test fails, its artifacts (project files and +/// .binlog files) are copied to the TestResults directory under a +/// subdirectory named after the test. In CI, these are uploaded as pipeline artifacts. +/// Inspect .binlog files with +/// MSBuild Structured Log Viewer. +/// Logger output captured during the test is included in assertion failure messages. +/// /// [TestClass] [TestCategory("Integration")] @@ -53,8 +61,6 @@ public void TestInitialize() } this.logger.Clear(); - var globalJsonStatus = workspaceGlobalJson ?? "NOT FOUND"; - this.logger.Log(LogLevel.Information, default, $"TestInit: cwd={Directory.GetCurrentDirectory()}, globalJson={globalJsonStatus}, testDir={this.testDir}", null, (s, _) => s); } private static string? FindWorkspaceGlobalJsonPath() @@ -86,8 +92,17 @@ public void TestCleanup() { if (this.TestContext.CurrentTestOutcome != UnitTestOutcome.Passed) { - Console.WriteLine($"[DIAG] Test failed — preserving test artifacts at: {this.testDir}"); - return; + // Copy artifacts to TestResults for CI upload, then still clean up temp. + try + { + var destDir = Path.Combine(this.TestContext.TestResultsDirectory!, nameof(BinLogProcessorTests), this.TestContext.TestName!); + CopyDirectory(this.testDir, destDir); + Console.WriteLine($"Test artifacts copied to: {destDir}"); + } + catch (Exception ex) + { + Console.WriteLine($"Failed to copy test artifacts: {ex.Message}"); + } } try @@ -126,14 +141,8 @@ public async Task SingleTargetProject_ExtractsBasicProperties() var binlogPath = await BuildProjectAsync(projectDir, "SingleTarget.csproj"); var results = this.processor.ExtractProjectInfo(binlogPath); - // Build diagnostic context that appears in assertion failure messages - var binlogSize = new FileInfo(binlogPath).Length; - var projectPaths = string.Join(", ", results.Select(r => $"'{r.ProjectPath}'")); - var logMessages = this.logger.GetMessages(); - var diag = $"binlog='{binlogPath}' ({binlogSize} bytes), results.Count={results.Count}, " + - $"projectPaths=[{projectPaths}], logMessages=[{logMessages}]"; - - results.Should().NotBeEmpty(diag); + results.Should().NotBeEmpty( + $"binlog='{binlogPath}' ({new FileInfo(binlogPath).Length} bytes), log=[{this.logger.GetMessages()}]"); var projectInfo = results.First(p => p.ProjectPath != null && @@ -1606,6 +1615,21 @@ public async Task PublishAotWithGlobalSelfContained_BothCaptured() projectInfo.SelfContained.Should().Be(true); } + private static void CopyDirectory(string sourceDir, string destDir) + { + Directory.CreateDirectory(destDir); + + foreach (var file in Directory.GetFiles(sourceDir)) + { + File.Copy(file, Path.Combine(destDir, Path.GetFileName(file)), overwrite: true); + } + + foreach (var dir in Directory.GetDirectories(sourceDir)) + { + CopyDirectory(dir, Path.Combine(destDir, Path.GetFileName(dir))); + } + } + private static void WriteFile(string directory, string fileName, string content) { File.WriteAllText(Path.Combine(directory, fileName), content); @@ -1618,49 +1642,22 @@ private static void WriteMinimalProgram(string directory) private static async Task BuildProjectAsync(string projectDir, string projectFile) { - // Log SDK version for diagnostics (helps identify version mismatches on CI) - var sdkVersionResult = await RunProcessCaptureAsync(projectDir, "dotnet", "--version"); - Console.WriteLine($"[DIAG] dotnet --version in {projectDir}: {sdkVersionResult.Trim()}"); - var binlogPath = Path.Combine(projectDir, "build.binlog"); await RunDotNetAsync(projectDir, $"build \"{projectFile}\" -bl:\"{binlogPath}\" /p:UseAppHost=false"); if (!File.Exists(binlogPath)) { - throw new InvalidOperationException($"Build did not produce binlog at: {binlogPath}. SDK={sdkVersionResult.Trim()}"); + throw new InvalidOperationException($"Build did not produce binlog at: {binlogPath}"); } - Console.WriteLine($"[DIAG] Built binlog: {binlogPath} ({new FileInfo(binlogPath).Length} bytes)"); return binlogPath; } private static async Task RunDotNetAsync(string workingDirectory, string arguments) { - // dotnet build already includes restore by default, no need for separate restore await RunProcessAsync(workingDirectory, "dotnet", arguments); } - private static async Task RunProcessCaptureAsync(string workingDirectory, string fileName, string arguments) - { - var psi = new ProcessStartInfo - { - FileName = fileName, - Arguments = arguments, - WorkingDirectory = workingDirectory, - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - }; - - using var process = Process.Start(psi) - ?? throw new InvalidOperationException($"Failed to start: {fileName} {arguments}"); - - var stdout = await process.StandardOutput.ReadToEndAsync(); - await process.WaitForExitAsync(); - return stdout; - } - private static async Task RunProcessAsync(string workingDirectory, string fileName, string arguments) { var psi = new ProcessStartInfo From e90456b9966cd317f2b01bb0c473526687a36588 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Tue, 17 Mar 2026 17:52:42 -0700 Subject: [PATCH 20/26] Address feedback --- .../dotnet/DotNetProjectInfoProvider.cs | 1 + .../dotnet/PathRebasingUtility.cs | 6 ++++-- .../nuget/MSBuildBinaryLogComponentDetector.cs | 12 +++++++++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetProjectInfoProvider.cs b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetProjectInfoProvider.cs index e94d01d31..bf2755ea6 100644 --- a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetProjectInfoProvider.cs +++ b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetProjectInfoProvider.cs @@ -237,6 +237,7 @@ private bool IsApplication(string assemblyPath) if (!string.IsNullOrWhiteSpace(sdkVersion)) { + this.sdkVersionCache[projectDirectory] = sdkVersion; var globalJsonComponent = new DetectedComponent(new DotNetComponent(sdkVersion)); var recorder = componentRecorder.CreateSingleFileComponentRecorder(globalJsonPath); recorder.RegisterUsage(globalJsonComponent, isExplicitReferencedDependency: true); diff --git a/src/Microsoft.ComponentDetection.Detectors/dotnet/PathRebasingUtility.cs b/src/Microsoft.ComponentDetection.Detectors/dotnet/PathRebasingUtility.cs index 69221e3b0..fb1361e59 100644 --- a/src/Microsoft.ComponentDetection.Detectors/dotnet/PathRebasingUtility.cs +++ b/src/Microsoft.ComponentDetection.Detectors/dotnet/PathRebasingUtility.cs @@ -54,7 +54,7 @@ internal static class PathRebasingUtility artifactPath = NormalizeDirectory(artifactPath)!; // Nothing to do if the paths are the same (no rebasing needed). - if (artifactPath.Equals(sourceDirectoryBasedPath, StringComparison.Ordinal)) + if (artifactPath.Equals(sourceDirectoryBasedPath, StringComparison.OrdinalIgnoreCase)) { return null; } @@ -63,7 +63,9 @@ internal static class PathRebasingUtility var sourceDirectoryRelativePath = NormalizeDirectory(Path.GetRelativePath(sourceDirectory, sourceDirectoryBasedPath))!; // If the artifact path has the same relative portion, extract the root prefix. - if (artifactPath.EndsWith(sourceDirectoryRelativePath, StringComparison.Ordinal)) + // Use case-insensitive comparison: Windows paths are case-insensitive, and on + // Linux the paths will naturally have consistent casing. + if (artifactPath.EndsWith(sourceDirectoryRelativePath, StringComparison.OrdinalIgnoreCase)) { return artifactPath[..^sourceDirectoryRelativePath.Length]; } diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs index 25b9dd70c..48ca6a99a 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs @@ -169,7 +169,7 @@ protected override async Task> OnPrepareDetectionAsy { // Collect all requests and sort them so binlogs are processed first // This ensures we have project info available when processing assets files - var allRequests = await processRequests.ToList(); + var allRequests = await processRequests.ToList().ToTask(cancellationToken); this.Logger.LogDebug("Preparing detection: collected {Count} files", allRequests.Count); @@ -291,10 +291,12 @@ private void IndexProjectInfo(MSBuildProjectInfo projectInfo, List asset // Index by assets file path for lookup when processing lock files. // Use AddOrUpdate+MergeWith so that multiple binlogs for the same project // (e.g., build and publish passes) form a superset rather than keeping only the first. + // Normalize to forward slashes so lookup from OS-native ComponentStream.Location matches. if (!string.IsNullOrEmpty(projectInfo.ProjectAssetsFile)) { + var normalizedAssetsPath = PathRebasingUtility.NormalizePath(projectInfo.ProjectAssetsFile); this.projectInfoByAssetsFile.AddOrUpdate( - projectInfo.ProjectAssetsFile, + normalizedAssetsPath, _ => projectInfo, (_, existing) => existing.MergeWith(projectInfo)); assetsFilesFound.Add(projectInfo.ProjectAssetsFile); @@ -455,7 +457,11 @@ private async Task ProcessAssetsFileAsync(ProcessRequest processRequest, Cancell /// private MSBuildProjectInfo? FindProjectInfoForAssetsFile(string assetsFilePath) { - this.projectInfoByAssetsFile.TryGetValue(assetsFilePath, out var projectInfo); + // Normalize to forward slashes for consistent lookup. + // BinLogProcessor stores keys with forward slashes, but ComponentStream.Location + // uses OS-native separators (backslashes on Windows). + var normalizedPath = PathRebasingUtility.NormalizePath(assetsFilePath); + this.projectInfoByAssetsFile.TryGetValue(normalizedPath, out var projectInfo); return projectInfo; } From 3bc20679c4bc3e50c7f578753c3fea3c9e6933cd Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Tue, 17 Mar 2026 20:11:06 -0700 Subject: [PATCH 21/26] Address feedback --- .../dotnet/PathRebasingUtility.cs | 13 +++- .../MSBuildBinaryLogComponentDetector.cs | 59 ++++++++++--------- 2 files changed, 41 insertions(+), 31 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/dotnet/PathRebasingUtility.cs b/src/Microsoft.ComponentDetection.Detectors/dotnet/PathRebasingUtility.cs index fb1361e59..7b3b24c1a 100644 --- a/src/Microsoft.ComponentDetection.Detectors/dotnet/PathRebasingUtility.cs +++ b/src/Microsoft.ComponentDetection.Detectors/dotnet/PathRebasingUtility.cs @@ -16,6 +16,13 @@ namespace Microsoft.ComponentDetection.Detectors.DotNet; /// internal static class PathRebasingUtility { + /// + /// OS-aware string comparison for filesystem paths. + /// Case-insensitive on Windows, case-sensitive on Linux/macOS. + /// + internal static readonly StringComparison PathComparison = + OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + /// /// Normalizes a path by replacing backslashes with forward slashes. /// Windows accepts / as a separator, so forward-slash normalization works everywhere. @@ -54,7 +61,7 @@ internal static class PathRebasingUtility artifactPath = NormalizeDirectory(artifactPath)!; // Nothing to do if the paths are the same (no rebasing needed). - if (artifactPath.Equals(sourceDirectoryBasedPath, StringComparison.OrdinalIgnoreCase)) + if (artifactPath.Equals(sourceDirectoryBasedPath, PathComparison)) { return null; } @@ -65,7 +72,7 @@ internal static class PathRebasingUtility // If the artifact path has the same relative portion, extract the root prefix. // Use case-insensitive comparison: Windows paths are case-insensitive, and on // Linux the paths will naturally have consistent casing. - if (artifactPath.EndsWith(sourceDirectoryRelativePath, StringComparison.OrdinalIgnoreCase)) + if (artifactPath.EndsWith(sourceDirectoryRelativePath, PathComparison)) { return artifactPath[..^sourceDirectoryRelativePath.Length]; } @@ -142,7 +149,7 @@ internal static string RebasePath(string path, string originalRoot, string newRo foreach (var kvp in dictionary) { var normalizedKey = NormalizePath(kvp.Key); - if (normalizedKey.EndsWith(relativePath, StringComparison.OrdinalIgnoreCase)) + if (normalizedKey.EndsWith(relativePath, PathComparison)) { // Derive the build-machine root from this match. rebaseRoot = normalizedKey[..^relativePath.Length]; diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs index 48ca6a99a..f622a853e 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs @@ -3,10 +3,8 @@ namespace Microsoft.ComponentDetection.Detectors.NuGet; using System; using System.Collections.Concurrent; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Reactive.Linq; -using System.Reactive.Threading.Tasks; using System.Threading; using System.Threading.Tasks; using global::NuGet.Frameworks; @@ -167,27 +165,30 @@ protected override async Task> OnPrepareDetectionAsy IDictionary detectorArgs, CancellationToken cancellationToken = default) { - // Collect all requests and sort them so binlogs are processed first - // This ensures we have project info available when processing assets files - var allRequests = await processRequests.ToList().ToTask(cancellationToken); - - this.Logger.LogDebug("Preparing detection: collected {Count} files", allRequests.Count); - - // Separate binlogs and assets files - var binlogRequests = allRequests - .Where(r => r.ComponentStream.Location.EndsWith(".binlog", StringComparison.OrdinalIgnoreCase)) - .ToList(); - - var assetsRequests = allRequests - .Where(r => r.ComponentStream.Location.EndsWith("project.assets.json", StringComparison.OrdinalIgnoreCase)) - .ToList(); - - this.Logger.LogDebug("Found {BinlogCount} binlog files and {AssetsCount} assets files", binlogRequests.Count, assetsRequests.Count); + // Process binlogs inline as they arrive (synchronous) and buffer only assets files. + // This avoids materializing the entire observable into memory while still ensuring + // all binlog project info is available before any assets file is processed. + var assetsRequests = new List(); + var binlogCount = 0; + + await processRequests.ForEachAsync( + request => + { + if (request.ComponentStream.Location.EndsWith(".binlog", StringComparison.OrdinalIgnoreCase)) + { + this.ProcessBinlogFile(request); + binlogCount++; + } + else if (request.ComponentStream.Location.EndsWith("project.assets.json", StringComparison.OrdinalIgnoreCase)) + { + assetsRequests.Add(request); + } + }, + cancellationToken); - // Return binlogs first, then assets files - var orderedRequests = binlogRequests.Concat(assetsRequests); + this.Logger.LogDebug("Processed {BinlogCount} binlog files, found {AssetsCount} assets files", binlogCount, assetsRequests.Count); - return orderedRequests.ToObservable(); + return assetsRequests.ToObservable(); } /// @@ -196,13 +197,8 @@ protected override async Task OnFileFoundAsync( IDictionary detectorArgs, CancellationToken cancellationToken = default) { - var fileExtension = Path.GetExtension(processRequest.ComponentStream.Location); - - if (fileExtension.Equals(".binlog", StringComparison.OrdinalIgnoreCase)) - { - this.ProcessBinlogFile(processRequest); - } - else if (processRequest.ComponentStream.Location.EndsWith("project.assets.json", StringComparison.OrdinalIgnoreCase)) + // Binlogs are already processed in OnPrepareDetectionAsync; only assets files reach here. + if (processRequest.ComponentStream.Location.EndsWith("project.assets.json", StringComparison.OrdinalIgnoreCase)) { await this.ProcessAssetsFileAsync(processRequest, cancellationToken); } @@ -607,4 +603,11 @@ private async Task ProcessLockFileFallbackAsync(LockFile lockFile, string locati // This matches DotNetComponentDetector's behavior for the fallback path await this.projectInfoProvider.RegisterDotNetComponentsAsync(lockFile, location, this.ComponentRecorder, cancellationToken); } + + /// + protected override Task OnDetectionFinishedAsync() + { + this.projectInfoByAssetsFile.Clear(); + return Task.CompletedTask; + } } From d7d76648430d7d59e9fe4ef9bfdbe42f5c9fc118 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Wed, 18 Mar 2026 20:26:39 -0700 Subject: [PATCH 22/26] Address feedback --- .../MSBuildBinaryLogComponentDetector.cs | 123 +++++++++--------- .../MSBuildBinaryLogComponentDetectorTests.cs | 43 ++++-- 2 files changed, 97 insertions(+), 69 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs index f622a853e..53087f396 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs @@ -59,18 +59,14 @@ public class MSBuildBinaryLogComponentDetector : FileComponentDetector, IExperim /// Stores project information extracted from binlogs, keyed by assets file path. /// /// - /// All binlog files are processed before any assets files (guaranteed by ), - /// so by the time an assets file is processed this dictionary contains the merged superset of project - /// info from every binlog that referenced that project. This is intentional: a repository may produce - /// separate binlogs for different build passes (e.g., build vs. publish). Properties like - /// or are - /// typically only set in the publish pass, so merging ensures those properties are available when - /// processing the shared project.assets.json. - /// - /// Memory impact: each is roughly 15–16 KB (including inner builds - /// and PackageReference/PackageDownload dictionaries). A repository with 100K projects would use - /// approximately 1.5 GB for project info storage alone — significant but proportional to the - /// multi-GB binlog files that such a repository would produce. + /// All binlog files are processed eagerly in before + /// any assets file reaches , so this dictionary is fully + /// populated by the time assets files are processed. When multiple binlogs reference the + /// same project (e.g., build and publish passes), their info is merged via + /// so that properties like + /// or + /// (typically only set in the publish pass) are available when processing the shared + /// project.assets.json. /// private readonly ConcurrentDictionary projectInfoByAssetsFile = new(StringComparer.OrdinalIgnoreCase); @@ -143,7 +139,7 @@ internal MSBuildBinaryLogComponentDetector( public override IEnumerable Categories => [Enum.GetName(typeof(DetectorClass), DetectorClass.NuGet)!]; /// - public override IList SearchPatterns { get; } = ["*.binlog", "project.assets.json"]; + public override IList SearchPatterns { get; } = ["project.assets.json"]; /// public override IEnumerable SupportedComponentTypes { get; } = [ComponentType.NuGet, ComponentType.DotNet]; @@ -160,49 +156,38 @@ public override Task ExecuteDetectorAsync(ScanRequ } /// - protected override async Task> OnPrepareDetectionAsync( + protected override Task> OnPrepareDetectionAsync( IObservable processRequests, IDictionary detectorArgs, CancellationToken cancellationToken = default) { - // Process binlogs inline as they arrive (synchronous) and buffer only assets files. - // This avoids materializing the entire observable into memory while still ensuring - // all binlog project info is available before any assets file is processed. - var assetsRequests = new List(); - var binlogCount = 0; - - await processRequests.ForEachAsync( - request => - { - if (request.ComponentStream.Location.EndsWith(".binlog", StringComparison.OrdinalIgnoreCase)) - { - this.ProcessBinlogFile(request); - binlogCount++; - } - else if (request.ComponentStream.Location.EndsWith("project.assets.json", StringComparison.OrdinalIgnoreCase)) - { - assetsRequests.Add(request); - } - }, - cancellationToken); - - this.Logger.LogDebug("Processed {BinlogCount} binlog files, found {AssetsCount} assets files", binlogCount, assetsRequests.Count); + // Phase 1: Eagerly discover and process all binlog files. + // This completes before OnPrepareDetectionAsync returns, guaranteeing all project info + // is indexed before any assets file reaches OnFileFoundAsync. + var binlogStreams = this.ComponentStreamEnumerableFactory.GetComponentStreams( + this.CurrentScanRequest.SourceDirectory, + ["*.binlog"], + this.CurrentScanRequest.DirectoryExclusionPredicate); + + foreach (var stream in binlogStreams) + { + this.ProcessBinlogFile(stream.Location); + } - return assetsRequests.ToObservable(); + // Phase 2: Return the original observable unchanged. + // SearchPatterns only includes project.assets.json, so the observable already + // contains only assets files — no filtering needed. + return Task.FromResult(processRequests); } /// - protected override async Task OnFileFoundAsync( + protected override Task OnFileFoundAsync( ProcessRequest processRequest, IDictionary detectorArgs, - CancellationToken cancellationToken = default) - { - // Binlogs are already processed in OnPrepareDetectionAsync; only assets files reach here. - if (processRequest.ComponentStream.Location.EndsWith("project.assets.json", StringComparison.OrdinalIgnoreCase)) - { - await this.ProcessAssetsFileAsync(processRequest, cancellationToken); - } - } + CancellationToken cancellationToken = default) => + + // Only assets files reach here; binlogs are processed in OnPrepareDetectionAsync. + this.ProcessAssetsFileAsync(processRequest, cancellationToken); /// /// Determines whether a project should be classified as development-only. @@ -237,6 +222,34 @@ private static bool IsDevelopmentOnlyProject(MSBuildProjectInfo projectInfo) => return null; } + /// + /// Maps the MSBuild OutputType property to "application" or "library". + /// Returns null for empty/unknown values rather than guessing. + /// + private static string? GetTargetType(string? outputType) + { + if (string.IsNullOrEmpty(outputType)) + { + return null; + } + + // https://learn.microsoft.com/dotnet/csharp/language-reference/compiler-options/output#outputtype + if (outputType.Equals("Exe", StringComparison.OrdinalIgnoreCase) || + outputType.Equals("WinExe", StringComparison.OrdinalIgnoreCase) || + outputType.Equals("AppContainerExe", StringComparison.OrdinalIgnoreCase)) + { + return "application"; + } + + if (outputType.Equals("Library", StringComparison.OrdinalIgnoreCase) || + outputType.Equals("Module", StringComparison.OrdinalIgnoreCase)) + { + return "library"; + } + + return null; + } + /// /// Determines if a project is self-contained using MSBuild properties from the binlog. /// Reads the SelfContained and PublishAot properties directly. @@ -244,9 +257,8 @@ private static bool IsDevelopmentOnlyProject(MSBuildProjectInfo projectInfo) => private static bool IsSelfContainedFromProjectInfo(MSBuildProjectInfo projectInfo) => projectInfo.SelfContained == true || projectInfo.PublishAot == true; - private void ProcessBinlogFile(ProcessRequest processRequest) + private void ProcessBinlogFile(string binlogPath) { - var binlogPath = processRequest.ComponentStream.Location; var assetsFilesFound = new List(); try @@ -340,16 +352,11 @@ private void RegisterDotNetComponent(MSBuildProjectInfo projectInfo, LockFile? l var singleFileComponentRecorder = this.ComponentRecorder.CreateSingleFileComponentRecorder(projectInfo.ProjectPath); - // Determine target type from OutputType property - // This is equivalent to DotNetComponentDetector's IsApplication check via PE headers - string? targetType = null; - if (!string.IsNullOrEmpty(projectInfo.OutputType)) - { - targetType = projectInfo.OutputType.Equals("Exe", StringComparison.OrdinalIgnoreCase) || - projectInfo.OutputType.Equals("WinExe", StringComparison.OrdinalIgnoreCase) - ? "application" - : "library"; - } + // Determine target type from OutputType property. + // Known application types: Exe, WinExe, AppContainerExe + // Known library types: Library, Module + // Unknown values are left as null (don't assume). + var targetType = GetTargetType(projectInfo.OutputType); // Primary self-contained check from binlog properties (SelfContained, PublishAot) var isSelfContainedFromBinlog = IsSelfContainedFromProjectInfo(projectInfo); diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs index 9ac2d2a29..c1e2c497b 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs @@ -675,18 +675,23 @@ private static ITaskItem CreateTaskItemMock(string itemSpec, bool isDevelopmentD var recorder = new ComponentRecorder(); - var requests = new[] - { - CreateProcessRequest(recorder, binlogPath, "fake-binlog-content"), - CreateProcessRequest(recorder, assetsLocation, assetsJson), - }; + // Binlogs are discovered via ComponentStreamEnumerableFactory in OnPrepareDetectionAsync. + streamFactoryMock + .Setup(x => x.GetComponentStreams( + It.IsAny(), + It.Is>(p => p.Any(s => s == "*.binlog")), + It.IsAny(), + It.IsAny())) + .Returns([CreateComponentStream(binlogPath)]); + // Walker observable only returns assets files (SearchPatterns = ["project.assets.json"]). + var assetsRequest = CreateProcessRequest(recorder, assetsLocation, assetsJson); walkerMock .Setup(x => x.GetFilteredComponentStreamObservable( It.IsAny(), It.IsAny>(), It.IsAny())) - .Returns(requests.ToObservable()); + .Returns(new[] { assetsRequest }.ToObservable()); var scanRequest = new ScanRequest( new DirectoryInfo(Path.GetTempPath()), @@ -744,18 +749,26 @@ private static ITaskItem CreateTaskItemMock(string itemSpec, bool isDevelopmentD var recorder = new ComponentRecorder(); - // Build process requests: one per binlog file, then the assets file - var requests = binlogProjectInfos.Keys - .Select(binlogPath => CreateProcessRequest(recorder, binlogPath, "fake-binlog-content")) - .Append(CreateProcessRequest(recorder, assetsLocation, assetsJson)) + // Binlogs are discovered via ComponentStreamEnumerableFactory in OnPrepareDetectionAsync. + var binlogStreams = binlogProjectInfos.Keys + .Select(CreateComponentStream) .ToArray(); + streamFactoryMock + .Setup(x => x.GetComponentStreams( + It.IsAny(), + It.Is>(p => p.Any(s => s == "*.binlog")), + It.IsAny(), + It.IsAny())) + .Returns(binlogStreams); + // Walker observable only returns assets files (SearchPatterns = ["project.assets.json"]). + var assetsRequest = CreateProcessRequest(recorder, assetsLocation, assetsJson); walkerMock .Setup(x => x.GetFilteredComponentStreamObservable( It.IsAny(), It.IsAny>(), It.IsAny())) - .Returns(requests.ToObservable()); + .Returns(new[] { assetsRequest }.ToObservable()); var scanRequest = new ScanRequest( new DirectoryInfo(Path.GetTempPath()), @@ -785,6 +798,14 @@ private static ProcessRequest CreateProcessRequest(IComponentRecorder recorder, }; } + private static IComponentStream CreateComponentStream(string location) + { + var mock = new Mock(); + mock.SetupGet(x => x.Location).Returns(location); + mock.SetupGet(x => x.Pattern).Returns(Path.GetFileName(location)); + return mock.Object; + } + /// /// Creates a minimal valid project.assets.json with a single package. /// From 4bb52e2e50101187e465a32d9ffd1a6d44a2449f Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Thu, 19 Mar 2026 21:02:47 -0700 Subject: [PATCH 23/26] Support cancellation of binlog processing --- .../nuget/BinLogProcessor.cs | 9 ++- .../nuget/IBinLogProcessor.cs | 4 +- .../MSBuildBinaryLogComponentDetector.cs | 56 +++++++++---------- .../nuget/MSBuildProjectInfo.cs | 3 +- .../MSBuildBinaryLogComponentDetectorTests.cs | 20 ++++++- 5 files changed, 56 insertions(+), 36 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs index 5439d2111..ea2787827 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/BinLogProcessor.cs @@ -5,6 +5,7 @@ namespace Microsoft.ComponentDetection.Detectors.NuGet; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; using Microsoft.Build.Framework; using Microsoft.Build.Logging.StructuredLogger; using Microsoft.ComponentDetection.Detectors.DotNet; @@ -24,8 +25,10 @@ internal class BinLogProcessor : IBinLogProcessor public BinLogProcessor(Microsoft.Extensions.Logging.ILogger logger) => this.logger = logger; /// - public IReadOnlyList ExtractProjectInfo(string binlogPath, string? sourceDirectory = null) + public IReadOnlyList ExtractProjectInfo(string binlogPath, string? sourceDirectory = null, CancellationToken cancellationToken = default) { + cancellationToken.ThrowIfCancellationRequested(); + // Maps project path to the primary MSBuildProjectInfo for that project var projectInfoByPath = new Dictionary(StringComparer.OrdinalIgnoreCase); @@ -137,7 +140,9 @@ e is BuildMessageEventArgs msg && } }; - reader.Replay(binlogPath); + var progress = new Progress(); + progress.Updated += _ => cancellationToken.ThrowIfCancellationRequested(); + reader.Replay(binlogPath, progress); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/IBinLogProcessor.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/IBinLogProcessor.cs index 2bbdea69d..8d8aa70cc 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/IBinLogProcessor.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/IBinLogProcessor.cs @@ -1,6 +1,7 @@ namespace Microsoft.ComponentDetection.Detectors.NuGet; using System.Collections.Generic; +using System.Threading; /// /// Interface for processing MSBuild binary log files to extract project information. @@ -19,6 +20,7 @@ internal interface IBinLogProcessor /// The source directory on the scanning machine, used to rebase paths when the binlog /// was produced on a different machine. May be null to skip rebasing. /// + /// Token to cancel binlog replay. /// Collection of project information extracted from the binlog, with paths rebased. - IReadOnlyList ExtractProjectInfo(string binlogPath, string? sourceDirectory = null); + IReadOnlyList ExtractProjectInfo(string binlogPath, string? sourceDirectory = null, CancellationToken cancellationToken = default); } diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs index 53087f396..f85887175 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs @@ -171,7 +171,8 @@ protected override Task> OnPrepareDetectionAsync( foreach (var stream in binlogStreams) { - this.ProcessBinlogFile(stream.Location); + cancellationToken.ThrowIfCancellationRequested(); + this.ProcessBinlogFile(stream.Location, cancellationToken); } // Phase 2: Return the original observable unchanged. @@ -222,6 +223,25 @@ private static bool IsDevelopmentOnlyProject(MSBuildProjectInfo projectInfo) => return null; } + /// + /// Finds the inner build matching the given target framework, or returns the outer project info. + /// + private static MSBuildProjectInfo GetInnerBuildOrDefault(MSBuildProjectInfo projectInfo, NuGetFramework? framework) + { + if (framework != null) + { + var innerBuild = projectInfo.InnerBuilds.FirstOrDefault( + ib => !string.IsNullOrEmpty(ib.TargetFramework) && + NuGetFramework.Parse(ib.TargetFramework).Equals(framework)); + if (innerBuild != null) + { + return innerBuild; + } + } + + return projectInfo; + } + /// /// Maps the MSBuild OutputType property to "application" or "library". /// Returns null for empty/unknown values rather than guessing. @@ -257,7 +277,7 @@ private static bool IsDevelopmentOnlyProject(MSBuildProjectInfo projectInfo) => private static bool IsSelfContainedFromProjectInfo(MSBuildProjectInfo projectInfo) => projectInfo.SelfContained == true || projectInfo.PublishAot == true; - private void ProcessBinlogFile(string binlogPath) + private void ProcessBinlogFile(string binlogPath, CancellationToken cancellationToken) { var assetsFilesFound = new List(); @@ -265,7 +285,7 @@ private void ProcessBinlogFile(string binlogPath) { this.Logger.LogDebug("Processing binlog file: {BinlogPath}", binlogPath); - var projectInfos = this.binLogProcessor.ExtractProjectInfo(binlogPath, this.sourceDirectory); + var projectInfos = this.binLogProcessor.ExtractProjectInfo(binlogPath, this.sourceDirectory, cancellationToken); if (projectInfos.Count == 0) { @@ -497,21 +517,8 @@ private void ProcessLockFileWithProjectInfo(LockFile lockFile, MSBuildProjectInf var singleFileComponentRecorder = this.ComponentRecorder.CreateSingleFileComponentRecorder(recorderLocation); // Get the project info for the target framework (use inner build if available) - MSBuildProjectInfo GetProjectInfoForTarget(LockFileTarget target) - { - if (target.TargetFramework != null) - { - var innerBuild = projectInfo.InnerBuilds.FirstOrDefault( - ib => !string.IsNullOrEmpty(ib.TargetFramework) && - NuGetFramework.Parse(ib.TargetFramework).Equals(target.TargetFramework)); - if (innerBuild != null) - { - return innerBuild; - } - } - - return projectInfo; - } + MSBuildProjectInfo GetProjectInfoForTarget(LockFileTarget target) => + GetInnerBuildOrDefault(projectInfo, target.TargetFramework); foreach (var target in lockFile.Targets) { @@ -563,18 +570,7 @@ bool IsFrameworkOrDevDependency(LockFileTargetLibrary library) => /// private bool IsPackageDownloadDevDependency(string packageName, NuGetFramework? framework, MSBuildProjectInfo projectInfo) { - // Get the project info for this framework (use inner build if available) - var targetProjectInfo = projectInfo; - if (framework != null) - { - var innerBuild = projectInfo.InnerBuilds.FirstOrDefault( - ib => !string.IsNullOrEmpty(ib.TargetFramework) && - NuGetFramework.Parse(ib.TargetFramework).Equals(framework)); - if (innerBuild != null) - { - targetProjectInfo = innerBuild; - } - } + var targetProjectInfo = GetInnerBuildOrDefault(projectInfo, framework); // Project-level override: all deps are dev deps if (IsDevelopmentOnlyProject(targetProjectInfo)) diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildProjectInfo.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildProjectInfo.cs index e4cc89379..5a565b413 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildProjectInfo.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildProjectInfo.cs @@ -245,7 +245,8 @@ public MSBuildProjectInfo MergeWith(MSBuildProjectInfo other) { // Merge boolean properties: true wins. For all classification booleans (IsTestProject, // IsDevelopment, IsShipping, etc.), if any pass reports true it is sufficient to classify - // the project accordingly. These values are not expected to differ across passes. + // the project accordingly. These values are not expected to differ across passes, if they + // do we want true value to win. this.IsDevelopment = MergeBool(this.IsDevelopment, other.IsDevelopment); this.IsPackable = MergeBool(this.IsPackable, other.IsPackable); this.IsShipping = MergeBool(this.IsShipping, other.IsShipping); diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs index c1e2c497b..b3fbd90fb 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs @@ -1,5 +1,6 @@ namespace Microsoft.ComponentDetection.Detectors.Tests.NuGet; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -601,6 +602,21 @@ public async Task MultipleBinlogs_MergesTestProjectFlag_AllDepsAreDev() recorder.GetEffectiveDevDependencyValue(component.Component.Id).Should().BeTrue(); } + // ================================================================ + // Cancellation + // ================================================================ + [TestMethod] + public void PreCancelledToken_ThrowsOperationCanceledException() + { + using var cts = new CancellationTokenSource(); + cts.Cancel(); + + var processor = new BinLogProcessor(new Mock>().Object); + var act = () => processor.ExtractProjectInfo("any.binlog", cancellationToken: cts.Token); + + act.Should().Throw(); + } + // ================================================================ // Helpers – project info construction // ================================================================ @@ -649,7 +665,7 @@ private static ITaskItem CreateTaskItemMock(string itemSpec, bool isDevelopmentD { var binLogProcessorMock = new Mock(); binLogProcessorMock - .Setup(x => x.ExtractProjectInfo(binlogPath, It.IsAny())) + .Setup(x => x.ExtractProjectInfo(binlogPath, It.IsAny(), It.IsAny())) .Returns(projectInfos); var walkerMock = new Mock(); @@ -722,7 +738,7 @@ private static ITaskItem CreateTaskItemMock(string itemSpec, bool isDevelopmentD foreach (var (binlogPath, projectInfos) in binlogProjectInfos) { binLogProcessorMock - .Setup(x => x.ExtractProjectInfo(binlogPath, It.IsAny())) + .Setup(x => x.ExtractProjectInfo(binlogPath, It.IsAny(), It.IsAny())) .Returns(projectInfos); } From 5958e0dad8221e22c7e9f51ff1b77715107c2ee6 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Thu, 19 Mar 2026 22:07:29 -0700 Subject: [PATCH 24/26] Make default off and document --- docs/detectors/README.md | 11 ++--- docs/detectors/nuget.md | 42 +++++++++++++++++++ docs/feature-overview.md | 2 +- .../MSBuildBinaryLogComponentDetector.cs | 2 +- .../Extensions/ServiceCollectionExtensions.cs | 1 - 5 files changed, 50 insertions(+), 8 deletions(-) diff --git a/docs/detectors/README.md b/docs/detectors/README.md index 3309e9eed..f15c1a72a 100644 --- a/docs/detectors/README.md +++ b/docs/detectors/README.md @@ -70,11 +70,12 @@ - [NuGet](nuget.md) -| Detector | Status | -| ------------------------------------------------ | ------ | -| NuGetComponentDetector | Stable | -| NuGetPackagesConfigDetector | Stable | -| NuGetProjectModelProjectCentricComponentDetector | Stable | +| Detector | Status | +| ------------------------------------------------ | ---------- | +| NuGetComponentDetector | Stable | +| NuGetPackagesConfigDetector | Stable | +| NuGetProjectModelProjectCentricComponentDetector | Stable | +| MSBuildBinaryLogComponentDetector | DefaultOff | - [Pip](pip.md) diff --git a/docs/detectors/nuget.md b/docs/detectors/nuget.md index f1e8f0670..e2dee5576 100644 --- a/docs/detectors/nuget.md +++ b/docs/detectors/nuget.md @@ -36,6 +36,48 @@ The `NuGetPackagesConfig` detector raises NuGet components referenced by project [7]: https://learn.microsoft.com/en-us/nuget/reference/packages-config +## MSBuildBinaryLog + +The `MSBuildBinaryLog` detector is a **DefaultOff** detector intended to eventually replace both the `NuGetProjectCentric` and `DotNet` detectors. It combines MSBuild binary log (binlog) information with `project.assets.json` to provide enhanced component detection with project-level classifications. + +It looks for `project.assets.json` files and separately discovers `*.binlog` files. The binlog provides build-time context that isn't available from `project.assets.json` alone. + +### MSBuild Properties + +The detector extracts the following MSBuild properties from binlog data: + +| Property | Usage | +| --- | --- | +| `NETCoreSdkVersion` | Registered as the SDK version for the DotNet component. More accurate than `dotnet --version`, which can differ due to `global.json` rollforward. | +| `OutputType` | Classifies projects as "application" (`Exe`, `WinExe`, `AppContainerExe`) or "library" (`Library`, `Module`). The `DotNet` detector uses PE header inspection, which requires compiled output. | +| `ProjectAssetsFile` | Maps binlog project info to the corresponding `project.assets.json` on disk. | +| `TargetFramework` / `TargetFrameworks` | Identifies inner builds for multi-targeted projects and determines per-TFM properties. | +| `IsTestProject` | When `true`, all dependencies of the project are marked as development dependencies. | +| `IsShipping` | When `false`, all dependencies are marked as development dependencies. | +| `IsDevelopment` | When `true`, all dependencies are marked as development dependencies. | +| `IsPackable` | Indicates whether the project produces a NuGet package. | +| `SelfContained` | Detects self-contained deployment. Combined with lock file heuristics. | +| `PublishAot` | When `true`, project is treated as self-contained. Typically only set during publish pass. | + +### PackageReference and PackageDownload Metadata + +The detector reads `IsDevelopmentDependency` metadata from `PackageReference` and `PackageDownload` items in the binlog: + +- **PackageReference**: When `IsDevelopmentDependency` is `true`, the package and all of its transitive dependencies are marked as development dependencies. +- **PackageDownload**: Packages are registered as development dependencies by default unless explicitly overridden via `IsDevelopmentDependency` metadata. + +### Multi-Targeting and Multi-Pass Merging + +For multi-targeted projects, the detector understands the MSBuild outer/inner build structure. Properties from inner builds (per-TFM) are tracked separately, and when a project appears in multiple binlogs (e.g., a build pass and a publish pass), their properties are merged so that values like `PublishAot` (typically only set during publish) are available when processing the shared `project.assets.json`. + +### Fallback Mode + +When no binlog is available for a project, the detector falls back to standard NuGet detection behavior (equivalent to the `NuGetProjectCentric` detector). + +### Enabling the Detector + +Pass `--DetectorArgs MSBuildBinaryLog=EnableIfDefaultOff` and ensure a `*.binlog` file is present in the scan directory (e.g., by building with `dotnet build -bl`). + ## Known Limitations - Any components that are only found in `*.nuspec` or `*.nupkg` files will not be detected with the latest NuGet Detector approach, because the NuGet detector that scans `*.nuspec` or `*.nupkg` files overreports. This is due to of NuGet's [restore behaviour][8] which downloads all possible dependencies before [resolving the final dependency graph][9]. diff --git a/docs/feature-overview.md b/docs/feature-overview.md index aabee8bf1..0c96ebebb 100644 --- a/docs/feature-overview.md +++ b/docs/feature-overview.md @@ -11,7 +11,7 @@ | NPM |
  • package.json
  • package-lock.json
  • npm-shrinkwrap.json
  • lerna.json
| - | ✔ (dev-dependencies in package.json, dev flag in package-lock.json) | ✔ | | Yarn (v1, v2) |
  • package.json
  • yarn.lock
| - | ✔ (dev-dependencies in package.json) | ✔ | | Pnpm |
  • shrinkwrap.yaml
  • pnpm-lock.yaml
| - | ✔ (packages/{package}/dev flag) | ✔ | -| NuGet |
  • project.assets.json
  • *.nupkg
  • *.nuspec
  • packages.config
  • nuget.config
| - | - | ✔ (required project.assets.json) | +| NuGet |
  • project.assets.json
  • *.nupkg
  • *.nuspec
  • packages.config
  • nuget.config
  • *.binlog (DefaultOff)
| - | - | ✔ (required project.assets.json) | | Pip (Python) |
  • setup.py
  • requirements.txt
  • *setup=distutils.core.run_setup({setup.py}); setup.install_requires*
  • dist package METADATA file
|
  • Python 2 or Python 3
  • Internet connection
| ❌ | ✔ | | Poetry (Python) |
  • poetry.lock
      • | - | ✔ | ❌ | | Ruby |
      • gemfile.lock
      | - | ❌ | ✔ | diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs index f85887175..98e275dd5 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs @@ -48,7 +48,7 @@ namespace Microsoft.ComponentDetection.Detectors.NuGet; /// - Fallback mode: When no binlog info is available, falls back to standard NuGet detection. /// /// -public class MSBuildBinaryLogComponentDetector : FileComponentDetector, IExperimentalDetector +public class MSBuildBinaryLogComponentDetector : FileComponentDetector, IDefaultOffComponentDetector { private readonly IBinLogProcessor binLogProcessor; private readonly IFileUtilityService fileUtilityService; diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs index 837964bfc..c3b64f74d 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs @@ -73,7 +73,6 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); // Detectors // CocoaPods From 09e27d583c012f9ecef589f58f50098807f51b87 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Thu, 26 Mar 2026 08:23:49 -0700 Subject: [PATCH 25/26] Don't catch OperationCanceledException --- .../nuget/MSBuildBinaryLogComponentDetector.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs index 98e275dd5..6f82a5cec 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs @@ -308,6 +308,10 @@ private void ProcessBinlogFile(string binlogPath, CancellationToken cancellation projectInfos.Count); } } + catch (OperationCanceledException) + { + throw; + } catch (Exception ex) { this.Logger.LogWarning(ex, "Failed to process binlog file: {BinlogPath}", binlogPath); @@ -468,6 +472,10 @@ private async Task ProcessAssetsFileAsync(ProcessRequest processRequest, Cancell await this.ProcessLockFileFallbackAsync(lockFile, assetsFilePath, cancellationToken); } } + catch (OperationCanceledException) + { + throw; + } catch (Exception ex) { this.Logger.LogWarning(ex, "Failed to process NuGet lockfile: {LockFile}", assetsFilePath); From 7e82a330c35bdfdaf3d993def36e5ec15ccb4f69 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Mon, 30 Mar 2026 10:46:42 -0700 Subject: [PATCH 26/26] Update docs, refine tests --- docs/detectors/nuget.md | 3 +- .../dotnet/DotNetProjectInfoProvider.cs | 2 + .../nuget/LockFileUtilities.cs | 29 ++---- .../MSBuildBinaryLogComponentDetector.cs | 30 ++++--- .../nuget/BinLogProcessorTests.cs | 38 ++++++++ .../MSBuildBinaryLogComponentDetectorTests.cs | 89 +++++++++++-------- 6 files changed, 115 insertions(+), 76 deletions(-) diff --git a/docs/detectors/nuget.md b/docs/detectors/nuget.md index e2dee5576..2f55f9785 100644 --- a/docs/detectors/nuget.md +++ b/docs/detectors/nuget.md @@ -63,7 +63,8 @@ The detector extracts the following MSBuild properties from binlog data: The detector reads `IsDevelopmentDependency` metadata from `PackageReference` and `PackageDownload` items in the binlog: -- **PackageReference**: When `IsDevelopmentDependency` is `true`, the package and all of its transitive dependencies are marked as development dependencies. +- **PackageReference**: When `IsDevelopmentDependency` is `true`, the package and **all of its transitive dependencies** are marked as development dependencies. This allows annotating a single top-level dependency to classify its entire closure as dev-only. If a transitive dependency should remain non-dev, reference it directly (or transitively via a non-overridden dependency) so that it is also registered through a non-dev path. +- **PackageReference**: When `IsDevelopmentDependency` is `false`, the package and all of its transitive dependencies are classified using the explicit override rather than the default heuristics. - **PackageDownload**: Packages are registered as development dependencies by default unless explicitly overridden via `IsDevelopmentDependency` metadata. ### Multi-Targeting and Multi-Pass Merging diff --git a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetProjectInfoProvider.cs b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetProjectInfoProvider.cs index bf2755ea6..784fb2362 100644 --- a/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetProjectInfoProvider.cs +++ b/src/Microsoft.ComponentDetection.Detectors/dotnet/DotNetProjectInfoProvider.cs @@ -53,9 +53,11 @@ public DotNetProjectInfoProvider( /// /// Initializes source directory paths for path rebasing. Call once per scan. + /// Clears any cached state from a previous scan. /// public void Initialize(string? sourceDirectory, string? sourceFileRootDirectory) { + this.sdkVersionCache.Clear(); this.sourceDirectory = this.NormalizeDirectory(sourceDirectory); this.sourceFileRootDirectory = this.NormalizeDirectory(sourceFileRootDirectory); } diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/LockFileUtilities.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/LockFileUtilities.cs index 7a682556e..9dd4bb74b 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/LockFileUtilities.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/LockFileUtilities.cs @@ -19,7 +19,7 @@ namespace Microsoft.ComponentDetection.Detectors.NuGet; /// Shared utility methods for processing NuGet lock files (project.assets.json). /// Used by both NuGetProjectModelProjectCentricComponentDetector and MSBuildBinaryLogComponentDetector. ///
-public static class LockFileUtilities +internal static class LockFileUtilities { /// /// Dependency type constant for project references in project.assets.json. @@ -139,30 +139,23 @@ static bool IsAPlaceholderItem(LockFileItem item) => /// The version range to match (mutually exclusive with version). /// Optional logger for debug messages. /// The matching library, or null if not found. - public static LockFileLibrary? GetLibraryComponentWithDependencyLookup( - IList? libraries, + public static LockFileLibrary GetLibraryComponentWithDependencyLookup( + IList libraries, string dependencyId, Version? version, VersionRange? versionRange, ILogger? logger = null) { - if (libraries == null) - { - return null; - } - if ((version == null && versionRange == null) || (version != null && versionRange != null)) { - logger?.LogDebug("Either version or versionRange must be specified, but not both for {DependencyId}.", dependencyId); - return null; + throw new ArgumentException($"Either {nameof(version)} or {nameof(versionRange)} must be specified, but not both."); } var matchingLibraryNames = libraries.Where(x => string.Equals(x.Name, dependencyId, StringComparison.OrdinalIgnoreCase)).ToList(); if (matchingLibraryNames.Count == 0) { - logger?.LogDebug("No library found matching: {DependencyId}", dependencyId); - return null; + throw new InvalidOperationException("Project.assets.json is malformed, no library could be found matching: " + dependencyId); } LockFileLibrary? matchingLibrary; @@ -308,17 +301,7 @@ internal static (List Libraries, HashSet ComponentIds) var libraries = new List(); foreach (var lib in GetTopLevelLibraries(lockFile)) { - var resolved = GetLibraryComponentWithDependencyLookup(lockFile.Libraries, lib.Name, lib.Version, lib.VersionRange, logger); - if (resolved != null) - { - libraries.Add(resolved); - } - else - { - logger?.LogWarning( - "Could not resolve top-level dependency {DependencyName}. The project.assets.json may be malformed.", - lib.Name); - } + libraries.Add(GetLibraryComponentWithDependencyLookup(lockFile.Libraries, lib.Name, lib.Version, lib.VersionRange, logger)); } var componentIds = libraries diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs index 6f82a5cec..eadf73907 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/MSBuildBinaryLogComponentDetector.cs @@ -68,6 +68,9 @@ public class MSBuildBinaryLogComponentDetector : FileComponentDetector, IDefault /// (typically only set in the publish pass) are available when processing the shared /// project.assets.json. /// + // OrdinalIgnoreCase: on Windows, filesystem paths are case-insensitive. On Linux, paths + // are naturally case-consistent so case-insensitive comparison is harmless. This is + // pre-existing behavior inherited from the dictionary used for assets file lookup. private readonly ConcurrentDictionary projectInfoByAssetsFile = new(StringComparer.OrdinalIgnoreCase); // Source directory passed to BinLogProcessor for path rebasing. @@ -362,25 +365,20 @@ private void LogMissingAssetsWarnings(MSBuildProjectInfo projectInfo) /// /// For target type (application/library), we use the OutputType property from the binlog /// which is equivalent to what DotNetComponentDetector determines by inspecting the PE headers. + /// For multi-targeted projects, the per-TFM inner build OutputType is used when available. /// /// When a lock file is available, self-contained detection uses both binlog properties /// (SelfContained, PublishAot) and the lock file heuristic (ILCompiler in libraries, /// runtime download dependencies matching framework references) for comprehensive coverage. /// - private void RegisterDotNetComponent(MSBuildProjectInfo projectInfo, LockFile? lockFile = null) + private void RegisterDotNetComponent(MSBuildProjectInfo projectInfo, string recorderLocation, LockFile? lockFile = null) { - if (string.IsNullOrEmpty(projectInfo.NETCoreSdkVersion) || string.IsNullOrEmpty(projectInfo.ProjectPath)) + if (string.IsNullOrEmpty(projectInfo.NETCoreSdkVersion) || string.IsNullOrEmpty(recorderLocation)) { return; } - var singleFileComponentRecorder = this.ComponentRecorder.CreateSingleFileComponentRecorder(projectInfo.ProjectPath); - - // Determine target type from OutputType property. - // Known application types: Exe, WinExe, AppContainerExe - // Known library types: Library, Module - // Unknown values are left as null (don't assume). - var targetType = GetTargetType(projectInfo.OutputType); + var singleFileComponentRecorder = this.ComponentRecorder.CreateSingleFileComponentRecorder(recorderLocation); // Primary self-contained check from binlog properties (SelfContained, PublishAot) var isSelfContainedFromBinlog = IsSelfContainedFromProjectInfo(projectInfo); @@ -388,9 +386,12 @@ private void RegisterDotNetComponent(MSBuildProjectInfo projectInfo, LockFile? l if (lockFile != null) { // When lock file is available, check per-target self-contained - // combining binlog properties and lock file heuristics + // combining binlog properties and lock file heuristics. + // Use per-TFM inner build for OutputType when available. foreach (var target in lockFile.Targets) { + var innerInfo = GetInnerBuildOrDefault(projectInfo, target.TargetFramework); + var targetType = GetTargetType(innerInfo.OutputType); var isSelfContained = isSelfContainedFromBinlog || LockFileUtilities.IsSelfContainedFromLockFile(lockFile.PackageSpec, target.TargetFramework, target); var projectType = LockFileUtilities.GetTargetTypeWithSelfContained(targetType, isSelfContained); @@ -408,7 +409,9 @@ private void RegisterDotNetComponent(MSBuildProjectInfo projectInfo, LockFile? l } // Binlog-only path: no lock file available or no targets in lock file - var projectTypeFromBinlog = LockFileUtilities.GetTargetTypeWithSelfContained(targetType, isSelfContainedFromBinlog); + // Determine target type from outer build OutputType property. + var targetTypeFromBinlog = GetTargetType(projectInfo.OutputType); + var projectTypeFromBinlog = LockFileUtilities.GetTargetTypeWithSelfContained(targetTypeFromBinlog, isSelfContainedFromBinlog); // Get target frameworks from binlog properties var targetFrameworks = new List(); @@ -569,8 +572,9 @@ bool IsFrameworkOrDevDependency(LockFileTargetLibrary library) => lockFile, (packageName, framework) => this.IsPackageDownloadDevDependency(packageName, framework, projectInfo)); - // Register DotNet component with combined binlog + lock file self-contained detection - this.RegisterDotNetComponent(projectInfo, lockFile); + // Register DotNet component with combined binlog + lock file self-contained detection. + // Use the same recorder location as NuGet components for consistent grouping. + this.RegisterDotNetComponent(projectInfo, recorderLocation, lockFile); } /// diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs index 2e1a3337c..d595b2cbd 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/BinLogProcessorTests.cs @@ -6,6 +6,7 @@ namespace Microsoft.ComponentDetection.Detectors.Tests.NuGet; using System.IO; using System.Linq; using System.Runtime.InteropServices; +using System.Threading; using System.Threading.Tasks; using AwesomeAssertions; using Microsoft.ComponentDetection.Detectors.NuGet; @@ -1723,6 +1724,43 @@ private static async Task RunProcessWithEnvAsync( } } + [TestMethod] + public async Task BinLogReader_Replay_PropagatesCancellationFromProgress() + { + // This test verifies the external behavior we depend on: BinLogReader.Replay + // propagates OperationCanceledException thrown from the Progress callback. + // If this behavior changes upstream, we need to find an alternative cancellation + // strategy in BinLogProcessor. + var projectDir = Path.Combine(this.testDir, "CancelProgress"); + Directory.CreateDirectory(projectDir); + + var content = """ + + + net8.0 + + + """; + WriteFile(projectDir, "CancelProgress.csproj", content); + + var binlogPath = await BuildProjectAsync(projectDir, "CancelProgress.csproj"); + + var reader = new Microsoft.Build.Logging.StructuredLogger.BinLogReader(); + Microsoft.Build.Logging.StructuredLogger.Strings.Initialize(); + + using var cts = new CancellationTokenSource(); + var progress = new Microsoft.Build.Logging.StructuredLogger.Progress(); + progress.Updated += _ => + { + // Cancel after the first progress update to ensure Replay is mid-flight. + cts.Cancel(); + cts.Token.ThrowIfCancellationRequested(); + }; + + var act = () => reader.Replay(binlogPath, progress); + act.Should().Throw(); + } + /// /// Logger that captures messages so they can be included in assertion failure output. /// diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs index b3fbd90fb..b8f8ac108 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/nuget/MSBuildBinaryLogComponentDetectorTests.cs @@ -5,6 +5,7 @@ namespace Microsoft.ComponentDetection.Detectors.Tests.NuGet; using System.IO; using System.Linq; using System.Reactive.Linq; +using System.Runtime.InteropServices; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -30,9 +31,14 @@ namespace Microsoft.ComponentDetection.Detectors.Tests.NuGet; [TestClass] public class MSBuildBinaryLogComponentDetectorTests : BaseDetectorTest { - private const string ProjectPath = @"C:\test\TestProject.csproj"; - private const string AssetsFilePath = @"C:\test\obj\project.assets.json"; - private const string BinlogFilePath = @"C:\test\build.binlog"; + private static readonly string RootDir = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "C:" : "/"; + private static readonly string ProjectPath = Path.Combine(RootDir, "test", "TestProject.csproj"); + private static readonly string AssetsFilePath = Path.Combine(RootDir, "test", "obj", "project.assets.json"); + private static readonly string BinlogFilePath = Path.Combine(RootDir, "test", "build.binlog"); + private static readonly string PublishBinlogFilePath = Path.Combine(RootDir, "test", "publish.binlog"); + private static readonly string TestBinlogFilePath = Path.Combine(RootDir, "test", "test.binlog"); + private static readonly string PackagesFolder = Path.Combine(RootDir, "Users", "test", ".nuget", "packages") + Path.DirectorySeparatorChar; + private static readonly string ProjectOutputPath = Path.Combine(RootDir, "test", "obj"); private readonly Mock commandLineInvocationServiceMock; private readonly Mock fileUtilityServiceMock; @@ -455,8 +461,8 @@ public async Task WithBinlog_NoBinlogMatch_FallsBackToStandardProcessing() { // Binlog contains info for a DIFFERENT project; assets file project path doesn't match. var otherInfo = CreateProjectInfo( - projectPath: @"C:\other\OtherProject.csproj", - assetsFilePath: @"C:\other\obj\project.assets.json"); + projectPath: Path.Combine(RootDir, "other", "OtherProject.csproj"), + assetsFilePath: Path.Combine(RootDir, "other", "obj", "project.assets.json")); otherInfo.IsTestProject = true; var assetsJson = SimpleAssetsJson("Newtonsoft.Json", "13.0.1"); @@ -496,8 +502,8 @@ public async Task MultipleBinlogs_BuildThenPublishSelfContained_MergedAsSelfCont var binlogs = new Dictionary> { - [@"C:\test\build.binlog"] = [buildInfo], - [@"C:\test\publish.binlog"] = [publishInfo], + [BinlogFilePath] = [buildInfo], + [PublishBinlogFilePath] = [publishInfo], }; var (result, recorder) = await ExecuteWithMultipleBinlogsAsync(binlogs, assetsJson); @@ -528,8 +534,8 @@ public async Task MultipleBinlogs_BuildThenPublishNotSelfContained_MergedAsNotSe var binlogs = new Dictionary> { - [@"C:\test\build.binlog"] = [buildInfo], - [@"C:\test\publish.binlog"] = [publishInfo], + [BinlogFilePath] = [buildInfo], + [PublishBinlogFilePath] = [publishInfo], }; var (result, recorder) = await ExecuteWithMultipleBinlogsAsync(binlogs, assetsJson); @@ -561,8 +567,8 @@ public async Task MultipleBinlogs_BuildThenPublishAot_MergedAsSelfContained() var binlogs = new Dictionary> { - [@"C:\test\build.binlog"] = [buildInfo], - [@"C:\test\publish.binlog"] = [publishInfo], + [BinlogFilePath] = [buildInfo], + [PublishBinlogFilePath] = [publishInfo], }; var (result, recorder) = await ExecuteWithMultipleBinlogsAsync(binlogs, assetsJson); @@ -590,8 +596,8 @@ public async Task MultipleBinlogs_MergesTestProjectFlag_AllDepsAreDev() var binlogs = new Dictionary> { - [@"C:\test\build.binlog"] = [buildInfo], - [@"C:\test\test.binlog"] = [testInfo], + [BinlogFilePath] = [buildInfo], + [TestBinlogFilePath] = [testInfo], }; var (result, recorder) = await ExecuteWithMultipleBinlogsAsync(binlogs, assetsJson); @@ -621,14 +627,14 @@ public void PreCancelledToken_ThrowsOperationCanceledException() // Helpers – project info construction // ================================================================ private static MSBuildProjectInfo CreateProjectInfo( - string projectPath = ProjectPath, - string assetsFilePath = AssetsFilePath, + string? projectPath = null, + string? assetsFilePath = null, string? targetFramework = "net8.0") { return new MSBuildProjectInfo { - ProjectPath = projectPath, - ProjectAssetsFile = assetsFilePath, + ProjectPath = projectPath ?? ProjectPath, + ProjectAssetsFile = assetsFilePath ?? AssetsFilePath, TargetFramework = targetFramework, }; } @@ -660,9 +666,11 @@ private static ITaskItem CreateTaskItemMock(string itemSpec, bool isDevelopmentD private static async Task<(IndividualDetectorScanResult Result, IComponentRecorder Recorder)> ExecuteWithBinlogAsync( IReadOnlyList projectInfos, string assetsJson, - string binlogPath = BinlogFilePath, - string assetsLocation = AssetsFilePath) + string? binlogPath = null, + string? assetsLocation = null) { + binlogPath ??= BinlogFilePath; + assetsLocation ??= AssetsFilePath; var binLogProcessorMock = new Mock(); binLogProcessorMock .Setup(x => x.ExtractProjectInfo(binlogPath, It.IsAny(), It.IsAny())) @@ -732,8 +740,9 @@ private static ITaskItem CreateTaskItemMock(string itemSpec, bool isDevelopmentD private static async Task<(IndividualDetectorScanResult Result, IComponentRecorder Recorder)> ExecuteWithMultipleBinlogsAsync( Dictionary> binlogProjectInfos, string assetsJson, - string assetsLocation = AssetsFilePath) + string? assetsLocation = null) { + assetsLocation ??= AssetsFilePath; var binLogProcessorMock = new Mock(); foreach (var (binlogPath, projectInfos) in binlogProjectInfos) { @@ -822,6 +831,8 @@ private static IComponentStream CreateComponentStream(string location) return mock.Object; } + private static string JsonEscape(string path) => path.Replace(@"\", @"\\"); + /// /// Creates a minimal valid project.assets.json with a single package. /// @@ -848,13 +859,13 @@ private static string SimpleAssetsJson(string packageName, string version) => $$ "projectFileDependencyGroups": { "net8.0": [ "{{packageName}} >= {{version}}" ] }, - "packageFolders": { "C:\\Users\\test\\.nuget\\packages\\": {} }, + "packageFolders": { "{{JsonEscape(PackagesFolder)}}": {} }, "project": { "version": "1.0.0", "restore": { "projectName": "TestProject", - "projectPath": "C:\\test\\TestProject.csproj", - "outputPath": "C:\\test\\obj" + "projectPath": "{{JsonEscape(ProjectPath)}}", + "outputPath": "{{JsonEscape(ProjectOutputPath)}}" }, "frameworks": { "net8.0": { @@ -872,7 +883,7 @@ private static string SimpleAssetsJson(string packageName, string version) => $$ /// Assets JSON with a top-level package that has a transitive dependency. /// Microsoft.Extensions.Logging → Microsoft.Extensions.Logging.Abstractions. /// - private static string TransitiveAssetsJson() => """ + private static string TransitiveAssetsJson() => $$""" { "version": 3, "targets": { @@ -905,13 +916,13 @@ private static string TransitiveAssetsJson() => """ "projectFileDependencyGroups": { "net8.0": [ "Microsoft.Extensions.Logging >= 8.0.0" ] }, - "packageFolders": { "C:\\Users\\test\\.nuget\\packages\\": {} }, + "packageFolders": { "{{JsonEscape(PackagesFolder)}}": {} }, "project": { "version": "1.0.0", "restore": { "projectName": "TestProject", - "projectPath": "C:\\test\\TestProject.csproj", - "outputPath": "C:\\test\\obj" + "projectPath": "{{JsonEscape(ProjectPath)}}", + "outputPath": "{{JsonEscape(ProjectOutputPath)}}" }, "frameworks": { "net8.0": { @@ -928,7 +939,7 @@ private static string TransitiveAssetsJson() => """ /// /// Assets JSON with a NuGet package and a project reference. /// - private static string ProjectReferenceAssetsJson() => """ + private static string ProjectReferenceAssetsJson() => $$""" { "version": 3, "targets": { @@ -956,13 +967,13 @@ private static string ProjectReferenceAssetsJson() => """ "projectFileDependencyGroups": { "net8.0": [ "Newtonsoft.Json >= 13.0.1" ] }, - "packageFolders": { "C:\\Users\\test\\.nuget\\packages\\": {} }, + "packageFolders": { "{{JsonEscape(PackagesFolder)}}": {} }, "project": { "version": "1.0.0", "restore": { "projectName": "TestProject", - "projectPath": "C:\\test\\TestProject.csproj", - "outputPath": "C:\\test\\obj" + "projectPath": "{{JsonEscape(ProjectPath)}}", + "outputPath": "{{JsonEscape(ProjectOutputPath)}}" }, "frameworks": { "net8.0": { @@ -979,19 +990,19 @@ private static string ProjectReferenceAssetsJson() => """ /// /// Assets JSON with a PackageDownload dependency. /// - private static string PackageDownloadAssetsJson() => """ + private static string PackageDownloadAssetsJson() => $$""" { "version": 3, "targets": { "net8.0": {} }, "libraries": {}, "projectFileDependencyGroups": { "net8.0": [] }, - "packageFolders": { "C:\\Users\\test\\.nuget\\packages\\": {} }, + "packageFolders": { "{{JsonEscape(PackagesFolder)}}": {} }, "project": { "version": "1.0.0", "restore": { "projectName": "TestProject", - "projectPath": "C:\\test\\TestProject.csproj", - "outputPath": "C:\\test\\obj" + "projectPath": "{{JsonEscape(ProjectPath)}}", + "outputPath": "{{JsonEscape(ProjectOutputPath)}}" }, "frameworks": { "net8.0": { @@ -1009,7 +1020,7 @@ private static string PackageDownloadAssetsJson() => """ /// /// Assets JSON with two target frameworks (net8.0 and net6.0), each containing the same package. /// - private static string MultiTargetAssetsJson() => """ + private static string MultiTargetAssetsJson() => $$""" { "version": 3, "targets": { @@ -1039,13 +1050,13 @@ private static string MultiTargetAssetsJson() => """ "net8.0": [ "Newtonsoft.Json >= 13.0.1" ], "net6.0": [ "Newtonsoft.Json >= 13.0.1" ] }, - "packageFolders": { "C:\\Users\\test\\.nuget\\packages\\": {} }, + "packageFolders": { "{{JsonEscape(PackagesFolder)}}": {} }, "project": { "version": "1.0.0", "restore": { "projectName": "TestProject", - "projectPath": "C:\\test\\TestProject.csproj", - "outputPath": "C:\\test\\obj" + "projectPath": "{{JsonEscape(ProjectPath)}}", + "outputPath": "{{JsonEscape(ProjectOutputPath)}}" }, "frameworks": { "net8.0": {