[WIP] New nugraph global tool to create dependency graphs from the co…

…mmand line

Chisel.sln

@@ -32,6 +32,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SqlClientSample", "samples\
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Identity.Client", "samples\Microsoft.Identity.Client\Microsoft.Identity.Client.csproj", "{F40FA01A-EB18-4785-9A3C-379F2E7A7A02}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "nugraph", "src\nugraph\nugraph.csproj", "{7E6E162B-49CA-4D32-8DD9-EAD5E6FE253C}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -60,6 +62,10 @@ Global
 		{F40FA01A-EB18-4785-9A3C-379F2E7A7A02}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{F40FA01A-EB18-4785-9A3C-379F2E7A7A02}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{F40FA01A-EB18-4785-9A3C-379F2E7A7A02}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7E6E162B-49CA-4D32-8DD9-EAD5E6FE253C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7E6E162B-49CA-4D32-8DD9-EAD5E6FE253C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7E6E162B-49CA-4D32-8DD9-EAD5E6FE253C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7E6E162B-49CA-4D32-8DD9-EAD5E6FE253C}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{845EDA2A-5207-4C6D-ABE9-9635F4630D90} = {0CC84E67-19D2-480B-B36A-6BB15A9109E7}
@@ -68,5 +74,6 @@ Global
 		{8B1B3D6A-7100-4DFB-97C9-CF5ACF1A3B08} = {AC8C6685-EDF9-443A-BAF6-A5E7CF777B2A}
 		{611D4DE0-F729-48A6-A496-2EA3B5DF8EC6} = {0CC84E67-19D2-480B-B36A-6BB15A9109E7}
 		{F40FA01A-EB18-4785-9A3C-379F2E7A7A02} = {0CC84E67-19D2-480B-B36A-6BB15A9109E7}
+		{7E6E162B-49CA-4D32-8DD9-EAD5E6FE253C} = {89268D80-B21D-4C76-AF7F-796AAD1E00D9}
 	EndGlobalSection
 EndGlobal

src/nugraph/Dotnet.cs

@@ -0,0 +1,73 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Text.Json.Serialization;
+using System.Threading.Tasks;
+using CliWrap;
+
+namespace nugraph;
+
+internal static partial class Dotnet
+{
+    public static async Task<ProjectInfo> RestoreAsync(FileSystemInfo? source)
+    {
+        var stdout = new StringBuilder();
+        var stderr = new StringBuilder();
+        var jsonPipe = new JsonPipeTarget<Result>(SourceGenerationContext.Default.Result);
+        var dotnet = Cli.Wrap("dotnet")
+            .WithArguments(args =>
+            {
+                args.Add("restore");
+                if (source != null)
+                {
+                    args.Add(source.FullName);
+                }
+
+                // !!! Requires a recent .NET SDK (see https://github.com/dotnet/msbuild/issues/3911)
+                // arguments.Add("--target:ResolvePackageAssets"); // may enable if the project is an exe in order to get RuntimeCopyLocalItems + NativeCopyLocalItems
+                args.Add($"--getProperty:{nameof(Property.ProjectAssetsFile)}");
+                args.Add($"--getProperty:{nameof(Property.TargetFramework)}");
+                args.Add($"--getProperty:{nameof(Property.TargetFrameworks)}");
+                args.Add($"--getItem:{nameof(Item.RuntimeCopyLocalItems)}");
+                args.Add($"--getItem:{nameof(Item.NativeCopyLocalItems)}");
+            })
+            .WithEnvironmentVariables(env => env.Set("DOTNET_NOLOGO", "1"))
+            .WithValidation(CommandResultValidation.None)
+            .WithStandardOutputPipe(PipeTarget.Merge(jsonPipe, PipeTarget.ToStringBuilder(stdout)))
+            .WithStandardErrorPipe(PipeTarget.ToStringBuilder(stderr));
+
+        var commandResult = await dotnet.ExecuteAsync();
+
+        if (!commandResult.IsSuccess)
+        {
+            var message = stderr.Length > 0 ? stderr.ToString() : stdout.ToString();
+            throw new Exception($"Running \"{dotnet}\" in \"{dotnet.WorkingDirPath}\" failed with exit code {commandResult.ExitCode}.{Environment.NewLine}{message}");
+        }
+
+        var (properties, items) = jsonPipe.Result ?? throw new Exception($"Running \"{dotnet}\" in \"{dotnet.WorkingDirPath}\" returned a literal 'null' JSON payload");
+        var copyLocalPackages = items.RuntimeCopyLocalItems.Concat(items.NativeCopyLocalItems).Select(e => e.NuGetPackageId).ToHashSet();
+        return new ProjectInfo(properties.ProjectAssetsFile, properties.GetTargetFrameworks(), copyLocalPackages);
+    }
+
+    public record ProjectInfo(string ProjectAssetsFile, IReadOnlyCollection<string> TargetFrameworks, IReadOnlyCollection<string> CopyLocalPackages);
+
+    [JsonSerializable(typeof(Result))]
+    private partial class SourceGenerationContext : JsonSerializerContext;
+
+    private record Result(Property Properties, Item Items);
+
+    private record Property(string ProjectAssetsFile, string TargetFramework, string TargetFrameworks)
+    {
+        public IReadOnlyCollection<string> GetTargetFrameworks()
+        {
+            var targetFrameworks = TargetFrameworks.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToHashSet();
+            return targetFrameworks.Count > 0 ? targetFrameworks : [TargetFramework];
+        }
+    }
+
+    private record Item(CopyLocalItem[] RuntimeCopyLocalItems, CopyLocalItem[] NativeCopyLocalItems);
+
+    private record CopyLocalItem(string NuGetPackageId);
+}

src/nugraph/GraphCommand.cs

@@ -0,0 +1,188 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using System.Threading;
+using System.Threading.Tasks;
+using Chisel;
+using NuGet.Commands;
+using NuGet.Common;
+using NuGet.Configuration;
+using NuGet.Frameworks;
+using NuGet.LibraryModel;
+using NuGet.ProjectModel;
+using NuGet.Protocol.Core.Types;
+using NuGet.Versioning;
+using OneOf;
+using Spectre.Console;
+using Spectre.Console.Cli;
+
+namespace nugraph;
+
+[GenerateOneOf]
+public partial class FileOrPackages : OneOfBase<FileSystemInfo?, string[]>
+{
+    public override string ToString() => Match(file => file?.FullName ?? Environment.CurrentDirectory, ids => string.Join(", ", ids));
+}
+
+[Description("Generates dependency graphs for .NET projects and NuGet packages.")]
+internal class GraphCommand(IAnsiConsole console) : AsyncCommand<GraphCommandSettings>
+{
+    public override async Task<int> ExecuteAsync(CommandContext commandContext, GraphCommandSettings settings)
+    {
+        if (settings.PrintVersion)
+        {
+            console.WriteLine($"nugraph {GetVersion()}");
+            return 0;
+        }
+
+        var source = settings.GetSource();
+        var graphUrl = await console.Status().StartAsync($"Generating dependency graph for {source}", async _ =>
+        {
+            var graph = await source.Match(
+                f => ComputeDependencyGraphAsync(f, settings),
+                f => ComputeDependencyGraphAsync(f, settings, new SpectreLogger(console, settings.LogLevel), CancellationToken.None)
+            );
+            return await WriteGraphAsync(graph, settings);
+        });
+
+        if (graphUrl != null)
+        {
+            var url = graphUrl.ToString();
+            console.WriteLine(url);
+            Process.Start(new ProcessStartInfo(url) { UseShellExecute = true });
+        }
+        else if (settings.OutputFile != null)
+        {
+            console.MarkupLineInterpolated($"The {source} dependency graph has been written to [lime]{new Uri(settings.OutputFile.FullName)}[/]");
+        }
+
+        return 0;
+    }
+
+    private static string GetVersion()
+    {
+        var assembly = typeof(GraphCommand).Assembly;
+        var version = assembly.GetCustomAttribute<AssemblyInformationalVersionAttribute>()?.InformationalVersion ?? assembly.GetCustomAttribute<AssemblyFileVersionAttribute>()?.Version;
+        if (version == null)
+            return "0.0.0";
+
+        return SemanticVersion.TryParse(version, out var semanticVersion) ? semanticVersion.ToNormalizedString() : version;
+    }
+
+    private static async Task<DependencyGraph> ComputeDependencyGraphAsync(FileSystemInfo? source, GraphCommandSettings settings)
+    {
+        var projectInfo = await Dotnet.RestoreAsync(source);
+        var targetFramework = settings.Framework ?? projectInfo.TargetFrameworks.First();
+        var lockFile = new LockFileFormat().Read(projectInfo.ProjectAssetsFile);
+        Predicate<Package> filter = projectInfo.CopyLocalPackages.Count > 0 ? package => projectInfo.CopyLocalPackages.Contains(package.Name) : _ => true;
+        var (packages, roots) = lockFile.ReadPackages(targetFramework, settings.RuntimeIdentifier, filter);
+        return new DependencyGraph(packages, roots, ignores: settings.GraphIgnore);
+    }
+
+    private static async Task<DependencyGraph> ComputeDependencyGraphAsync(string[] packageIds, GraphCommandSettings settings, ILogger logger, CancellationToken cancellationToken)
+    {
+        var nugetSettings = Settings.LoadDefaultSettings(null);
+        using var sourceCacheContext = new SourceCacheContext();
+        var packageSources = GetPackageSources(nugetSettings, logger);
+        var packageIdentityResolver = new NuGetPackageResolver(nugetSettings, logger, packageSources, sourceCacheContext);
+
+        var packageInformation = new ConcurrentBag<FindPackageByIdDependencyInfo>();
+        var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = settings.MaxDegreeOfParallelism, CancellationToken = cancellationToken };
+        await Parallel.ForEachAsync(packageIds, parallelOptions, async (packageId, ct) =>
+        {
+            var packageInfo = await packageIdentityResolver.ResolvePackageInfoAsync(packageId, ct);
+            packageInformation.Add(packageInfo);
+        });
+
+        var dependencyGraphSpec = new DependencyGraphSpec(isReadOnly: true);
+        var projectName = $"dependency graph of {string.Join(", ", packageInformation.Select(e => e.PackageIdentity.Id))}";
+        // TODO: Figure out how to best guess which framework to use if none is specified.
+        var targetFramework = packageInformation.First().DependencyGroups.Select(e => e.TargetFramework).OrderBy(e => e, NuGetFrameworkSorter.Instance).ToList();
+        var framework = settings.Framework == null ? targetFramework.First() : NuGetFramework.Parse(settings.Framework);
+        IList<TargetFrameworkInformation> targetFrameworks = [ new TargetFrameworkInformation { FrameworkName = framework } ];
+        var projectSpec = new PackageSpec(targetFrameworks)
+        {
+            FilePath = projectName,
+            Name = projectName,
+            RestoreMetadata = new ProjectRestoreMetadata
+            {
+                ProjectName = projectName,
+                ProjectPath = projectName,
+                ProjectUniqueName = Guid.NewGuid().ToString(),
+                ProjectStyle = ProjectStyle.PackageReference,
+                // The output path is required, else we get NuGet.Commands.RestoreSpecException: Invalid restore input. Missing required property 'OutputPath' for project type 'PackageReference'.
+                // But it won't be used anyway since restore is performed with RestoreRunner.RunWithoutCommit instead of RestoreRunner.RunAsync
+                OutputPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.InternetCache), "nugraph"),
+                OriginalTargetFrameworks = targetFrameworks.Select(e => e.ToString()).ToList(),
+                Sources = packageSources,
+            },
+            Dependencies = packageInformation.Select(e => new LibraryDependency(new LibraryRange(e.PackageIdentity.Id, new VersionRange(e.PackageIdentity.Version), LibraryDependencyTarget.Package))).ToList(),
+        };
+        dependencyGraphSpec.AddProject(projectSpec);
+        dependencyGraphSpec.AddRestore(projectSpec.RestoreMetadata.ProjectUniqueName);
+
+        var restoreCommandProvidersCache = new RestoreCommandProvidersCache();
+        var dependencyGraphSpecRequestProvider = new DependencyGraphSpecRequestProvider(restoreCommandProvidersCache, dependencyGraphSpec, nugetSettings);
+        var restoreContext = new RestoreArgs
+        {
+            CacheContext = sourceCacheContext,
+            Log = logger,
+            GlobalPackagesFolder = SettingsUtility.GetGlobalPackagesFolder(nugetSettings),
+            PreLoadedRequestProviders = [ dependencyGraphSpecRequestProvider ],
+        };
+
+        var requests = await RestoreRunner.GetRequests(restoreContext);
+        // TODO: Single() => how can I be sure? If only one request? And how can I be sure that there's only one request created out of the restore context?
+        var restoreResultPair = (await RestoreRunner.RunWithoutCommit(requests, restoreContext)).Single();
+        // TODO: filter log messages, only those with LogLevel == Error ?
+        if (!restoreResultPair.Result.Success)
+            throw new Exception(string.Join(Environment.NewLine, restoreResultPair.Result.LogMessages.Select(e => $"[{e.Code}] {e.Message}")));
+
+        var lockFile = restoreResultPair.Result.LockFile;
+        // TODO: build the package and roots out of restoreResultPair.Result.RestoreGraphs instead of the lock file?
+        var (packages, roots) = lockFile.ReadPackages(targetFrameworks.First().TargetAlias, settings.RuntimeIdentifier);
+        return new DependencyGraph(packages, roots, settings.GraphIgnore);
+    }
+
+    private static IList<PackageSource> GetPackageSources(ISettings settings, ILogger logger)
+    {
+        var packageSourceProvider = new PackageSourceProvider(settings);
+        var packageSources = packageSourceProvider.LoadPackageSources().Where(e => e.IsEnabled).Distinct().ToList();
+
+        if (packageSources.Count == 0)
+        {
+            var officialPackageSource = new PackageSource(NuGetConstants.V3FeedUrl, NuGetConstants.NuGetHostName);
+            packageSources.Add(officialPackageSource);
+            var configFilePaths = settings.GetConfigFilePaths().Distinct();
+            logger.LogWarning($"No NuGet sources could be found in {string.Join(", ", configFilePaths)}. Using {officialPackageSource}");
+        }
+
+        return packageSources;
+    }
+
+    private static async Task<Uri?> WriteGraphAsync(DependencyGraph graph, GraphCommandSettings settings)
+    {
+        await using var fileStream = settings.OutputFile?.OpenWrite();
+        await using var memoryStream = fileStream == null ? new MemoryStream(capacity: 2048) : null;
+        var stream = (fileStream ?? memoryStream as Stream)!;
+        await using (var streamWriter = new StreamWriter(stream, leaveOpen: true))
+        {
+            var isMermaid = fileStream == null || Path.GetExtension(fileStream.Name) is ".mmd" or ".mermaid";
+            var graphWriter = isMermaid ? GraphWriter.Mermaid(streamWriter) : GraphWriter.Graphviz(streamWriter);
+            var graphOptions = new GraphOptions
+            {
+                Direction = settings.GraphDirection,
+                IncludeVersions = settings.GraphIncludeVersions,
+                WriteIgnoredPackages = settings.GraphWriteIgnoredPackages,
+            };
+            graphWriter.Write(graph, graphOptions);
+        }
+
+        return memoryStream == null ? null : Mermaid.GetLiveEditorUri(memoryStream.GetBuffer().AsSpan(0, Convert.ToInt32(memoryStream.Position)), settings.MermaidEditorMode);
+    }
+}

src/nugraph/GraphCommandSettings.cs

@@ -0,0 +1,105 @@
+using System.ComponentModel;
+using System.Diagnostics.CodeAnalysis;
+using System.IO;
+using Chisel;
+using NuGet.Common;
+using Spectre.Console.Cli;
+
+namespace nugraph;
+
+[SuppressMessage("ReSharper", "AutoPropertyCanBeMadeGetOnly.Global", Justification = "Required for Spectre.Console.Cli binding")]
+[SuppressMessage("ReSharper", "UnusedAutoPropertyAccessor.Global", Justification = "Required for Spectre.Console.Cli binding")]
+internal class GraphCommandSettings : CommandSettings
+{
+    // TODO: Support only one package? Users can create a project with multiple package reference.
+    [CommandArgument(0, "[SOURCE]")]
+    [Description("The source of the graph. Can be either a directory containing a .NET project, a .NET project file (csproj) or names of NuGet packages.")]
+    public string[] Sources { get; init; } = [];
+
+    // TODO: perform of NuGet package id (including version) in the Validate method
+    internal FileOrPackages GetSource()
+    {
+        if (Sources.Length == 0)
+            return (FileSystemInfo?)null;
+
+        if (Sources.Length == 1)
+        {
+            var file = new FileInfo(Sources[0]);
+            if (file.Exists)
+            {
+                return file;
+            }
+
+            var directory = new DirectoryInfo(Sources[0]);
+            if (directory.Exists)
+            {
+                return directory;
+            }
+        }
+
+        return Sources;
+    }
+
+    [CommandOption("-V|--version")]
+    [Description("Prints version information")]
+    public bool PrintVersion { get; init; }
+
+    [CommandOption("-o|--output <OUTPUT>")]
+    [Description("The path to the dependency graph output file. If not specified, the dependency graph URL is written on the standard output and opened in the browser.")]
+    public FileInfo? OutputFile { get; init; }
+
+    [CommandOption("-f|--framework <FRAMEWORK>")]
+    [Description("The target framework to consider when building the dependency graph.")]
+    public string? Framework { get; init; }
+
+    [CommandOption("-r|--runtime <RUNTIME_IDENTIFIER>")]
+    [Description("The target runtime to consider when building the dependency graph.")]
+    public string? RuntimeIdentifier { get; init; }
+
+    // TODO: option to choose Mermaid with https://mermaid.live vs Graphviz/DOT with https://edotor.net
+
+    // TODO: option to disable opening the url in the default web browser in case (thus only printing the URL on stdout)
+
+    [CommandOption("-m|--mode <MERMAID_MODE>")]
+    [Description($"The mode to use for the Mermaid Live Editor (https://mermaid.live). Possible values are [b]{nameof(MermaidEditorMode.View)}[/] and [b]{nameof(MermaidEditorMode.Edit)}[/]. " +
+                 $"Used only when no output path is specified.")]
+    [DefaultValue(MermaidEditorMode.View)]
+    public MermaidEditorMode MermaidEditorMode { get; init; }
+
+    [CommandOption("-d|--direction <GRAPH_DIRECTION>")]
+    [Description($"The direction of the dependency graph. Possible values are [b]{nameof(GraphDirection.LeftToRight)}[/] and [b]{nameof(GraphDirection.TopToBottom)}[/]")]
+    [DefaultValue(GraphDirection.LeftToRight)]
+    public GraphDirection GraphDirection { get; init; }
+
+    [CommandOption("-v|--include-version")]
+    [Description("Include package versions in the dependency graph. E.g. [b]Serilog/3.1.1[/] instead of [b]Serilog[/]")]
+    [DefaultValue(false)]
+    public bool GraphIncludeVersions { get; init; }
+
+    [CommandOption("-i|--ignore")]
+    [Description("Packages to ignore in the dependency graph. May be used multiple times.")]
+    public string[] GraphIgnore { get; init; } = [];
+
+    [CommandOption("-l|--log <LEVEL>")]
+    [Description($"The NuGet operations log level. Possible values are [b]{nameof(LogLevel.Debug)}[/], [b]{nameof(LogLevel.Verbose)}[/], [b]{nameof(LogLevel.Information)}[/], [b]{nameof(LogLevel.Minimal)}[/], [b]{nameof(LogLevel.Warning)}[/] and [b]{nameof(LogLevel.Error)}[/]")]
+#if DEBUG
+    [DefaultValue(LogLevel.Debug)]
+#else
+    [DefaultValue(LogLevel.Warning)]
+#endif
+    public LogLevel LogLevel { get; init; }
+
+    [CommandOption("--nuget-root")]
+    [Description("The NuGet root directory. Can be used to completely isolate nugraph from default NuGet operations.")]
+    public string? NuGetRoot { get; init; }
+
+    [CommandOption("--include-ignored-packages", IsHidden = true)]
+    [Description("Include ignored packages in the dependency graph. Used for debugging.")]
+    [DefaultValue(false)]
+    public bool GraphWriteIgnoredPackages { get; init; }
+
+    [CommandOption("--parallel", IsHidden = true)]
+    [Description("The maximum degree of parallelism.")]
+    [DefaultValue(16)]
+    public int MaxDegreeOfParallelism { get; init; }
+}