diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index bfac2917c4..1b466eddad 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -103,7 +103,7 @@ jobs: - name: Checkout source if: steps.check.outputs.result == '' - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Validate Microsoft Git version if: steps.check.outputs.result == '' @@ -138,7 +138,7 @@ jobs: - name: Checkout source if: steps.skip.outputs.result != 'true' - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: path: src diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index a6e619ff7c..766f188aea 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -127,6 +127,7 @@ public static class GitStatusCache { public const string Name = "gitStatusCache"; public static readonly string CachePath = Path.Combine(Name, "GitStatusCache.dat"); + public static readonly string TreeCount = Path.Combine(Name, "TreeCountCache.dat"); } } diff --git a/GVFS/GVFS.Common/Git/GitProcess.cs b/GVFS/GVFS.Common/Git/GitProcess.cs index 6b31c4f503..616d0ec22c 100644 --- a/GVFS/GVFS.Common/Git/GitProcess.cs +++ b/GVFS/GVFS.Common/Git/GitProcess.cs @@ -682,6 +682,11 @@ public Result MultiPackIndexRepack(string gitObjectDirectory, string batchSize) return this.InvokeGitAgainstDotGitFolder($"-c pack.threads=1 -c repack.packKeptObjects=true multi-pack-index repack --object-dir=\"{gitObjectDirectory}\" --batch-size={batchSize} --no-progress"); } + public Result GetHeadTreeId() + { + return this.InvokeGitAgainstDotGitFolder("show -s --format=%T HEAD", usePreCommandHook: false); + } + public Process GetGitProcess(string command, string workingDirectory, string dotGitDirectory, bool useReadObjectHook, bool redirectStandardError, string gitObjectsDirectory, bool usePreCommandHook) { ProcessStartInfo processInfo = new ProcessStartInfo(this.gitBinPath); diff --git a/GVFS/GVFS.Common/GitStatusCache.cs b/GVFS/GVFS.Common/GitStatusCache.cs index 0b633b6dfa..717e7aebca 100644 --- a/GVFS/GVFS.Common/GitStatusCache.cs +++ b/GVFS/GVFS.Common/GitStatusCache.cs @@ -52,6 +52,8 @@ public class GitStatusCache : IDisposable private object cacheFileLock = new object(); + internal static bool TEST_EnableHydrationSummary = true; + public GitStatusCache(GVFSContext context, GitStatusCacheConfig config) : this(context, config.BackoffTime) { @@ -315,6 +317,7 @@ private void RebuildStatusCacheIfNeeded(bool ignoreBackoff) if (needToRebuild) { this.statistics.RecordBackgroundStatusScanRun(); + this.UpdateHydrationSummary(); bool rebuildStatusCacheSucceeded = this.TryRebuildStatusCache(); @@ -336,6 +339,57 @@ private void RebuildStatusCacheIfNeeded(bool ignoreBackoff) } } + private void UpdateHydrationSummary() + { + if (!TEST_EnableHydrationSummary) + { + return; + } + + try + { + /* While not strictly part of git status, enlistment hydration summary is used + * in "git status" pre-command hook, and can take several seconds to compute on very large repos. + * Accessing it here ensures that the value is cached for when a user invokes "git status", + * and this is also a convenient place to log telemetry for it. + */ + EnlistmentHydrationSummary hydrationSummary = + EnlistmentHydrationSummary.CreateSummary(this.context.Enlistment, this.context.FileSystem); + EventMetadata metadata = new EventMetadata(); + metadata.Add("Area", EtwArea); + if (hydrationSummary.IsValid) + { + metadata[nameof(hydrationSummary.TotalFolderCount)] = hydrationSummary.TotalFolderCount; + metadata[nameof(hydrationSummary.TotalFileCount)] = hydrationSummary.TotalFileCount; + metadata[nameof(hydrationSummary.HydratedFolderCount)] = hydrationSummary.HydratedFolderCount; + metadata[nameof(hydrationSummary.HydratedFileCount)] = hydrationSummary.HydratedFileCount; + + this.context.Tracer.RelatedEvent( + EventLevel.Informational, + nameof(EnlistmentHydrationSummary), + metadata, + Keywords.Telemetry); + } + else + { + this.context.Tracer.RelatedWarning( + metadata, + $"{nameof(GitStatusCache)}{nameof(RebuildStatusCacheIfNeeded)}: hydration summary could not be calculdated.", + Keywords.Telemetry); + } + } + catch (Exception ex) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Area", EtwArea); + metadata.Add("Exception", ex.ToString()); + this.context.Tracer.RelatedError( + metadata, + $"{nameof(GitStatusCache)}{nameof(RebuildStatusCacheIfNeeded)}: Exception trying to update hydration summary cache.", + Keywords.Telemetry); + } + } + /// /// Rebuild the status cache. This will run the background status to /// generate status results, and update the serialized status cache diff --git a/GVFS/GVFS.Common/HealthCalculator/EnlistmentHealthCalculator.cs b/GVFS/GVFS.Common/HealthCalculator/EnlistmentHealthCalculator.cs index 2d048e9f22..b437694eef 100644 --- a/GVFS/GVFS.Common/HealthCalculator/EnlistmentHealthCalculator.cs +++ b/GVFS/GVFS.Common/HealthCalculator/EnlistmentHealthCalculator.cs @@ -1,5 +1,4 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.Linq; namespace GVFS.Common diff --git a/GVFS/GVFS.Common/HealthCalculator/EnlistmentHydrationSummary.cs b/GVFS/GVFS.Common/HealthCalculator/EnlistmentHydrationSummary.cs new file mode 100644 index 0000000000..02a32c2f5e --- /dev/null +++ b/GVFS/GVFS.Common/HealthCalculator/EnlistmentHydrationSummary.cs @@ -0,0 +1,174 @@ +using GVFS.Common.FileSystem; +using GVFS.Common.Git; +using System; +using System.IO; +using System.Linq; + +namespace GVFS.Common +{ + public class EnlistmentHydrationSummary + { + public int HydratedFileCount { get; private set; } + public int TotalFileCount { get; private set; } + public int HydratedFolderCount { get; private set; } + public int TotalFolderCount { get; private set; } + + public bool IsValid + { + get + { + return HydratedFileCount >= 0 + && HydratedFolderCount >= 0 + && TotalFileCount >= HydratedFileCount + && TotalFolderCount >= HydratedFolderCount; + } + } + + public string ToMessage() + { + if (!IsValid) + { + return "Error calculating hydration. Run 'gvfs health' for details."; + } + + int fileHydrationPercent = TotalFileCount == 0 ? 0 : (100 * HydratedFileCount) / TotalFileCount; + int folderHydrationPercent = TotalFolderCount == 0 ? 0 : ((100 * HydratedFolderCount) / TotalFolderCount); + return $"{fileHydrationPercent}% of files and {folderHydrationPercent}% of folders hydrated. Run 'gvfs health' for details."; + } + + public static EnlistmentHydrationSummary CreateSummary( + GVFSEnlistment enlistment, + PhysicalFileSystem fileSystem) + { + try + { + /* Getting all the file paths from git index is slow and we only need the total count, + * so we read the index file header instead of calling GetPathsFromGitIndex */ + int totalFileCount = GetIndexFileCount(enlistment, fileSystem); + + /* Getting all the directories is also slow, but not as slow as reading the entire index, + * GetTotalPathCount caches the count so this is only slow occasionally, + * and the GitStatusCache manager also calls this to ensure it is updated frequently. */ + int totalFolderCount = GetHeadTreeCount(enlistment, fileSystem); + + EnlistmentPathData pathData = new EnlistmentPathData(); + + /* FUTURE: These could be optimized to only deal with counts instead of full path lists */ + pathData.LoadPlaceholdersFromDatabase(enlistment); + pathData.LoadModifiedPaths(enlistment); + + int hydratedFileCount = pathData.ModifiedFilePaths.Count + pathData.PlaceholderFilePaths.Count; + int hydratedFolderCount = pathData.ModifiedFolderPaths.Count + pathData.PlaceholderFolderPaths.Count; + return new EnlistmentHydrationSummary() + { + HydratedFileCount = hydratedFileCount, + HydratedFolderCount = hydratedFolderCount, + TotalFileCount = totalFileCount, + TotalFolderCount = totalFolderCount, + }; + } + catch + { + return new EnlistmentHydrationSummary() + { + HydratedFileCount = -1, + HydratedFolderCount = -1, + TotalFileCount = -1, + TotalFolderCount = -1, + }; + } + } + + /// + /// Get the total number of files in the index. + /// + internal static int GetIndexFileCount(GVFSEnlistment enlistment, PhysicalFileSystem fileSystem) + { + string indexPath = Path.Combine(enlistment.WorkingDirectoryBackingRoot, GVFSConstants.DotGit.Index); + using (var indexFile = fileSystem.OpenFileStream(indexPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, callFlushFileBuffers: false)) + { + if (indexFile.Length < 12) + { + return -1; + } + /* The number of files in the index is a big-endian integer from + * the 4 bytes at offsets 8-11 of the index file. */ + indexFile.Position = 8; + var bytes = new byte[4]; + indexFile.Read( + bytes, // Destination buffer + offset: 0, // Offset in destination buffer, not in indexFile + count: 4); + if (BitConverter.IsLittleEndian) + { + Array.Reverse(bytes); + } + int count = BitConverter.ToInt32(bytes, 0); + return count; + } + } + + /// + /// Get the total number of trees in the repo at HEAD. + /// + /// + /// This is used as the denominator in displaying percentage of hydrated + /// directories as part of git status pre-command hook. + /// It can take several seconds to calculate, so we cache it near the git status cache. + /// + /// + /// The number of subtrees at HEAD, which may be 0. + /// Will return 0 if unsuccessful. + /// + internal static int GetHeadTreeCount(GVFSEnlistment enlistment, PhysicalFileSystem fileSystem) + { + var gitProcess = enlistment.CreateGitProcess(); + var headResult = gitProcess.GetHeadTreeId(); + if (headResult.ExitCodeIsFailure) + { + return 0; + } + var headSha = headResult.Output.Trim(); + var cacheFile = Path.Combine( + enlistment.DotGVFSRoot, + GVFSConstants.DotGVFS.GitStatusCache.TreeCount); + + // Load from cache if cache matches current HEAD. + if (fileSystem.FileExists(cacheFile)) + { + try + { + var lines = fileSystem.ReadLines(cacheFile).ToArray(); + if (lines.Length == 2 + && lines[0] == headSha + && int.TryParse(lines[1], out int cachedCount)) + { + return cachedCount; + } + } + catch + { + // Ignore errors reading the cache + } + } + + int totalPathCount = 0; + GitProcess.Result folderResult = gitProcess.LsTree( + GVFSConstants.DotGit.HeadName, + line => totalPathCount++, + recursive: true, + showDirectories: true); + try + { + fileSystem.CreateDirectory(Path.GetDirectoryName(cacheFile)); + fileSystem.WriteAllText(cacheFile, $"{headSha}\n{totalPathCount}"); + } + catch + { + // Ignore errors writing the cache + } + + return totalPathCount; + } + } +} diff --git a/GVFS/GVFS.Common/HealthCalculator/EnlistmentPathData.cs b/GVFS/GVFS.Common/HealthCalculator/EnlistmentPathData.cs index a7ecaafae4..911b3f8059 100644 --- a/GVFS/GVFS.Common/HealthCalculator/EnlistmentPathData.cs +++ b/GVFS/GVFS.Common/HealthCalculator/EnlistmentPathData.cs @@ -1,5 +1,10 @@ -using System.Collections; +using GVFS.Common.Database; +using GVFS.Common.FileSystem; +using GVFS.Common.Git; +using GVFS.Common.NamedPipes; +using System; using System.Collections.Generic; +using System.IO; using System.Linq; namespace GVFS.Common @@ -38,6 +43,157 @@ public void NormalizeAllPaths() this.ModifiedFilePaths = this.ModifiedFilePaths.Union(this.GitTrackingPaths).ToList(); } + /// + /// Get two lists of placeholders, one containing the files and the other the directories + /// Goes to the SQLite database for the placeholder lists + /// + /// The current GVFS enlistment being operated on + public void LoadPlaceholdersFromDatabase(GVFSEnlistment enlistment) + { + List filePlaceholders = new List(); + List folderPlaceholders = new List(); + + using (GVFSDatabase database = new GVFSDatabase(new PhysicalFileSystem(), enlistment.EnlistmentRoot, new SqliteDatabase())) + { + PlaceholderTable placeholderTable = new PlaceholderTable(database); + placeholderTable.GetAllEntries(out filePlaceholders, out folderPlaceholders); + } + + this.PlaceholderFilePaths.AddRange(filePlaceholders.Select(placeholderData => placeholderData.Path)); + this.PlaceholderFolderPaths.AddRange(folderPlaceholders.Select(placeholderData => placeholderData.Path)); + } + + /// + /// Call 'git ls-files' and 'git ls-tree' to get a list of all files and directories in the enlistment + /// + /// The current GVFS enlistmetn being operated on + public void LoadPathsFromGitIndex(GVFSEnlistment enlistment) + { + List skipWorktreeFiles = new List(); + GitProcess gitProcess = new GitProcess(enlistment); + + GitProcess.Result fileResult = gitProcess.LsFiles( + line => + { + if (line.First() == 'H') + { + skipWorktreeFiles.Add(TrimGitIndexLineWithSkipWorktree(line)); + } + + this.GitFilePaths.Add(TrimGitIndexLineWithSkipWorktree(line)); + }); + GitProcess.Result folderResult = gitProcess.LsTree( + GVFSConstants.DotGit.HeadName, + line => + { + this.GitFolderPaths.Add(TrimGitIndexLine(line)); + }, + recursive: true, + showDirectories: true); + + this.GitTrackingPaths.AddRange(skipWorktreeFiles); + } + + public void LoadModifiedPaths(GVFSEnlistment enlistment) + { + if (TryLoadModifiedPathsFromPipe(enlistment)) + { + return; + } + try + { + /* Most likely GVFS is not mounted. Give a basic effort to read the modified paths database */ + var filePath = Path.Combine(enlistment.DotGVFSRoot, GVFSConstants.DotGVFS.Databases.ModifiedPaths); + using (var file = File.Open(filePath, FileMode.OpenOrCreate, FileAccess.Read, FileShare.Read)) + using (var reader = new StreamReader(file)) + { + AddModifiedPaths(ReadModifiedPathDatabaseLines(reader)); + } + } + catch { } + } + + private IEnumerable ReadModifiedPathDatabaseLines(StreamReader r) + { + while (!r.EndOfStream) + { + var line = r.ReadLine(); + const string LinePrefix = "A "; + if (line.StartsWith(LinePrefix)) + { + line = line.Substring(LinePrefix.Length); + } + yield return line; + } + } + + /// + /// Talk to the mount process across the named pipe to get a list of the modified paths + /// + /// If/when modified paths are moved to SQLite go there instead + /// The enlistment being operated on + /// An array containing all of the modified paths in string format + private bool TryLoadModifiedPathsFromPipe(GVFSEnlistment enlistment) + { + using (NamedPipeClient pipeClient = new NamedPipeClient(enlistment.NamedPipeName)) + { + string[] modifiedPathsList = Array.Empty(); + + if (!pipeClient.Connect()) + { + return false; + } + + try + { + NamedPipeMessages.Message modifiedPathsMessage = new NamedPipeMessages.Message(NamedPipeMessages.ModifiedPaths.ListRequest, NamedPipeMessages.ModifiedPaths.CurrentVersion); + pipeClient.SendRequest(modifiedPathsMessage); + + NamedPipeMessages.Message modifiedPathsResponse = pipeClient.ReadResponse(); + if (!modifiedPathsResponse.Header.Equals(NamedPipeMessages.ModifiedPaths.SuccessResult)) + { + return false; + } + + modifiedPathsList = modifiedPathsResponse.Body.Split(new char[] { '\0' }, StringSplitOptions.RemoveEmptyEntries); + } + catch (BrokenPipeException e) + { + return false; + } + + AddModifiedPaths(modifiedPathsList); + return true; + } + } + + private void AddModifiedPaths(IEnumerable modifiedPathsList) + { + foreach (string path in modifiedPathsList) + { + if (path.Last() == GVFSConstants.GitPathSeparator) + { + path.TrimEnd(GVFSConstants.GitPathSeparator); + this.ModifiedFolderPaths.Add(path); + } + else + { + this.ModifiedFilePaths.Add(path); + } + } + } + + /// + /// Parse a line of the git index coming from the ls-files endpoint in the git process to get the path to that files + /// These paths begin with 'S' or 'H' depending on if they have the skip-worktree bit set + /// + /// The line from the output of the git index + /// The path extracted from the provided line of the git index + private static string TrimGitIndexLineWithSkipWorktree(string line) + { + return line.Substring(line.IndexOf(' ') + 1); + } + private void NormalizePaths(List paths) { for (int i = 0; i < paths.Count; i++) @@ -46,5 +202,15 @@ private void NormalizePaths(List paths) paths[i] = paths[i].Trim(GVFSConstants.GitPathSeparator); } } + + /// + /// Parse a line of the git index coming from the ls-tree endpoint in the git process to get the path to that file + /// + /// The line from the output of the git index + /// The path extracted from the provided line of the git index + private static string TrimGitIndexLine(string line) + { + return line.Substring(line.IndexOf('\t') + 1); + } } } diff --git a/GVFS/GVFS.Common/Maintenance/PrefetchStep.cs b/GVFS/GVFS.Common/Maintenance/PrefetchStep.cs index 901ec4cf20..b67ad6ae77 100644 --- a/GVFS/GVFS.Common/Maintenance/PrefetchStep.cs +++ b/GVFS/GVFS.Common/Maintenance/PrefetchStep.cs @@ -16,6 +16,7 @@ public class PrefetchStep : GitMaintenanceStep private const int LockWaitTimeMs = 100; private const int WaitingOnLockLogThreshold = 50; private const string PrefetchCommitsAndTreesLock = "prefetch-commits-trees.lock"; + private const int NoExistingPrefetchPacks = -1; private readonly TimeSpan timeBetweenPrefetches = TimeSpan.FromMinutes(70); public PrefetchStep(GVFSContext context, GitObjects gitObjects, bool requireCacheLock = true) @@ -80,6 +81,18 @@ protected override void PerformMaintenance() return; } + if (last == NoExistingPrefetchPacks) + { + /* If there are no existing prefetch packs, that means that either the + * first prefetch is still in progress or the clone was run with "--no-prefetch". + * In either case, we should not run prefetch as a maintenance task. + * If users want to prefetch after cloning with "--no-prefetch", they can run + * "gvfs prefetch" manually. Also, "git pull" and "git fetch" will run prefetch + * as a pre-command hook. */ + this.Context.Tracer.RelatedInfo(this.Area + ": Skipping prefetch since there are no existing prefetch packs"); + return; + } + DateTime lastDateTime = EpochConverter.FromUnixEpochSeconds(last); DateTime now = DateTime.UtcNow; @@ -150,7 +163,7 @@ private bool TryGetMaxGoodPrefetchTimestamp(out long maxGoodTimestamp, out strin .OrderBy(packInfo => packInfo.Timestamp) .ToList(); - maxGoodTimestamp = -1; + maxGoodTimestamp = NoExistingPrefetchPacks; int firstBadPack = -1; for (int i = 0; i < orderedPacks.Count; ++i) diff --git a/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs b/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs index 76a77eb64e..8bdc74f17a 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs @@ -65,11 +65,12 @@ public static ProcessResult InvokeGitAgainstGVFSRepo( bool removeWaitingMessages = true, bool removeUpgradeMessages = true, bool removePartialHydrationMessages = true, - bool removeFSMonitorMessages = true) + bool removeFSMonitorMessages = true, + bool removeGvfsHealthMessages = true) { ProcessResult result = GitProcess.InvokeProcess(gvfsRepoRoot, command, environmentVariables); - string output = FilterMessages(result.Output, false, false, false, removePartialHydrationMessages, removeFSMonitorMessages); - string errors = FilterMessages(result.Errors, true, removeWaitingMessages, removeUpgradeMessages, removePartialHydrationMessages, removeFSMonitorMessages); + string output = FilterMessages(result.Output, false, false, false, removePartialHydrationMessages, removeFSMonitorMessages, removeGvfsHealthMessages); + string errors = FilterMessages(result.Errors, true, removeWaitingMessages, removeUpgradeMessages, removePartialHydrationMessages, removeFSMonitorMessages, removeGvfsHealthMessages); return new ProcessResult( output, @@ -100,7 +101,8 @@ private static string FilterMessages( bool removeWaitingMessages, bool removeUpgradeMessages, bool removePartialHydrationMessages, - bool removeFSMonitorMessages) + bool removeFSMonitorMessages, + bool removeGvfsHealthMessages) { if (!string.IsNullOrEmpty(input) && (removeWaitingMessages || removeUpgradeMessages || removePartialHydrationMessages || removeFSMonitorMessages)) { @@ -111,6 +113,7 @@ private static string FilterMessages( (removeUpgradeMessages && line.StartsWith("A new version of VFS for Git is available.")) || (removeWaitingMessages && line.StartsWith("Waiting for ")) || (removePartialHydrationMessages && line.StartsWith("You are in a partially-hydrated checkout with ")) || + (removeGvfsHealthMessages && line.TrimEnd().EndsWith("Run 'gvfs health' for details.")) || (removeFSMonitorMessages && line.TrimEnd().EndsWith(" is incompatible with fsmonitor"))) { return false; diff --git a/GVFS/GVFS.Hooks/Program.cs b/GVFS/GVFS.Hooks/Program.cs index 4dc9aeb99e..0bfe0da543 100644 --- a/GVFS/GVFS.Hooks/Program.cs +++ b/GVFS/GVFS.Hooks/Program.cs @@ -87,6 +87,14 @@ private static void RunPreCommands(string[] args) case "pull": ProcessHelper.Run("gvfs", "prefetch --commits", redirectOutput: false); break; + case "status": + /* If status is being run to serialize for caching, skip the health display */ + if (!args.Any(arg => arg.StartsWith("--serialize", StringComparison.OrdinalIgnoreCase))) + { + /* Display a message about the hydration status of the repo */ + ProcessHelper.Run("gvfs", "health --status", redirectOutput: false); + } + break; } } diff --git a/GVFS/GVFS.UnitTests/Common/EnlistmentHydrationSummaryTests.cs b/GVFS/GVFS.UnitTests/Common/EnlistmentHydrationSummaryTests.cs new file mode 100644 index 0000000000..53591de8ae --- /dev/null +++ b/GVFS/GVFS.UnitTests/Common/EnlistmentHydrationSummaryTests.cs @@ -0,0 +1,125 @@ +using GVFS.Common; +using GVFS.Common.Git; +using GVFS.Tests.Should; +using GVFS.UnitTests.Mock.Common; +using GVFS.UnitTests.Mock.FileSystem; +using GVFS.UnitTests.Mock.Git; +using NUnit.Framework; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace GVFS.UnitTests.Common +{ + [TestFixture] + public class EnlistmentHydrationSummaryTests + { + private MockFileSystem fileSystem; + private MockGitProcess gitProcess; + private GVFSContext context; + private string gitParentPath; + private string gvfsMetadataPath; + private MockDirectory enlistmentDirectory; + + private const string HeadTreeId = "0123456789012345678901234567890123456789"; + private const int HeadPathCount = 42; + + public static IEnumerable<(string CachePrecontents, string ExpectedCachePostContents)> HeadTreeCountCacheContents + { + get + { + yield return (null, $"{HeadTreeId}\n{HeadPathCount}"); + yield return ($"{HeadTreeId}\n{HeadPathCount}", $"{HeadTreeId}\n{HeadPathCount}"); + yield return ($"{HeadTreeId}\n{HeadPathCount - 1}", $"{HeadTreeId}\n{HeadPathCount - 1}"); + yield return ($"{HeadTreeId.Replace("1", "a")}\n{HeadPathCount - 1}", $"{HeadTreeId}\n{HeadPathCount}"); + yield return ($"{HeadTreeId}\nabc", $"{HeadTreeId}\n{HeadPathCount}"); + yield return ($"{HeadTreeId}\nabc", $"{HeadTreeId}\n{HeadPathCount}"); + yield return ($"\n", $"{HeadTreeId}\n{HeadPathCount}"); + yield return ($"\nabc", $"{HeadTreeId}\n{HeadPathCount}"); + } + } + + [SetUp] + public void Setup() + { + MockTracer tracer = new MockTracer(); + + string enlistmentRoot = Path.Combine("mock:", "GVFS", "UnitTests", "Repo"); + string statusCachePath = Path.Combine("mock:", "GVFS", "UnitTests", "Repo", GVFSPlatform.Instance.Constants.DotGVFSRoot, "gitStatusCache"); + + this.gitProcess = new MockGitProcess(); + this.gitProcess.SetExpectedCommandResult($"--no-optional-locks status \"--serialize={statusCachePath}", () => new GitProcess.Result(string.Empty, string.Empty, 0), true); + MockGVFSEnlistment enlistment = new MockGVFSEnlistment(enlistmentRoot, "fake://repoUrl", "fake://gitBinPath", this.gitProcess); + enlistment.InitializeCachePathsFromKey("fake:\\gvfsSharedCache", "fakeCacheKey"); + + this.gitParentPath = enlistment.WorkingDirectoryBackingRoot; + this.gvfsMetadataPath = enlistment.DotGVFSRoot; + + this.enlistmentDirectory = new MockDirectory( + enlistmentRoot, + new MockDirectory[] + { + new MockDirectory(this.gitParentPath, folders: null, files: null), + }, + null); + + this.enlistmentDirectory.CreateFile(Path.Combine(this.gitParentPath, ".git", "config"), ".git config Contents", createDirectories: true); + this.enlistmentDirectory.CreateFile(Path.Combine(this.gitParentPath, ".git", "HEAD"), ".git HEAD Contents", createDirectories: true); + this.enlistmentDirectory.CreateFile(Path.Combine(this.gitParentPath, ".git", "logs", "HEAD"), "HEAD Contents", createDirectories: true); + this.enlistmentDirectory.CreateFile(Path.Combine(this.gitParentPath, ".git", "info", "always_exclude"), "always_exclude Contents", createDirectories: true); + this.enlistmentDirectory.CreateDirectory(Path.Combine(this.gitParentPath, ".git", "objects", "pack")); + + this.fileSystem = new MockFileSystem(this.enlistmentDirectory); + this.fileSystem.AllowMoveFile = true; + this.fileSystem.DeleteNonExistentFileThrowsException = false; + + this.context = new GVFSContext( + tracer, + this.fileSystem, + new MockGitRepo(tracer, enlistment, this.fileSystem), + enlistment); + } + + [TearDown] + public void TearDown() + { + this.fileSystem = null; + this.gitProcess = null; + this.context = null; + this.gitParentPath = null; + this.gvfsMetadataPath = null; + this.enlistmentDirectory = null; + } + + [TestCaseSource("HeadTreeCountCacheContents")] + public void HeadTreeCountCacheTests((string CachePrecontents, string ExpectedCachePostContents) args) + { + string totalPathCountPath = Path.Combine(this.gvfsMetadataPath, GVFSConstants.DotGVFS.GitStatusCache.TreeCount); + if (args.CachePrecontents != null) + { + this.enlistmentDirectory.CreateFile(totalPathCountPath, args.CachePrecontents, createDirectories: true); + } + + this.gitProcess.SetExpectedCommandResult("show -s --format=%T HEAD", + () => new GitProcess.Result(HeadTreeId, "", 0)); + this.gitProcess.SetExpectedCommandResult("ls-tree -r -d HEAD", + () => new GitProcess.Result( + string.Join("\n", Enumerable.Range(0, HeadPathCount) + .Select(x => x.ToString())), + "", 0)); + + Assert.AreEqual( + args.CachePrecontents != null, + this.fileSystem.FileExists(totalPathCountPath)); + + int result = EnlistmentHydrationSummary.GetHeadTreeCount(this.context.Enlistment, this.context.FileSystem); + + this.fileSystem.FileExists(totalPathCountPath).ShouldBeTrue(); + var postContents = this.fileSystem.ReadAllText(totalPathCountPath); + Assert.AreEqual( + args.ExpectedCachePostContents, + postContents); + Assert.AreEqual(postContents.Split('\n')[1], result.ToString()); + } + } +} diff --git a/GVFS/GVFS.UnitTests/Common/GitStatusCacheTests.cs b/GVFS/GVFS.UnitTests/Common/GitStatusCacheTests.cs index 75e754afea..20ea38171a 100644 --- a/GVFS/GVFS.UnitTests/Common/GitStatusCacheTests.cs +++ b/GVFS/GVFS.UnitTests/Common/GitStatusCacheTests.cs @@ -73,6 +73,7 @@ public void SetUp() this.fileSystem, new MockGitRepo(tracer, enlistment, this.fileSystem), enlistment); + GitStatusCache.TEST_EnableHydrationSummary = false; } [TearDown] @@ -84,6 +85,7 @@ public void TearDown() this.gitParentPath = null; this.gvfsMetadataPath = null; this.enlistmentDirectory = null; + GitStatusCache.TEST_EnableHydrationSummary = true; } [TestCase] diff --git a/GVFS/GVFS.UnitTests/Mock/Git/MockGitProcess.cs b/GVFS/GVFS.UnitTests/Mock/Git/MockGitProcess.cs index 9d8b5e93c5..c9095cc2c8 100644 --- a/GVFS/GVFS.UnitTests/Mock/Git/MockGitProcess.cs +++ b/GVFS/GVFS.UnitTests/Mock/Git/MockGitProcess.cs @@ -109,7 +109,20 @@ protected override Result InvokeGitImpl( CommandInfo matchedCommand = this.expectedCommandInfos.Last(commandMatchFunction); matchedCommand.ShouldNotBeNull("Unexpected command: " + command); - return matchedCommand.Result(); + var result = matchedCommand.Result(); + if (parseStdOutLine != null && !string.IsNullOrEmpty(result.Output)) + { + using (StringReader reader = new StringReader(result.Output)) + { + string line; + while ((line = reader.ReadLine()) != null) + { + parseStdOutLine(line); + } + } + /* Future: result.Output should be set to null in this case */ + } + return result; } public class Credential diff --git a/GVFS/GVFS/CommandLine/DehydrateVerb.cs b/GVFS/GVFS/CommandLine/DehydrateVerb.cs index f3d5224f5e..b739b9b2df 100644 --- a/GVFS/GVFS/CommandLine/DehydrateVerb.cs +++ b/GVFS/GVFS/CommandLine/DehydrateVerb.cs @@ -117,7 +117,7 @@ protected override void Execute(GVFSEnlistment enlistment) if (!this.Confirmed && fullDehydrate) { this.Output.WriteLine( -@"WARNING: THIS IS AN EXPERIMENTAL FEATURE +$@"WARNING: THIS IS AN EXPERIMENTAL FEATURE Dehydrate will back up your src folder, and then create a new, empty src folder with a fresh virtualization of the repo. All of your downloaded objects, branches, @@ -130,8 +130,7 @@ any of your uncommitted changes. in the backup folder, but it will be harder to find them because 'git status' will not work in the backup. -To actually execute the dehydrate, run 'gvfs dehydrate --confirm' from the parent -of your enlistment's src folder. +To actually execute the dehydrate, run 'gvfs dehydrate --confirm' from {enlistment.EnlistmentRoot}. "); return; @@ -155,6 +154,13 @@ from a parent of the folders list. return; } + if (fullDehydrate && Environment.CurrentDirectory.StartsWith(enlistment.WorkingDirectoryBackingRoot)) + { + /* If running from /src, the dehydrate would fail because of the handle we are holding on it. */ + this.Output.WriteLine($"Dehydrate must be run from {enlistment.EnlistmentRoot}"); + return; + } + if (this.NoStatus && !fullDehydrate) { this.ReportErrorAndExit(tracer, "Dehydrate --no-status not valid with --folders"); diff --git a/GVFS/GVFS/CommandLine/HealthVerb.cs b/GVFS/GVFS/CommandLine/HealthVerb.cs index b12b58d2c0..782c014c24 100644 --- a/GVFS/GVFS/CommandLine/HealthVerb.cs +++ b/GVFS/GVFS/CommandLine/HealthVerb.cs @@ -1,9 +1,6 @@ using CommandLine; using GVFS.Common; -using GVFS.Common.Database; using GVFS.Common.FileSystem; -using GVFS.Common.Git; -using GVFS.Common.NamedPipes; using System; using System.Collections.Generic; using System.Linq; @@ -29,10 +26,25 @@ public class HealthVerb : GVFSVerb.ForExistingEnlistment HelpText = "Get the health of a specific directory (default is the current working directory")] public string Directory { get; set; } + [Option( + 's', + "status", + Required = false, + HelpText = "Display only the hydration % of the repository, similar to 'git status' in a repository with sparse-checkout")] + public bool StatusOnly { get; set; } + protected override string VerbName => HealthVerbName; + internal PhysicalFileSystem FileSystem { get; set; } = new PhysicalFileSystem(); + protected override void Execute(GVFSEnlistment enlistment) { + if (this.StatusOnly) + { + this.OutputHydrationPercent(enlistment); + return; + } + // Now default to the current working directory when running the verb without a specified path if (string.IsNullOrEmpty(this.Directory) || this.Directory.Equals(".")) { @@ -53,9 +65,9 @@ protected override void Execute(GVFSEnlistment enlistment) EnlistmentPathData pathData = new EnlistmentPathData(); - this.GetPlaceholdersFromDatabase(enlistment, pathData); - this.GetModifiedPathsFromPipe(enlistment, pathData); - this.GetPathsFromGitIndex(enlistment, pathData); + pathData.LoadPlaceholdersFromDatabase(enlistment); + pathData.LoadModifiedPaths(enlistment); + pathData.LoadPathsFromGitIndex(enlistment); pathData.NormalizeAllPaths(); @@ -65,6 +77,12 @@ protected override void Execute(GVFSEnlistment enlistment) this.PrintOutput(enlistmentHealthData); } + private void OutputHydrationPercent(GVFSEnlistment enlistment) + { + var summary = EnlistmentHydrationSummary.CreateSummary(enlistment, this.FileSystem); + this.Output.WriteLine(summary.ToMessage()); + } + private void PrintOutput(EnlistmentHealthData enlistmentHealthData) { string trackedFilesCountFormatted = enlistmentHealthData.GitTrackedItemsCount.ToString("N0"); @@ -114,131 +132,5 @@ private string FormatPercent(decimal percent) { return percent.ToString("P0").PadLeft(4); } - - /// - /// Parse a line of the git index coming from the ls-tree endpoint in the git process to get the path to that file - /// - /// The line from the output of the git index - /// The path extracted from the provided line of the git index - private string TrimGitIndexLine(string line) - { - return line.Substring(line.IndexOf('\t') + 1); - } - - /// - /// Parse a line of the git index coming from the ls-files endpoint in the git process to get the path to that files - /// These paths begin with 'S' or 'H' depending on if they have the skip-worktree bit set - /// - /// The line from the output of the git index - /// The path extracted from the provided line of the git index - private string TrimGitIndexLineWithSkipWorktree(string line) - { - return line.Substring(line.IndexOf(' ') + 1); - } - - /// - /// Talk to the mount process across the named pipe to get a list of the modified paths - /// - /// If/when modified paths are moved to SQLite go there instead - /// The enlistment being operated on - /// An array containing all of the modified paths in string format - private void GetModifiedPathsFromPipe(GVFSEnlistment enlistment, EnlistmentPathData pathData) - { - using (NamedPipeClient pipeClient = new NamedPipeClient(enlistment.NamedPipeName)) - { - string[] modifiedPathsList = Array.Empty(); - - if (!pipeClient.Connect()) - { - this.ReportErrorAndExit("Unable to connect to GVFS. Try running 'gvfs mount'"); - } - - try - { - NamedPipeMessages.Message modifiedPathsMessage = new NamedPipeMessages.Message(NamedPipeMessages.ModifiedPaths.ListRequest, NamedPipeMessages.ModifiedPaths.CurrentVersion); - pipeClient.SendRequest(modifiedPathsMessage); - - NamedPipeMessages.Message modifiedPathsResponse = pipeClient.ReadResponse(); - if (!modifiedPathsResponse.Header.Equals(NamedPipeMessages.ModifiedPaths.SuccessResult)) - { - this.Output.WriteLine("Bad response from modified path pipe: " + modifiedPathsResponse.Header); - return; - } - - modifiedPathsList = modifiedPathsResponse.Body.Split(new char[] { '\0' }, StringSplitOptions.RemoveEmptyEntries); - } - catch (BrokenPipeException e) - { - this.ReportErrorAndExit("Unable to communicate with GVFS: " + e.ToString()); - } - - foreach (string path in modifiedPathsList) - { - if (path.Last() == GVFSConstants.GitPathSeparator) - { - path.TrimEnd(GVFSConstants.GitPathSeparator); - pathData.ModifiedFolderPaths.Add(path); - } - else - { - pathData.ModifiedFilePaths.Add(path); - } - } - } - } - - /// - /// Get two lists of placeholders, one containing the files and the other the directories - /// Goes to the SQLite database for the placeholder lists - /// - /// The current GVFS enlistment being operated on - /// Out parameter where the list of file placeholders will end up - /// Out parameter where the list of folder placeholders will end up - private void GetPlaceholdersFromDatabase(GVFSEnlistment enlistment, EnlistmentPathData pathData) - { - List filePlaceholders = new List(); - List folderPlaceholders = new List(); - - using (GVFSDatabase database = new GVFSDatabase(new PhysicalFileSystem(), enlistment.EnlistmentRoot, new SqliteDatabase())) - { - PlaceholderTable placeholderTable = new PlaceholderTable(database); - placeholderTable.GetAllEntries(out filePlaceholders, out folderPlaceholders); - } - - pathData.PlaceholderFilePaths.AddRange(filePlaceholders.Select(placeholderData => placeholderData.Path)); - pathData.PlaceholderFolderPaths.AddRange(folderPlaceholders.Select(placeholderData => placeholderData.Path)); - } - - /// - /// Call 'git ls-files' and 'git ls-tree' to get a list of all files and directories in the enlistment - /// - /// The current GVFS enlistmetn being operated on - /// The path data object where paths are being saved - private void GetPathsFromGitIndex(GVFSEnlistment enlistment, EnlistmentPathData pathData) - { - List skipWorktreeFiles = new List(); - GitProcess gitProcess = new GitProcess(enlistment); - - GitProcess.Result fileResult = gitProcess.LsFiles( - line => - { - if (line.First() == 'H') - { - skipWorktreeFiles.Add(this.TrimGitIndexLineWithSkipWorktree(line)); - } - - pathData.GitFilePaths.Add(this.TrimGitIndexLineWithSkipWorktree(line)); - }); - GitProcess.Result folderResult = gitProcess.LsTree( - GVFSConstants.DotGit.HeadName, - line => - { - pathData.GitFolderPaths.Add(this.TrimGitIndexLine(line)); - }, - recursive: true, - showDirectories: true); - - pathData.GitTrackingPaths.AddRange(skipWorktreeFiles); - } } }