From 6f8d08cee85101573666836e07b7534c4a1b8a38 Mon Sep 17 00:00:00 2001 From: Tyrie Vella Date: Thu, 13 Nov 2025 13:36:23 -0800 Subject: [PATCH 1/6] Add hydration percentages to git status hook For a few releases, `git status` would output a message like "You are in a partially-hydrated checkout with 1% of tracked files present." However, this implementation only measured files that had been modified; files that were hydrated but unmodified were not included, so the change was reverted. This change adds a line to the start of `git status` similar to "1% of files and 1% of folders hydrated. Run 'gvfs health' for details." I decided to show separate %s for files and folders because I think there is a meaningful difference between "0% files and 100% folders" and "10% files and 10% folders", but if combined both could display as "11% total hydration" (with ~1:10 ratio of folders to files). ** Changes ** 1. A new option is added to `gvfs health` verb, `--status`, to display the new output line. 2. Gvfs.Hooks.exe will call `gvfs health --status` on pre-command hook for `git status` except when called with --serialize. 3. The hydration %s are calculated from: - the number of files/folders in placeholder database - the number of files/folders in modified database - the total number of files, from git index header - the total number of folders, from `git ls-tree -r -d HEAD` 4. Because the git ls-tree command can be slow for large repos, its result is cached. - GitStatusCache is modified log the hydration stats to telemetry, which ensures the cache is up-to-date. - The new cache file is stored in a new file TreeCountCache.dat next to the existing GitStatusCache.dat. - The cache is keyed off the tree ID of HEAD. 5. Refactoring: Reading the placeholder, modified DB file/folder lists, and index file/folder lists are moved from HealthVerb to EnlistmentPathData 6. If the repo is not mounted, `gvfs health [--status]` will attempt to read the modified file list directly instead of getting it from the current mount process. --- GVFS/GVFS.Common/GVFSConstants.cs | 1 + GVFS/GVFS.Common/Git/GitProcess.cs | 5 + GVFS/GVFS.Common/GitStatusCache.cs | 54 ++++++ .../EnlistmentHealthCalculator.cs | 3 +- .../EnlistmentHydrationSummary.cs | 170 ++++++++++++++++++ .../HealthCalculator/EnlistmentPathData.cs | 168 ++++++++++++++++- GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs | 11 +- GVFS/GVFS.Hooks/Program.cs | 8 + .../Common/EnlistmentHydrationSummaryTests.cs | 125 +++++++++++++ .../Common/GitStatusCacheTests.cs | 2 + .../GVFS.UnitTests/Mock/Git/MockGitProcess.cs | 15 +- GVFS/GVFS/CommandLine/HealthVerb.cs | 156 +++------------- 12 files changed, 578 insertions(+), 140 deletions(-) create mode 100644 GVFS/GVFS.Common/HealthCalculator/EnlistmentHydrationSummary.cs create mode 100644 GVFS/GVFS.UnitTests/Common/EnlistmentHydrationSummaryTests.cs diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index a6e619ff7c..766f188aea 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -127,6 +127,7 @@ public static class GitStatusCache { public const string Name = "gitStatusCache"; public static readonly string CachePath = Path.Combine(Name, "GitStatusCache.dat"); + public static readonly string TreeCount = Path.Combine(Name, "TreeCountCache.dat"); } } diff --git a/GVFS/GVFS.Common/Git/GitProcess.cs b/GVFS/GVFS.Common/Git/GitProcess.cs index 6b31c4f503..616d0ec22c 100644 --- a/GVFS/GVFS.Common/Git/GitProcess.cs +++ b/GVFS/GVFS.Common/Git/GitProcess.cs @@ -682,6 +682,11 @@ public Result MultiPackIndexRepack(string gitObjectDirectory, string batchSize) return this.InvokeGitAgainstDotGitFolder($"-c pack.threads=1 -c repack.packKeptObjects=true multi-pack-index repack --object-dir=\"{gitObjectDirectory}\" --batch-size={batchSize} --no-progress"); } + public Result GetHeadTreeId() + { + return this.InvokeGitAgainstDotGitFolder("show -s --format=%T HEAD", usePreCommandHook: false); + } + public Process GetGitProcess(string command, string workingDirectory, string dotGitDirectory, bool useReadObjectHook, bool redirectStandardError, string gitObjectsDirectory, bool usePreCommandHook) { ProcessStartInfo processInfo = new ProcessStartInfo(this.gitBinPath); diff --git a/GVFS/GVFS.Common/GitStatusCache.cs b/GVFS/GVFS.Common/GitStatusCache.cs index 0b633b6dfa..717e7aebca 100644 --- a/GVFS/GVFS.Common/GitStatusCache.cs +++ b/GVFS/GVFS.Common/GitStatusCache.cs @@ -52,6 +52,8 @@ public class GitStatusCache : IDisposable private object cacheFileLock = new object(); + internal static bool TEST_EnableHydrationSummary = true; + public GitStatusCache(GVFSContext context, GitStatusCacheConfig config) : this(context, config.BackoffTime) { @@ -315,6 +317,7 @@ private void RebuildStatusCacheIfNeeded(bool ignoreBackoff) if (needToRebuild) { this.statistics.RecordBackgroundStatusScanRun(); + this.UpdateHydrationSummary(); bool rebuildStatusCacheSucceeded = this.TryRebuildStatusCache(); @@ -336,6 +339,57 @@ private void RebuildStatusCacheIfNeeded(bool ignoreBackoff) } } + private void UpdateHydrationSummary() + { + if (!TEST_EnableHydrationSummary) + { + return; + } + + try + { + /* While not strictly part of git status, enlistment hydration summary is used + * in "git status" pre-command hook, and can take several seconds to compute on very large repos. + * Accessing it here ensures that the value is cached for when a user invokes "git status", + * and this is also a convenient place to log telemetry for it. + */ + EnlistmentHydrationSummary hydrationSummary = + EnlistmentHydrationSummary.CreateSummary(this.context.Enlistment, this.context.FileSystem); + EventMetadata metadata = new EventMetadata(); + metadata.Add("Area", EtwArea); + if (hydrationSummary.IsValid) + { + metadata[nameof(hydrationSummary.TotalFolderCount)] = hydrationSummary.TotalFolderCount; + metadata[nameof(hydrationSummary.TotalFileCount)] = hydrationSummary.TotalFileCount; + metadata[nameof(hydrationSummary.HydratedFolderCount)] = hydrationSummary.HydratedFolderCount; + metadata[nameof(hydrationSummary.HydratedFileCount)] = hydrationSummary.HydratedFileCount; + + this.context.Tracer.RelatedEvent( + EventLevel.Informational, + nameof(EnlistmentHydrationSummary), + metadata, + Keywords.Telemetry); + } + else + { + this.context.Tracer.RelatedWarning( + metadata, + $"{nameof(GitStatusCache)}{nameof(RebuildStatusCacheIfNeeded)}: hydration summary could not be calculdated.", + Keywords.Telemetry); + } + } + catch (Exception ex) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Area", EtwArea); + metadata.Add("Exception", ex.ToString()); + this.context.Tracer.RelatedError( + metadata, + $"{nameof(GitStatusCache)}{nameof(RebuildStatusCacheIfNeeded)}: Exception trying to update hydration summary cache.", + Keywords.Telemetry); + } + } + /// /// Rebuild the status cache. This will run the background status to /// generate status results, and update the serialized status cache diff --git a/GVFS/GVFS.Common/HealthCalculator/EnlistmentHealthCalculator.cs b/GVFS/GVFS.Common/HealthCalculator/EnlistmentHealthCalculator.cs index 2d048e9f22..b437694eef 100644 --- a/GVFS/GVFS.Common/HealthCalculator/EnlistmentHealthCalculator.cs +++ b/GVFS/GVFS.Common/HealthCalculator/EnlistmentHealthCalculator.cs @@ -1,5 +1,4 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.Linq; namespace GVFS.Common diff --git a/GVFS/GVFS.Common/HealthCalculator/EnlistmentHydrationSummary.cs b/GVFS/GVFS.Common/HealthCalculator/EnlistmentHydrationSummary.cs new file mode 100644 index 0000000000..74e65021da --- /dev/null +++ b/GVFS/GVFS.Common/HealthCalculator/EnlistmentHydrationSummary.cs @@ -0,0 +1,170 @@ +using GVFS.Common.FileSystem; +using GVFS.Common.Git; +using System; +using System.IO; +using System.Linq; + +namespace GVFS.Common +{ + public class EnlistmentHydrationSummary + { + public int HydratedFileCount { get; private set; } + public int TotalFileCount { get; private set; } + public int HydratedFolderCount { get; private set; } + public int TotalFolderCount { get; private set; } + + public bool IsValid + { + get + { + return HydratedFileCount >= 0 + && HydratedFolderCount >= 0 + && TotalFileCount >= HydratedFileCount + && TotalFolderCount >= HydratedFolderCount; + } + } + + public string ToMessage() + { + if (!IsValid) + { + return "Error calculating hydration. Run 'gvfs health' for details."; + } + + int fileHydrationPercent = TotalFileCount == 0 ? 0 : (100 * HydratedFileCount) / TotalFileCount; + int folderHydrationPercent = TotalFolderCount == 0 ? 0 : ((100 * HydratedFolderCount) / TotalFolderCount); + return $"{fileHydrationPercent}% of files and {folderHydrationPercent}% of folders hydrated. Run 'gvfs health' for details."; + } + + public static EnlistmentHydrationSummary CreateSummary( + GVFSEnlistment enlistment, + PhysicalFileSystem fileSystem) + { + try + { + /* Getting all the file paths from git index is slow and we only need the total count, + * so we read the index file header instead of calling GetPathsFromGitIndex */ + int totalFileCount = GetIndexFileCount(enlistment, fileSystem); + + /* Getting all the directories is also slow, but not as slow as reading the entire index, + * GetTotalPathCount caches the count so this is only slow occasionally, + * and the GitStatusCache manager also calls this to ensure it is updated frequently. */ + int totalFolderCount = GetHeadTreeCount(enlistment, fileSystem); + + EnlistmentPathData pathData = new EnlistmentPathData(); + + /* FUTURE: These could be optimized to only deal with counts instead of full path lists */ + pathData.LoadPlaceholdersFromDatabase(enlistment); + pathData.LoadModifiedPaths(enlistment); + + int hydratedFileCount = pathData.ModifiedFilePaths.Count + pathData.PlaceholderFilePaths.Count; + int hydratedFolderCount = pathData.ModifiedFolderPaths.Count + pathData.PlaceholderFolderPaths.Count; + return new EnlistmentHydrationSummary() + { + HydratedFileCount = hydratedFileCount, + HydratedFolderCount = hydratedFolderCount, + TotalFileCount = totalFileCount, + TotalFolderCount = totalFolderCount, + }; + } + catch + { + return new EnlistmentHydrationSummary() + { + HydratedFileCount = -1, + HydratedFolderCount = -1, + TotalFileCount = -1, + TotalFolderCount = -1, + }; + } + } + + /// + /// Get the total number of files in the index. + /// + internal static int GetIndexFileCount(GVFSEnlistment enlistment, PhysicalFileSystem fileSystem) + { + string indexPath = Path.Combine(enlistment.WorkingDirectoryBackingRoot, GVFSConstants.DotGit.Index); + using (var indexFile = fileSystem.OpenFileStream(indexPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, callFlushFileBuffers: false)) + { + if (indexFile.Length < 12) + { + return -1; + } + /* The number of files in the index is a big-endian integer from bytes 9-12 of the index file. */ + indexFile.Position = 8; + var bytes = new byte[4]; + indexFile.Read(bytes, 0, 4); + if (BitConverter.IsLittleEndian) + { + Array.Reverse(bytes); + } + int count = BitConverter.ToInt32(bytes, 0); + return count; + } + } + + /// + /// Get the total number of trees in the repo at HEAD. + /// + /// + /// This is used as the denominator in displaying percentage of hydrated + /// directories as part of git status pre-command hook. + /// It can take several seconds to calculate, so we cache it near the git status cache. + /// + /// + /// The number of subtrees at HEAD, which may be 0. + /// Will return 0 if unsuccessful. + /// + internal static int GetHeadTreeCount(GVFSEnlistment enlistment, PhysicalFileSystem fileSystem) + { + var gitProcess = enlistment.CreateGitProcess(); + var headResult = gitProcess.GetHeadTreeId(); + if (headResult.ExitCodeIsFailure) + { + return 0; + } + var headSha = headResult.Output.Trim(); + var cacheFile = Path.Combine( + enlistment.DotGVFSRoot, + GVFSConstants.DotGVFS.GitStatusCache.TreeCount); + + // Load from cache if cache matches current HEAD. + if (fileSystem.FileExists(cacheFile)) + { + try + { + var lines = fileSystem.ReadLines(cacheFile).ToArray(); + if (lines.Length == 2 + && lines[0] == headSha + && int.TryParse(lines[1], out int cachedCount)) + { + return cachedCount; + } + } + catch + { + // Ignore errors reading the cache + } + } + + int totalPathCount = 0; + GitProcess.Result folderResult = gitProcess.LsTree( + GVFSConstants.DotGit.HeadName, + line => totalPathCount++, + recursive: true, + showDirectories: true); + try + { + fileSystem.CreateDirectory(Path.GetDirectoryName(cacheFile)); + fileSystem.WriteAllText(cacheFile, $"{headSha}\n{totalPathCount}"); + } + catch + { + // Ignore errors writing the cache + } + + return totalPathCount; + } + } +} diff --git a/GVFS/GVFS.Common/HealthCalculator/EnlistmentPathData.cs b/GVFS/GVFS.Common/HealthCalculator/EnlistmentPathData.cs index a7ecaafae4..911b3f8059 100644 --- a/GVFS/GVFS.Common/HealthCalculator/EnlistmentPathData.cs +++ b/GVFS/GVFS.Common/HealthCalculator/EnlistmentPathData.cs @@ -1,5 +1,10 @@ -using System.Collections; +using GVFS.Common.Database; +using GVFS.Common.FileSystem; +using GVFS.Common.Git; +using GVFS.Common.NamedPipes; +using System; using System.Collections.Generic; +using System.IO; using System.Linq; namespace GVFS.Common @@ -38,6 +43,157 @@ public void NormalizeAllPaths() this.ModifiedFilePaths = this.ModifiedFilePaths.Union(this.GitTrackingPaths).ToList(); } + /// + /// Get two lists of placeholders, one containing the files and the other the directories + /// Goes to the SQLite database for the placeholder lists + /// + /// The current GVFS enlistment being operated on + public void LoadPlaceholdersFromDatabase(GVFSEnlistment enlistment) + { + List filePlaceholders = new List(); + List folderPlaceholders = new List(); + + using (GVFSDatabase database = new GVFSDatabase(new PhysicalFileSystem(), enlistment.EnlistmentRoot, new SqliteDatabase())) + { + PlaceholderTable placeholderTable = new PlaceholderTable(database); + placeholderTable.GetAllEntries(out filePlaceholders, out folderPlaceholders); + } + + this.PlaceholderFilePaths.AddRange(filePlaceholders.Select(placeholderData => placeholderData.Path)); + this.PlaceholderFolderPaths.AddRange(folderPlaceholders.Select(placeholderData => placeholderData.Path)); + } + + /// + /// Call 'git ls-files' and 'git ls-tree' to get a list of all files and directories in the enlistment + /// + /// The current GVFS enlistmetn being operated on + public void LoadPathsFromGitIndex(GVFSEnlistment enlistment) + { + List skipWorktreeFiles = new List(); + GitProcess gitProcess = new GitProcess(enlistment); + + GitProcess.Result fileResult = gitProcess.LsFiles( + line => + { + if (line.First() == 'H') + { + skipWorktreeFiles.Add(TrimGitIndexLineWithSkipWorktree(line)); + } + + this.GitFilePaths.Add(TrimGitIndexLineWithSkipWorktree(line)); + }); + GitProcess.Result folderResult = gitProcess.LsTree( + GVFSConstants.DotGit.HeadName, + line => + { + this.GitFolderPaths.Add(TrimGitIndexLine(line)); + }, + recursive: true, + showDirectories: true); + + this.GitTrackingPaths.AddRange(skipWorktreeFiles); + } + + public void LoadModifiedPaths(GVFSEnlistment enlistment) + { + if (TryLoadModifiedPathsFromPipe(enlistment)) + { + return; + } + try + { + /* Most likely GVFS is not mounted. Give a basic effort to read the modified paths database */ + var filePath = Path.Combine(enlistment.DotGVFSRoot, GVFSConstants.DotGVFS.Databases.ModifiedPaths); + using (var file = File.Open(filePath, FileMode.OpenOrCreate, FileAccess.Read, FileShare.Read)) + using (var reader = new StreamReader(file)) + { + AddModifiedPaths(ReadModifiedPathDatabaseLines(reader)); + } + } + catch { } + } + + private IEnumerable ReadModifiedPathDatabaseLines(StreamReader r) + { + while (!r.EndOfStream) + { + var line = r.ReadLine(); + const string LinePrefix = "A "; + if (line.StartsWith(LinePrefix)) + { + line = line.Substring(LinePrefix.Length); + } + yield return line; + } + } + + /// + /// Talk to the mount process across the named pipe to get a list of the modified paths + /// + /// If/when modified paths are moved to SQLite go there instead + /// The enlistment being operated on + /// An array containing all of the modified paths in string format + private bool TryLoadModifiedPathsFromPipe(GVFSEnlistment enlistment) + { + using (NamedPipeClient pipeClient = new NamedPipeClient(enlistment.NamedPipeName)) + { + string[] modifiedPathsList = Array.Empty(); + + if (!pipeClient.Connect()) + { + return false; + } + + try + { + NamedPipeMessages.Message modifiedPathsMessage = new NamedPipeMessages.Message(NamedPipeMessages.ModifiedPaths.ListRequest, NamedPipeMessages.ModifiedPaths.CurrentVersion); + pipeClient.SendRequest(modifiedPathsMessage); + + NamedPipeMessages.Message modifiedPathsResponse = pipeClient.ReadResponse(); + if (!modifiedPathsResponse.Header.Equals(NamedPipeMessages.ModifiedPaths.SuccessResult)) + { + return false; + } + + modifiedPathsList = modifiedPathsResponse.Body.Split(new char[] { '\0' }, StringSplitOptions.RemoveEmptyEntries); + } + catch (BrokenPipeException e) + { + return false; + } + + AddModifiedPaths(modifiedPathsList); + return true; + } + } + + private void AddModifiedPaths(IEnumerable modifiedPathsList) + { + foreach (string path in modifiedPathsList) + { + if (path.Last() == GVFSConstants.GitPathSeparator) + { + path.TrimEnd(GVFSConstants.GitPathSeparator); + this.ModifiedFolderPaths.Add(path); + } + else + { + this.ModifiedFilePaths.Add(path); + } + } + } + + /// + /// Parse a line of the git index coming from the ls-files endpoint in the git process to get the path to that files + /// These paths begin with 'S' or 'H' depending on if they have the skip-worktree bit set + /// + /// The line from the output of the git index + /// The path extracted from the provided line of the git index + private static string TrimGitIndexLineWithSkipWorktree(string line) + { + return line.Substring(line.IndexOf(' ') + 1); + } + private void NormalizePaths(List paths) { for (int i = 0; i < paths.Count; i++) @@ -46,5 +202,15 @@ private void NormalizePaths(List paths) paths[i] = paths[i].Trim(GVFSConstants.GitPathSeparator); } } + + /// + /// Parse a line of the git index coming from the ls-tree endpoint in the git process to get the path to that file + /// + /// The line from the output of the git index + /// The path extracted from the provided line of the git index + private static string TrimGitIndexLine(string line) + { + return line.Substring(line.IndexOf('\t') + 1); + } } } diff --git a/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs b/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs index 76a77eb64e..8bdc74f17a 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs @@ -65,11 +65,12 @@ public static ProcessResult InvokeGitAgainstGVFSRepo( bool removeWaitingMessages = true, bool removeUpgradeMessages = true, bool removePartialHydrationMessages = true, - bool removeFSMonitorMessages = true) + bool removeFSMonitorMessages = true, + bool removeGvfsHealthMessages = true) { ProcessResult result = GitProcess.InvokeProcess(gvfsRepoRoot, command, environmentVariables); - string output = FilterMessages(result.Output, false, false, false, removePartialHydrationMessages, removeFSMonitorMessages); - string errors = FilterMessages(result.Errors, true, removeWaitingMessages, removeUpgradeMessages, removePartialHydrationMessages, removeFSMonitorMessages); + string output = FilterMessages(result.Output, false, false, false, removePartialHydrationMessages, removeFSMonitorMessages, removeGvfsHealthMessages); + string errors = FilterMessages(result.Errors, true, removeWaitingMessages, removeUpgradeMessages, removePartialHydrationMessages, removeFSMonitorMessages, removeGvfsHealthMessages); return new ProcessResult( output, @@ -100,7 +101,8 @@ private static string FilterMessages( bool removeWaitingMessages, bool removeUpgradeMessages, bool removePartialHydrationMessages, - bool removeFSMonitorMessages) + bool removeFSMonitorMessages, + bool removeGvfsHealthMessages) { if (!string.IsNullOrEmpty(input) && (removeWaitingMessages || removeUpgradeMessages || removePartialHydrationMessages || removeFSMonitorMessages)) { @@ -111,6 +113,7 @@ private static string FilterMessages( (removeUpgradeMessages && line.StartsWith("A new version of VFS for Git is available.")) || (removeWaitingMessages && line.StartsWith("Waiting for ")) || (removePartialHydrationMessages && line.StartsWith("You are in a partially-hydrated checkout with ")) || + (removeGvfsHealthMessages && line.TrimEnd().EndsWith("Run 'gvfs health' for details.")) || (removeFSMonitorMessages && line.TrimEnd().EndsWith(" is incompatible with fsmonitor"))) { return false; diff --git a/GVFS/GVFS.Hooks/Program.cs b/GVFS/GVFS.Hooks/Program.cs index 4dc9aeb99e..0bfe0da543 100644 --- a/GVFS/GVFS.Hooks/Program.cs +++ b/GVFS/GVFS.Hooks/Program.cs @@ -87,6 +87,14 @@ private static void RunPreCommands(string[] args) case "pull": ProcessHelper.Run("gvfs", "prefetch --commits", redirectOutput: false); break; + case "status": + /* If status is being run to serialize for caching, skip the health display */ + if (!args.Any(arg => arg.StartsWith("--serialize", StringComparison.OrdinalIgnoreCase))) + { + /* Display a message about the hydration status of the repo */ + ProcessHelper.Run("gvfs", "health --status", redirectOutput: false); + } + break; } } diff --git a/GVFS/GVFS.UnitTests/Common/EnlistmentHydrationSummaryTests.cs b/GVFS/GVFS.UnitTests/Common/EnlistmentHydrationSummaryTests.cs new file mode 100644 index 0000000000..5434b0304b --- /dev/null +++ b/GVFS/GVFS.UnitTests/Common/EnlistmentHydrationSummaryTests.cs @@ -0,0 +1,125 @@ +using GVFS.Common; +using GVFS.Common.Git; +using GVFS.Tests.Should; +using GVFS.UnitTests.Mock.Common; +using GVFS.UnitTests.Mock.FileSystem; +using GVFS.UnitTests.Mock.Git; +using NUnit.Framework; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace GVFS.UnitTests.Common +{ + [TestFixture] + public class EnlistmentHydrationSummaryTests + { + private MockFileSystem fileSystem; + private MockGitProcess gitProcess; + private GVFSContext context; + private string gitParentPath; + private string gvfsMetadataPath; + private MockDirectory enlistmentDirectory; + + private const string HeadTreeId = "0123456789012345678901234567890123456789"; + private const int HeadPathCount = 42; + + public static IEnumerable<(string CachePrecontents, string ExpectedCachePostContents)> HeadTreeCountCacheContents + { + get + { + yield return (null, $"{HeadTreeId}\n{HeadPathCount}"); + yield return ($"{HeadTreeId}\n{HeadPathCount}", $"{HeadTreeId}\n{HeadPathCount}"); + yield return ($"{HeadTreeId}\n{HeadPathCount - 1}", $"{HeadTreeId}\n{HeadPathCount - 1}"); + yield return ($"{HeadTreeId.Replace("1", "a")}\n{HeadPathCount - 1}", $"{HeadTreeId}\n{HeadPathCount}"); + yield return ($"{HeadTreeId}\nabc", $"{HeadTreeId}\n{HeadPathCount}"); + yield return ($"{HeadTreeId}\nabc", $"{HeadTreeId}\n{HeadPathCount}"); + yield return ($"\n", $"{HeadTreeId}\n{HeadPathCount}"); + yield return ($"\nabc", $"{HeadTreeId}\n{HeadPathCount}"); + } + } + + [SetUp] + public void Setup() + { + MockTracer tracer = new MockTracer(); + + string enlistmentRoot = Path.Combine("mock:", "GVFS", "UnitTests", "Repo"); + string statusCachePath = Path.Combine("mock:", "GVFS", "UnitTests", "Repo", GVFSPlatform.Instance.Constants.DotGVFSRoot, "gitStatusCache"); + + this.gitProcess = new MockGitProcess(); + this.gitProcess.SetExpectedCommandResult($"--no-optional-locks status \"--serialize={statusCachePath}", () => new GitProcess.Result(string.Empty, string.Empty, 0), true); + MockGVFSEnlistment enlistment = new MockGVFSEnlistment(enlistmentRoot, "fake://repoUrl", "fake://gitBinPath", this.gitProcess); + enlistment.InitializeCachePathsFromKey("fake:\\gvfsSharedCache", "fakeCacheKey"); + + this.gitParentPath = enlistment.WorkingDirectoryBackingRoot; + this.gvfsMetadataPath = enlistment.DotGVFSRoot; + + this.enlistmentDirectory = new MockDirectory( + enlistmentRoot, + new MockDirectory[] + { + new MockDirectory(this.gitParentPath, folders: null, files: null), + }, + null); + + this.enlistmentDirectory.CreateFile(Path.Combine(this.gitParentPath, ".git", "config"), ".git config Contents", createDirectories: true); + this.enlistmentDirectory.CreateFile(Path.Combine(this.gitParentPath, ".git", "HEAD"), ".git HEAD Contents", createDirectories: true); + this.enlistmentDirectory.CreateFile(Path.Combine(this.gitParentPath, ".git", "logs", "HEAD"), "HEAD Contents", createDirectories: true); + this.enlistmentDirectory.CreateFile(Path.Combine(this.gitParentPath, ".git", "info", "always_exclude"), "always_exclude Contents", createDirectories: true); + this.enlistmentDirectory.CreateDirectory(Path.Combine(this.gitParentPath, ".git", "objects", "pack")); + + this.fileSystem = new MockFileSystem(this.enlistmentDirectory); + this.fileSystem.AllowMoveFile = true; + this.fileSystem.DeleteNonExistentFileThrowsException = false; + + this.context = new GVFSContext( + tracer, + this.fileSystem, + new MockGitRepo(tracer, enlistment, this.fileSystem), + enlistment); + } + + [TearDown] + public void TearDown() + { + this.fileSystem = null; + this.gitProcess = null; + this.context = null; + this.gitParentPath = null; + this.gvfsMetadataPath = null; + this.enlistmentDirectory = null; + } + + [TestCaseSource("HeadTreeCountCacheContents")] + public void HeadTreeCountCacheTests((string CachePrecontents, string ExpectedCachePostContents) args) + { + string totalPathCountPath = Path.Combine(this.gvfsMetadataPath, GVFSConstants.DotGVFS.GitStatusCache.TreeCount); + if (args.CachePrecontents != null) + { + this.enlistmentDirectory.CreateFile(totalPathCountPath, args.CachePrecontents, createDirectories: true); + } + + this.gitProcess.SetExpectedCommandResult("git show -s --format=%T HEAD", + () => new GitProcess.Result(HeadTreeId, "", 0)); + this.gitProcess.SetExpectedCommandResult("ls-tree -r -d HEAD", + () => new GitProcess.Result( + string.Join("\n", Enumerable.Range(0, HeadPathCount) + .Select(x => x.ToString())), + "", 0)); + + Assert.AreEqual( + args.CachePrecontents != null, + this.fileSystem.FileExists(totalPathCountPath)); + + int result = EnlistmentHydrationSummary.GetHeadTreeCount(this.context.Enlistment, this.context.FileSystem); + + this.fileSystem.FileExists(totalPathCountPath).ShouldBeTrue(); + var postContents = this.fileSystem.ReadAllText(totalPathCountPath); + Assert.AreEqual( + args.ExpectedCachePostContents, + postContents); + Assert.AreEqual(postContents.Split('\n')[1], result.ToString()); + } + } +} diff --git a/GVFS/GVFS.UnitTests/Common/GitStatusCacheTests.cs b/GVFS/GVFS.UnitTests/Common/GitStatusCacheTests.cs index 75e754afea..20ea38171a 100644 --- a/GVFS/GVFS.UnitTests/Common/GitStatusCacheTests.cs +++ b/GVFS/GVFS.UnitTests/Common/GitStatusCacheTests.cs @@ -73,6 +73,7 @@ public void SetUp() this.fileSystem, new MockGitRepo(tracer, enlistment, this.fileSystem), enlistment); + GitStatusCache.TEST_EnableHydrationSummary = false; } [TearDown] @@ -84,6 +85,7 @@ public void TearDown() this.gitParentPath = null; this.gvfsMetadataPath = null; this.enlistmentDirectory = null; + GitStatusCache.TEST_EnableHydrationSummary = true; } [TestCase] diff --git a/GVFS/GVFS.UnitTests/Mock/Git/MockGitProcess.cs b/GVFS/GVFS.UnitTests/Mock/Git/MockGitProcess.cs index 9d8b5e93c5..c9095cc2c8 100644 --- a/GVFS/GVFS.UnitTests/Mock/Git/MockGitProcess.cs +++ b/GVFS/GVFS.UnitTests/Mock/Git/MockGitProcess.cs @@ -109,7 +109,20 @@ protected override Result InvokeGitImpl( CommandInfo matchedCommand = this.expectedCommandInfos.Last(commandMatchFunction); matchedCommand.ShouldNotBeNull("Unexpected command: " + command); - return matchedCommand.Result(); + var result = matchedCommand.Result(); + if (parseStdOutLine != null && !string.IsNullOrEmpty(result.Output)) + { + using (StringReader reader = new StringReader(result.Output)) + { + string line; + while ((line = reader.ReadLine()) != null) + { + parseStdOutLine(line); + } + } + /* Future: result.Output should be set to null in this case */ + } + return result; } public class Credential diff --git a/GVFS/GVFS/CommandLine/HealthVerb.cs b/GVFS/GVFS/CommandLine/HealthVerb.cs index b12b58d2c0..782c014c24 100644 --- a/GVFS/GVFS/CommandLine/HealthVerb.cs +++ b/GVFS/GVFS/CommandLine/HealthVerb.cs @@ -1,9 +1,6 @@ using CommandLine; using GVFS.Common; -using GVFS.Common.Database; using GVFS.Common.FileSystem; -using GVFS.Common.Git; -using GVFS.Common.NamedPipes; using System; using System.Collections.Generic; using System.Linq; @@ -29,10 +26,25 @@ public class HealthVerb : GVFSVerb.ForExistingEnlistment HelpText = "Get the health of a specific directory (default is the current working directory")] public string Directory { get; set; } + [Option( + 's', + "status", + Required = false, + HelpText = "Display only the hydration % of the repository, similar to 'git status' in a repository with sparse-checkout")] + public bool StatusOnly { get; set; } + protected override string VerbName => HealthVerbName; + internal PhysicalFileSystem FileSystem { get; set; } = new PhysicalFileSystem(); + protected override void Execute(GVFSEnlistment enlistment) { + if (this.StatusOnly) + { + this.OutputHydrationPercent(enlistment); + return; + } + // Now default to the current working directory when running the verb without a specified path if (string.IsNullOrEmpty(this.Directory) || this.Directory.Equals(".")) { @@ -53,9 +65,9 @@ protected override void Execute(GVFSEnlistment enlistment) EnlistmentPathData pathData = new EnlistmentPathData(); - this.GetPlaceholdersFromDatabase(enlistment, pathData); - this.GetModifiedPathsFromPipe(enlistment, pathData); - this.GetPathsFromGitIndex(enlistment, pathData); + pathData.LoadPlaceholdersFromDatabase(enlistment); + pathData.LoadModifiedPaths(enlistment); + pathData.LoadPathsFromGitIndex(enlistment); pathData.NormalizeAllPaths(); @@ -65,6 +77,12 @@ protected override void Execute(GVFSEnlistment enlistment) this.PrintOutput(enlistmentHealthData); } + private void OutputHydrationPercent(GVFSEnlistment enlistment) + { + var summary = EnlistmentHydrationSummary.CreateSummary(enlistment, this.FileSystem); + this.Output.WriteLine(summary.ToMessage()); + } + private void PrintOutput(EnlistmentHealthData enlistmentHealthData) { string trackedFilesCountFormatted = enlistmentHealthData.GitTrackedItemsCount.ToString("N0"); @@ -114,131 +132,5 @@ private string FormatPercent(decimal percent) { return percent.ToString("P0").PadLeft(4); } - - /// - /// Parse a line of the git index coming from the ls-tree endpoint in the git process to get the path to that file - /// - /// The line from the output of the git index - /// The path extracted from the provided line of the git index - private string TrimGitIndexLine(string line) - { - return line.Substring(line.IndexOf('\t') + 1); - } - - /// - /// Parse a line of the git index coming from the ls-files endpoint in the git process to get the path to that files - /// These paths begin with 'S' or 'H' depending on if they have the skip-worktree bit set - /// - /// The line from the output of the git index - /// The path extracted from the provided line of the git index - private string TrimGitIndexLineWithSkipWorktree(string line) - { - return line.Substring(line.IndexOf(' ') + 1); - } - - /// - /// Talk to the mount process across the named pipe to get a list of the modified paths - /// - /// If/when modified paths are moved to SQLite go there instead - /// The enlistment being operated on - /// An array containing all of the modified paths in string format - private void GetModifiedPathsFromPipe(GVFSEnlistment enlistment, EnlistmentPathData pathData) - { - using (NamedPipeClient pipeClient = new NamedPipeClient(enlistment.NamedPipeName)) - { - string[] modifiedPathsList = Array.Empty(); - - if (!pipeClient.Connect()) - { - this.ReportErrorAndExit("Unable to connect to GVFS. Try running 'gvfs mount'"); - } - - try - { - NamedPipeMessages.Message modifiedPathsMessage = new NamedPipeMessages.Message(NamedPipeMessages.ModifiedPaths.ListRequest, NamedPipeMessages.ModifiedPaths.CurrentVersion); - pipeClient.SendRequest(modifiedPathsMessage); - - NamedPipeMessages.Message modifiedPathsResponse = pipeClient.ReadResponse(); - if (!modifiedPathsResponse.Header.Equals(NamedPipeMessages.ModifiedPaths.SuccessResult)) - { - this.Output.WriteLine("Bad response from modified path pipe: " + modifiedPathsResponse.Header); - return; - } - - modifiedPathsList = modifiedPathsResponse.Body.Split(new char[] { '\0' }, StringSplitOptions.RemoveEmptyEntries); - } - catch (BrokenPipeException e) - { - this.ReportErrorAndExit("Unable to communicate with GVFS: " + e.ToString()); - } - - foreach (string path in modifiedPathsList) - { - if (path.Last() == GVFSConstants.GitPathSeparator) - { - path.TrimEnd(GVFSConstants.GitPathSeparator); - pathData.ModifiedFolderPaths.Add(path); - } - else - { - pathData.ModifiedFilePaths.Add(path); - } - } - } - } - - /// - /// Get two lists of placeholders, one containing the files and the other the directories - /// Goes to the SQLite database for the placeholder lists - /// - /// The current GVFS enlistment being operated on - /// Out parameter where the list of file placeholders will end up - /// Out parameter where the list of folder placeholders will end up - private void GetPlaceholdersFromDatabase(GVFSEnlistment enlistment, EnlistmentPathData pathData) - { - List filePlaceholders = new List(); - List folderPlaceholders = new List(); - - using (GVFSDatabase database = new GVFSDatabase(new PhysicalFileSystem(), enlistment.EnlistmentRoot, new SqliteDatabase())) - { - PlaceholderTable placeholderTable = new PlaceholderTable(database); - placeholderTable.GetAllEntries(out filePlaceholders, out folderPlaceholders); - } - - pathData.PlaceholderFilePaths.AddRange(filePlaceholders.Select(placeholderData => placeholderData.Path)); - pathData.PlaceholderFolderPaths.AddRange(folderPlaceholders.Select(placeholderData => placeholderData.Path)); - } - - /// - /// Call 'git ls-files' and 'git ls-tree' to get a list of all files and directories in the enlistment - /// - /// The current GVFS enlistmetn being operated on - /// The path data object where paths are being saved - private void GetPathsFromGitIndex(GVFSEnlistment enlistment, EnlistmentPathData pathData) - { - List skipWorktreeFiles = new List(); - GitProcess gitProcess = new GitProcess(enlistment); - - GitProcess.Result fileResult = gitProcess.LsFiles( - line => - { - if (line.First() == 'H') - { - skipWorktreeFiles.Add(this.TrimGitIndexLineWithSkipWorktree(line)); - } - - pathData.GitFilePaths.Add(this.TrimGitIndexLineWithSkipWorktree(line)); - }); - GitProcess.Result folderResult = gitProcess.LsTree( - GVFSConstants.DotGit.HeadName, - line => - { - pathData.GitFolderPaths.Add(this.TrimGitIndexLine(line)); - }, - recursive: true, - showDirectories: true); - - pathData.GitTrackingPaths.AddRange(skipWorktreeFiles); - } } } From ab159f43e3cc208056c94db1fed8dc32d9c08de7 Mon Sep 17 00:00:00 2001 From: Tyrie Vella Date: Mon, 17 Nov 2025 16:22:37 -0800 Subject: [PATCH 2/6] Fix unit tests --- GVFS/GVFS.UnitTests/Common/EnlistmentHydrationSummaryTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.UnitTests/Common/EnlistmentHydrationSummaryTests.cs b/GVFS/GVFS.UnitTests/Common/EnlistmentHydrationSummaryTests.cs index 5434b0304b..53591de8ae 100644 --- a/GVFS/GVFS.UnitTests/Common/EnlistmentHydrationSummaryTests.cs +++ b/GVFS/GVFS.UnitTests/Common/EnlistmentHydrationSummaryTests.cs @@ -100,7 +100,7 @@ public void HeadTreeCountCacheTests((string CachePrecontents, string ExpectedCac this.enlistmentDirectory.CreateFile(totalPathCountPath, args.CachePrecontents, createDirectories: true); } - this.gitProcess.SetExpectedCommandResult("git show -s --format=%T HEAD", + this.gitProcess.SetExpectedCommandResult("show -s --format=%T HEAD", () => new GitProcess.Result(HeadTreeId, "", 0)); this.gitProcess.SetExpectedCommandResult("ls-tree -r -d HEAD", () => new GitProcess.Result( From 635c7ce81bbe4c782d6170aa2015d824531e537d Mon Sep 17 00:00:00 2001 From: Tyrie Vella Date: Tue, 18 Nov 2025 12:17:14 -0800 Subject: [PATCH 3/6] Exit early for full dehydrate in src directory --- GVFS/GVFS/CommandLine/DehydrateVerb.cs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/GVFS/GVFS/CommandLine/DehydrateVerb.cs b/GVFS/GVFS/CommandLine/DehydrateVerb.cs index f3d5224f5e..b739b9b2df 100644 --- a/GVFS/GVFS/CommandLine/DehydrateVerb.cs +++ b/GVFS/GVFS/CommandLine/DehydrateVerb.cs @@ -117,7 +117,7 @@ protected override void Execute(GVFSEnlistment enlistment) if (!this.Confirmed && fullDehydrate) { this.Output.WriteLine( -@"WARNING: THIS IS AN EXPERIMENTAL FEATURE +$@"WARNING: THIS IS AN EXPERIMENTAL FEATURE Dehydrate will back up your src folder, and then create a new, empty src folder with a fresh virtualization of the repo. All of your downloaded objects, branches, @@ -130,8 +130,7 @@ any of your uncommitted changes. in the backup folder, but it will be harder to find them because 'git status' will not work in the backup. -To actually execute the dehydrate, run 'gvfs dehydrate --confirm' from the parent -of your enlistment's src folder. +To actually execute the dehydrate, run 'gvfs dehydrate --confirm' from {enlistment.EnlistmentRoot}. "); return; @@ -155,6 +154,13 @@ from a parent of the folders list. return; } + if (fullDehydrate && Environment.CurrentDirectory.StartsWith(enlistment.WorkingDirectoryBackingRoot)) + { + /* If running from /src, the dehydrate would fail because of the handle we are holding on it. */ + this.Output.WriteLine($"Dehydrate must be run from {enlistment.EnlistmentRoot}"); + return; + } + if (this.NoStatus && !fullDehydrate) { this.ReportErrorAndExit(tracer, "Dehydrate --no-status not valid with --folders"); From 14f0d3064c1803049ae0ed03913e10aabfac6fd7 Mon Sep 17 00:00:00 2001 From: Tyrie Vella Date: Wed, 19 Nov 2025 15:51:10 -0800 Subject: [PATCH 4/6] Ensure --no-prefetch clone is honored by maintenance The first prefetch for a large repository can take some time and (if gvfs.trust- pack-indexes is false) heavy CPU and disk resources. If a user cloned with --no-prefetch, let's assume they knew what they were doing and not automatically prefetch after 15 minutes. A typical use-case for --no-prefetch is in a CI pipeline which only needs the head of a branch. --- GVFS/GVFS.Common/Maintenance/PrefetchStep.cs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.Common/Maintenance/PrefetchStep.cs b/GVFS/GVFS.Common/Maintenance/PrefetchStep.cs index 901ec4cf20..b67ad6ae77 100644 --- a/GVFS/GVFS.Common/Maintenance/PrefetchStep.cs +++ b/GVFS/GVFS.Common/Maintenance/PrefetchStep.cs @@ -16,6 +16,7 @@ public class PrefetchStep : GitMaintenanceStep private const int LockWaitTimeMs = 100; private const int WaitingOnLockLogThreshold = 50; private const string PrefetchCommitsAndTreesLock = "prefetch-commits-trees.lock"; + private const int NoExistingPrefetchPacks = -1; private readonly TimeSpan timeBetweenPrefetches = TimeSpan.FromMinutes(70); public PrefetchStep(GVFSContext context, GitObjects gitObjects, bool requireCacheLock = true) @@ -80,6 +81,18 @@ protected override void PerformMaintenance() return; } + if (last == NoExistingPrefetchPacks) + { + /* If there are no existing prefetch packs, that means that either the + * first prefetch is still in progress or the clone was run with "--no-prefetch". + * In either case, we should not run prefetch as a maintenance task. + * If users want to prefetch after cloning with "--no-prefetch", they can run + * "gvfs prefetch" manually. Also, "git pull" and "git fetch" will run prefetch + * as a pre-command hook. */ + this.Context.Tracer.RelatedInfo(this.Area + ": Skipping prefetch since there are no existing prefetch packs"); + return; + } + DateTime lastDateTime = EpochConverter.FromUnixEpochSeconds(last); DateTime now = DateTime.UtcNow; @@ -150,7 +163,7 @@ private bool TryGetMaxGoodPrefetchTimestamp(out long maxGoodTimestamp, out strin .OrderBy(packInfo => packInfo.Timestamp) .ToList(); - maxGoodTimestamp = -1; + maxGoodTimestamp = NoExistingPrefetchPacks; int firstBadPack = -1; for (int i = 0; i < orderedPacks.Count; ++i) From d44a3165a27529cd8bc9e6c6ea48d26cca21d289 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 14:54:19 +0000 Subject: [PATCH 5/6] build(deps): bump actions/checkout from 5 to 6 Bumps [actions/checkout](https://github.com/actions/checkout) from 5 to 6. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index bfac2917c4..1b466eddad 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -103,7 +103,7 @@ jobs: - name: Checkout source if: steps.check.outputs.result == '' - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Validate Microsoft Git version if: steps.check.outputs.result == '' @@ -138,7 +138,7 @@ jobs: - name: Checkout source if: steps.skip.outputs.result != 'true' - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: path: src From af794083b2f01df4e8692fded9acbb4d0a86a433 Mon Sep 17 00:00:00 2001 From: Tyrie Vella Date: Tue, 25 Nov 2025 10:51:20 -0800 Subject: [PATCH 6/6] Update comments --- .../HealthCalculator/EnlistmentHydrationSummary.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/GVFS/GVFS.Common/HealthCalculator/EnlistmentHydrationSummary.cs b/GVFS/GVFS.Common/HealthCalculator/EnlistmentHydrationSummary.cs index 74e65021da..02a32c2f5e 100644 --- a/GVFS/GVFS.Common/HealthCalculator/EnlistmentHydrationSummary.cs +++ b/GVFS/GVFS.Common/HealthCalculator/EnlistmentHydrationSummary.cs @@ -91,10 +91,14 @@ internal static int GetIndexFileCount(GVFSEnlistment enlistment, PhysicalFileSys { return -1; } - /* The number of files in the index is a big-endian integer from bytes 9-12 of the index file. */ + /* The number of files in the index is a big-endian integer from + * the 4 bytes at offsets 8-11 of the index file. */ indexFile.Position = 8; var bytes = new byte[4]; - indexFile.Read(bytes, 0, 4); + indexFile.Read( + bytes, // Destination buffer + offset: 0, // Offset in destination buffer, not in indexFile + count: 4); if (BitConverter.IsLittleEndian) { Array.Reverse(bytes);