Skip to content

gvfs dehydrate --folders without unmounting #1890

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
tyrielv wants to merge 1 commit into from
Closed

gvfs dehydrate --folders without unmounting #1890

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,8 +201,14 @@ public override FileSystemResult UpdatePlaceholderIfNeeded(

public override FileSystemResult DehydrateFolder(string relativePath)
{
// Don't need to do anything here because the parent will reproject the folder.
return new FileSystemResult(FSResult.Ok, 0);
// The folder should have already been deleted, but
// its tombstone also needs to be deleted to allow reprojection.
var result = this.virtualizationInstance.DeleteFile(
relativePath,
UpdateType.AllowTombstone,
out UpdateFailureCause failureCause);

return new FileSystemResult(HResultToFSResult(result), unchecked((int)result));
}

// TODO: Need ProjFS 13150199 to be fixed so that GVFS doesn't leak memory if the enumeration cancelled.
Expand Down
40 changes: 35 additions & 5 deletions GVFS/GVFS.Virtualization/FileSystemCallbacks.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ public class FileSystemCallbacks : IDisposable, IHeartBeatMetadataProvider
GitCommandLineParser.Verbs.UpdateIndex;

private readonly string logsHeadPath;
private readonly ConcurrentHashSet<string> dehydratingFolders = new ConcurrentHashSet<string>();

private GVFSContext context;
private IPlaceholderCollection placeholderDatabase;
Expand Down Expand Up @@ -305,6 +306,10 @@ public bool TryDehydrateFolder(string relativePath, out string errorMessage)

try
{
this.dehydratingFolders.Add(relativePath);

var absolutePath = Path.Combine(this.context.Enlistment.WorkingDirectoryBackingRoot, relativePath);
this.context.FileSystem.DeleteDirectory(absolutePath, recursive: true);
relativePath = GVFSDatabase.NormalizePath(relativePath);
removedPlaceholders = this.placeholderDatabase.RemoveAllEntriesForFolder(relativePath);
removedModifiedPaths = this.modifiedPaths.RemoveAllEntriesForFolder(relativePath);
Expand All @@ -321,6 +326,10 @@ public bool TryDehydrateFolder(string relativePath, out string errorMessage)
EventMetadata metadata = this.CreateEventMetadata(relativePath, ex);
this.context.Tracer.RelatedError(metadata, errorMessage);
}
finally
{
this.dehydratingFolders.TryRemove(relativePath);
}

if (!string.IsNullOrEmpty(errorMessage))
{
Expand Down Expand Up @@ -466,12 +475,18 @@ public virtual void OnFileSymLinkCreated(string newLinkRelativePath)

public void OnFileDeleted(string relativePath)
{
this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFileDeleted(relativePath));
if (!this.IsDehydrating(relativePath))
{
this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFileDeleted(relativePath));
}
}

public void OnFilePreDelete(string relativePath)
{
this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFilePreDelete(relativePath));
if (!this.IsDehydrating(relativePath))
{
this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFilePreDelete(relativePath));
}
}

/// <summary>
Expand Down Expand Up @@ -506,17 +521,26 @@ public virtual void OnFolderRenamed(string oldRelativePath, string newRelativePa

public void OnFolderDeleted(string relativePath)
{
this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFolderDeleted(relativePath));
if (!this.IsDehydrating(relativePath))
{
this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFolderDeleted(relativePath));
}
}

public void OnPossibleTombstoneFolderCreated(string relativePath)
{
this.GitIndexProjection.OnPossibleTombstoneFolderCreated(relativePath);
if (!this.IsDehydrating(relativePath))
{
this.GitIndexProjection.OnPossibleTombstoneFolderCreated(relativePath);
}
}

public void OnFolderPreDelete(string relativePath)
{
this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFolderPreDelete(relativePath));
if (!this.IsDehydrating(relativePath))
{
this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFolderPreDelete(relativePath));
}
}

public void OnPlaceholderFileCreated(string relativePath, string sha, string triggeringProcessImageFileName)
Expand Down Expand Up @@ -1025,6 +1049,12 @@ private EventMetadata CreateEventMetadata(
return metadata;
}

private bool IsDehydrating(string relativePath)
{
return this.dehydratingFolders.Any(f =>
relativePath.StartsWith(f + GVFSConstants.GitPathSeparatorString, GVFSPlatform.Instance.Constants.PathComparison));
}

private class PlaceHolderCreateCounter
{
private long count;
Expand Down
167 changes: 129 additions & 38 deletions GVFS/GVFS/CommandLine/DehydrateVerb.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,16 +181,16 @@

this.Output.WriteLine();

this.Unmount(tracer);

string error;
if (!DiskLayoutUpgrade.TryCheckDiskLayoutVersion(tracer, enlistment.EnlistmentRoot, out error))
{
this.ReportErrorAndExit(tracer, error);
}

if (fullDehydrate)
{
this.Unmount(tracer);

string error;
if (!DiskLayoutUpgrade.TryCheckDiskLayoutVersion(tracer, enlistment.EnlistmentRoot, out error))
{
this.ReportErrorAndExit(tracer, error);
}

RetryConfig retryConfig;
if (!RetryConfig.TryLoadFromGitConfig(tracer, enlistment, out retryConfig, out error))
{
Expand All @@ -216,7 +216,7 @@
{
if (cleanStatus)
{
this.DehydrateFolders(tracer, enlistment, folders);
this.DehydrateFolders(tracer, enlistment, folders, backupRoot);
}
else
{
Expand All @@ -231,8 +231,15 @@
}
}

private void DehydrateFolders(JsonTracer tracer, GVFSEnlistment enlistment, string[] folders)
private void DehydrateFolders(JsonTracer tracer, GVFSEnlistment enlistment, string[] folders, string backupRoot)
{
if (!this.TryBackupNonSrcFiles(tracer, enlistment, backupRoot))
{
this.Output.WriteLine();
this.WriteMessage(tracer, "ERROR: Backup failed. ");
return;
}

List<string> foldersToDehydrate = new List<string>();
List<string> folderErrors = new List<string>();

Expand All @@ -241,7 +248,7 @@
{
if (!ModifiedPathsDatabase.TryLoadOrCreate(
tracer,
Path.Combine(enlistment.DotGVFSRoot, GVFSConstants.DotGVFS.Databases.ModifiedPaths),
Path.Combine(GetBackupDatabasesPath(backupRoot), GVFSConstants.DotGVFS.Databases.ModifiedPaths),
this.fileSystem,
out ModifiedPathsDatabase modifiedPaths,
out string error))
Expand All @@ -252,7 +259,7 @@

using (modifiedPaths)
{
string ioError;

Check warning on line 262 in GVFS/GVFS/CommandLine/DehydrateVerb.cs

View workflow job for this annotation

GitHub Actions / Build and Unit Test (Debug) 

The variable 'ioError' is declared but never used

Check warning on line 262 in GVFS/GVFS/CommandLine/DehydrateVerb.cs

View workflow job for this annotation

GitHub Actions / Build and Unit Test (Release) 

The variable 'ioError' is declared but never used
foreach (string folder in folders)
{
string normalizedPath = GVFSDatabase.NormalizePath(folder);
Expand All @@ -271,26 +278,13 @@
else
{
string fullPath = Path.Combine(enlistment.WorkingDirectoryBackingRoot, folder);
if (this.fileSystem.DirectoryExists(fullPath))
if (!this.fileSystem.DirectoryExists(fullPath))
{
// Since directories are deleted last and will be empty at that point we can skip errors
// while trying to delete it and leave the empty directory and continue to dehydrate
if (!this.TryIO(tracer, () => this.fileSystem.DeleteDirectory(fullPath, ignoreDirectoryDeleteExceptions: true), $"Deleting '{fullPath}'", out ioError))
{
this.WriteMessage(tracer, $"Cannot {this.ActionName} folder '{folder}': removing '{folder}' failed.");
this.WriteMessage(tracer, "Ensure no applications are accessing the folder and retry.");
this.WriteMessage(tracer, $"More details: {ioError}");
folderErrors.Add($"{folder}\0{ioError}");
}
else
{
foldersToDehydrate.Add(folder);
}
this.WriteMessage(tracer, $"Cannot {this.ActionName} folder '{folder}': '{folder}' does not exist.");
foldersToDehydrate.Add(folder);
}
else
{
this.WriteMessage(tracer, $"Cannot {this.ActionName} folder '{folder}': '{folder}' does not exist.");

// Still add to foldersToDehydrate so that any placeholders or modified paths get cleaned up
foldersToDehydrate.Add(folder);
}
Expand All @@ -306,15 +300,9 @@
this.ReportErrorAndExit(tracer, $"{this.ActionName} for folders failed.");
}

// We can skip the version check because dehydrating folders requires that a git status
// be run first, and running git status requires that the repo already be mounted (meaning
// we don't need to perform another version check again)
this.Mount(
tracer,
skipVersionCheck: true);

if (foldersToDehydrate.Count > 0)
{
string backupSrc = GetBackupSrcPath(backupRoot);
this.SendDehydrateMessage(tracer, enlistment, folderErrors, foldersToDehydrate);
}

Expand All @@ -329,6 +317,11 @@
}
}

private static string GetBackupSrcPath(string backupRoot)
{
return Path.Combine(backupRoot, "src");
}

private bool IsFolderValid(string folderPath)
{
if (folderPath == GVFSConstants.DotGit.Root ||
Expand All @@ -353,10 +346,15 @@
{
if (!pipeClient.Connect())
{
this.ReportErrorAndExit("Unable to connect to GVFS. Try running 'gvfs mount'");
this.Output.WriteLine("Mounting...");
this.Mount(tracer, skipVersionCheck: false);
if (!pipeClient.Connect())
{
this.ReportErrorAndExit("Unable to connect to GVFS. Try running 'gvfs mount'");
}
}

NamedPipeMessages.DehydrateFolders.Request request = new NamedPipeMessages.DehydrateFolders.Request(string.Join(FolderListSeparator, folders));
NamedPipeMessages.DehydrateFolders.Request request = new NamedPipeMessages.DehydrateFolders.Request(string.Join(";", folders));
pipeClient.SendRequest(request.CreateMessage());
response = NamedPipeMessages.DehydrateFolders.Response.FromMessage(NamedPipeMessages.Message.FromString(pipeClient.ReadRawResponse()));
}
Expand Down Expand Up @@ -535,12 +533,83 @@
}
}

private bool TryBackupNonSrcFiles(ITracer tracer, GVFSEnlistment enlistment, string backupRoot)
{
string backupSrc = GetBackupSrcPath(backupRoot);
string backupGit = Path.Combine(backupRoot, ".git");
string backupGvfs = Path.Combine(backupRoot, GVFSPlatform.Instance.Constants.DotGVFSRoot);
string backupDatabases = GetBackupDatabasesPath(backupGvfs);

string errorMessage = string.Empty;
if (!this.ShowStatusWhileRunning(
() =>
{
string ioError;
if (!this.TryIO(tracer, () => Directory.CreateDirectory(backupRoot), "Create backup directory", out ioError) ||
!this.TryIO(tracer, () => Directory.CreateDirectory(backupGit), "Create backup .git directory", out ioError) ||
!this.TryIO(tracer, () => Directory.CreateDirectory(backupGvfs), "Create backup .gvfs directory", out ioError) ||
!this.TryIO(tracer, () => Directory.CreateDirectory(backupDatabases), "Create backup .gvfs databases directory", out ioError))
{
errorMessage = "Failed to create backup folders at " + backupRoot + ": " + ioError;
return false;
}

// ... backup the .gvfs hydration-related data structures...
string databasesFolder = Path.Combine(enlistment.DotGVFSRoot, GVFSConstants.DotGVFS.Databases.Name);
if (!this.TryCopyFilesInFolder(tracer, databasesFolder, backupDatabases, searchPattern: "*", filenamesToSkip: "RepoMetadata.dat"))
{
return false;
}

// ... backup everything related to the .git\index...
if (!this.TryIO(
tracer,
() => File.Copy(
Path.Combine(enlistment.DotGitRoot, GVFSConstants.DotGit.IndexName),
Path.Combine(backupGit, GVFSConstants.DotGit.IndexName)),
"Backup the git index",
out errorMessage) ||
!this.TryIO(
tracer,
() => File.Copy(
Path.Combine(enlistment.DotGVFSRoot, GitIndexProjection.ProjectionIndexBackupName),
Path.Combine(backupGvfs, GitIndexProjection.ProjectionIndexBackupName)),
"Backup GVFS_projection",
out errorMessage))
{
return false;
}

// ... backup all .git\*.lock files
if (!this.TryCopyFilesInFolder(tracer, enlistment.DotGitRoot, backupGit, searchPattern: "*.lock"))
{
return false;
}

return true;
},
"Backing up your files"))
{
this.Output.WriteLine();
this.WriteMessage(tracer, "ERROR: " + errorMessage);

return false;
}

return true;
}

private static string GetBackupDatabasesPath(string backupGvfs)
{
return Path.Combine(backupGvfs, GVFSConstants.DotGVFS.Databases.Name);
}

private bool TryBackupFiles(ITracer tracer, GVFSEnlistment enlistment, string backupRoot)
{
string backupSrc = Path.Combine(backupRoot, "src");
string backupSrc = GetBackupSrcPath(backupRoot);
string backupGit = Path.Combine(backupRoot, ".git");
string backupGvfs = Path.Combine(backupRoot, GVFSPlatform.Instance.Constants.DotGVFSRoot);
string backupDatabases = Path.Combine(backupGvfs, GVFSConstants.DotGVFS.Databases.Name);
string backupDatabases = GetBackupDatabasesPath(backupRoot);

string errorMessage = string.Empty;
if (!this.ShowStatusWhileRunning(
Expand Down Expand Up @@ -638,6 +707,28 @@
return true;
}

private bool TryCopyFilesInFolder(ITracer tracer, string folderPath, string backupPath, string searchPattern, params string[] filenamesToSkip)
{
string errorMessage;
foreach (string file in Directory.GetFiles(folderPath, searchPattern))
{
string fileName = Path.GetFileName(file);
if (!filenamesToSkip.Any(x => x.Equals(fileName, GVFSPlatform.Instance.Constants.PathComparison)))
{
if (!this.TryIO(
tracer,
() => File.Copy(file, file.Replace(folderPath, backupPath)),
$"Backing up {Path.GetFileName(file)}",
out errorMessage))
{
return false;
}
}
}

return true;
}

private bool TryDownloadGitObjects(ITracer tracer, GVFSEnlistment enlistment, RetryConfig retryConfig)
{
string errorMessage = null;
Expand Down
Loading