ExportToFs / ImportFromFs API (ydb-platform#28072)

Author: stanislav-shchetinin

ydb/public/api/grpc/ydb_export_v1.proto

@@ -14,4 +14,8 @@ service ExportService {
     // Exports data to S3.
     // Method starts an asynchronous operation that can be cancelled while it is in progress.
     rpc ExportToS3(Export.ExportToS3Request) returns (Export.ExportToS3Response);
+
+    // Exports data to file system.
+    // Method starts an asynchronous operation that can be cancelled while it is in progress.
+    rpc ExportToFs(ExportToFsRequest) returns (ExportToFsResponse);
 }

ydb/public/api/grpc/ydb_import_v1.proto

@@ -11,6 +11,10 @@ service ImportService {
     // Method starts an asynchronous operation that can be cancelled while it is in progress.
     rpc ImportFromS3(Import.ImportFromS3Request) returns (Import.ImportFromS3Response);
 
+    // Imports data from file system.
+    // Method starts an asynchronous operation that can be cancelled while it is in progress.
+    rpc ImportFromFs(Import.ImportFromFsRequest) returns (Import.ImportFromFsResponse);
+
     // List objects from existing export stored in S3 bucket
     rpc ListObjectsInS3Export(Import.ListObjectsInS3ExportRequest) returns (Import.ListObjectsInS3ExportResponse);
 

ydb/public/api/protos/ydb_export.proto

@@ -181,3 +181,54 @@ message EncryptionSettings {
         bytes key = 1 [(Ydb.sensitive) = true];
     }
 }
+
+/// File system (FS)
+message ExportToFsSettings {
+    message Item {
+        // Database path to a table to be exported
+        string source_path = 1 [(required) = true];
+
+        /* The tables are exported to one or more files in FS.
+        The files are saved to the destination_path directory. */
+        string destination_path = 2 [(required) = true];
+    }
+
+    // Base path on FS where to write export
+    // Path to the mounted directory in the case of NFS
+    // Example: /mnt/exports
+    string base_path = 1 [(required) = true];
+    
+    // List of items to export
+    repeated Item items = 2;
+    
+    // Optional description
+    string description = 3 [(length).le = 128];
+    
+    // Number of retries for failed operations
+    uint32 number_of_retries = 4;
+    
+    // Codec used to compress data. Codecs are available:
+    // - zstd.
+    // - zstd-N, where N is compression level, e.g. zstd-3.
+    string compression = 5;
+}
+
+message ExportToFsResult {
+}
+
+message ExportToFsMetadata {
+    ExportToFsSettings settings = 1;
+    ExportProgress.Progress progress = 2;
+    repeated ExportItemProgress items_progress = 3;
+}
+
+message ExportToFsRequest {
+    Ydb.Operations.OperationParams operation_params = 1;
+    ExportToFsSettings settings = 2 [(required) = true];
+}
+
+message ExportToFsResponse {
+    // operation.result = ExportToFsResult
+    // operation.metadata = ExportToFsMetadata
+    Ydb.Operations.Operation operation = 1;
+}

ydb/public/api/protos/ydb_import.proto

@@ -45,7 +45,8 @@ message ImportFromS3Settings {
             The S3 object name begins with a prefix, followed by:
                 * '/data_PartNumber', where 'PartNumber' represents the index of the part, starting at zero;
                 * '/scheme.pb' - object with information about scheme, indexes, etc;
-                * '/permissions.pb' - object with information about ACL and owner.
+                * '/permissions.pb' - object with information about ACL and owner;
+                * '/metadata.json' - object with metadata about the backup.
             */
 
             // The S3 object prefix can be either provided explicitly
@@ -121,6 +122,65 @@ message ImportFromS3Response {
     Ydb.Operations.Operation operation = 1;
 }
 
+/// File system (FS)
+message ImportFromFsSettings {
+    message Item {
+        /* YDB tables in FS are stored in a directory structure (see ydb_export.proto).
+           The directory contains:
+             * '/data_PartNumber', where 'PartNumber' represents the index of the part, starting at zero;
+             * '/scheme.pb' - object with information about scheme, indexes, etc;
+             * '/permissions.pb' - object with information about ACL and owner;
+             * '/metadata.json' - object with metadata about the backup.
+           The path in FS is specified relative to base_path.
+           Example: "my_export/table1"
+        */
+        string source_path = 1 [(required) = true];
+
+        // Database path to a table to import to.
+        string destination_path = 2 [(required) = true];
+    }
+
+    // Base path on FS where the export is located
+    // Path to the mounted directory in the case of NFS
+    // Example: /mnt/exports
+    string base_path = 1 [(required) = true];
+
+    repeated Item items = 2; // Empty collection means import of all export objects
+
+    // Optional description
+    string description = 3 [(length).le = 128];
+
+    // Number of retries for failed file operations
+    uint32 number_of_retries = 4;
+
+    // Prevent importing of ACL and owner. If true, objects are created with empty ACL
+    // and their owner will be the user who started the import.
+    bool no_acl = 5;
+
+    // Skip checksum validation during import
+    bool skip_checksum_validation = 6;
+}
+
+message ImportFromFsResult {
+}
+
+message ImportFromFsMetadata {
+    ImportFromFsSettings settings = 1;
+    ImportProgress.Progress progress = 2;
+    repeated ImportItemProgress items_progress = 3;
+}
+
+message ImportFromFsRequest {
+    Ydb.Operations.OperationParams operation_params = 1;
+    ImportFromFsSettings settings = 2 [(required) = true];
+}
+
+message ImportFromFsResponse {
+    // operation.result = ImportFromFsResult
+    // operation.metadata = ImportFromFsMetadata
+    Ydb.Operations.Operation operation = 1;
+}
+
 message ListObjectsInS3ExportSettings {
     message Item {
         // Database object path