From 29d16531cd5570b6cb93082fff948bdebaecf0c8 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Fri, 31 Oct 2025 12:23:49 +0300 Subject: [PATCH 1/2] done --- ydb/public/api/grpc/ydb_export_v1.proto | 4 ++ ydb/public/api/grpc/ydb_import_v1.proto | 4 ++ ydb/public/api/protos/ydb_export.proto | 50 ++++++++++++++++++++ ydb/public/api/protos/ydb_import.proto | 61 ++++++++++++++++++++++++- 4 files changed, 118 insertions(+), 1 deletion(-) diff --git a/ydb/public/api/grpc/ydb_export_v1.proto b/ydb/public/api/grpc/ydb_export_v1.proto index e3768157751c..e4cf609acf5f 100644 --- a/ydb/public/api/grpc/ydb_export_v1.proto +++ b/ydb/public/api/grpc/ydb_export_v1.proto @@ -14,4 +14,8 @@ service ExportService { // Exports data to S3. // Method starts an asynchronous operation that can be cancelled while it is in progress. rpc ExportToS3(Export.ExportToS3Request) returns (Export.ExportToS3Response); + + // Exports data to NFS. + // Method starts an asynchronous operation that can be cancelled while it is in progress. + rpc ExportToNfs(ExportToNfsRequest) returns (ExportToNfsResponse); } diff --git a/ydb/public/api/grpc/ydb_import_v1.proto b/ydb/public/api/grpc/ydb_import_v1.proto index f921c12121a6..df8d42f69a09 100644 --- a/ydb/public/api/grpc/ydb_import_v1.proto +++ b/ydb/public/api/grpc/ydb_import_v1.proto @@ -11,6 +11,10 @@ service ImportService { // Method starts an asynchronous operation that can be cancelled while it is in progress. rpc ImportFromS3(Import.ImportFromS3Request) returns (Import.ImportFromS3Response); + // Imports data from NFS. + // Method starts an asynchronous operation that can be cancelled while it is in progress. + rpc ImportFromNfs(Import.ImportFromNfsRequest) returns (Import.ImportFromNfsResponse); + // List objects from existing export stored in S3 bucket rpc ListObjectsInS3Export(Import.ListObjectsInS3ExportRequest) returns (Import.ListObjectsInS3ExportResponse); diff --git a/ydb/public/api/protos/ydb_export.proto b/ydb/public/api/protos/ydb_export.proto index 1a0a28ea1368..1a87dfda81b0 100644 --- a/ydb/public/api/protos/ydb_export.proto +++ b/ydb/public/api/protos/ydb_export.proto @@ -181,3 +181,53 @@ message EncryptionSettings { bytes key = 1 [(Ydb.sensitive) = true]; } } + +/// NFS +message ExportToNfsSettings { + message Item { + // Database path to a table to be exported + string source_path = 1 [(required) = true]; + + /* The tables are exported to one or more files in NFS. + The files are saved to the destination_path directory. */ + string destination_path = 2 [(required) = true]; + } + + // Base path on mounted NFS where to write export + // Example: /mnt/exports + string base_path = 1 [(required) = true]; + + // List of items to export + repeated Item items = 2; + + // Optional description + string description = 3 [(length).le = 128]; + + // Number of retries for failed operations + uint32 number_of_retries = 4; + + // Codec used to compress data. Codecs are available: + // - zstd. + // - zstd-N, where N is compression level, e.g. zstd-3. + string compression = 5; +} + +message ExportToNfsResult { +} + +message ExportToNfsMetadata { + ExportToNfsSettings settings = 1; + ExportProgress.Progress progress = 2; + repeated ExportItemProgress items_progress = 3; +} + +message ExportToNfsRequest { + Ydb.Operations.OperationParams operation_params = 1; + ExportToNfsSettings settings = 2 [(required) = true]; +} + +message ExportToNfsResponse { + // operation.result = ExportToNfsResult + // operation.metadata = ExportToNfsMetadata + Ydb.Operations.Operation operation = 1; +} diff --git a/ydb/public/api/protos/ydb_import.proto b/ydb/public/api/protos/ydb_import.proto index 86c3a0d7f371..95f5b2703837 100644 --- a/ydb/public/api/protos/ydb_import.proto +++ b/ydb/public/api/protos/ydb_import.proto @@ -45,7 +45,8 @@ message ImportFromS3Settings { The S3 object name begins with a prefix, followed by: * '/data_PartNumber', where 'PartNumber' represents the index of the part, starting at zero; * '/scheme.pb' - object with information about scheme, indexes, etc; - * '/permissions.pb' - object with information about ACL and owner. + * '/permissions.pb' - object with information about ACL and owner; + * '/metadata.json' - object with metadata about the backup. */ // The S3 object prefix can be either provided explicitly @@ -121,6 +122,64 @@ message ImportFromS3Response { Ydb.Operations.Operation operation = 1; } +/// NFS +message ImportFromNfsSettings { + message Item { + /* YDB tables in NFS are stored in a directory structure (see ydb_export.proto). + The directory contains: + * '/data_PartNumber', where 'PartNumber' represents the index of the part, starting at zero; + * '/scheme.pb' - object with information about scheme, indexes, etc; + * '/permissions.pb' - object with information about ACL and owner; + * '/metadata.json' - object with metadata about the backup. + The path in NFS is specified relative to base_path. + Example: "my_export/table1" + */ + string source_path = 1 [(required) = true]; + + // Database path to a table to import to. + string destination_path = 2 [(required) = true]; + } + + // Base path on mounted NFS where the export is located + // Example: /mnt/exports + string base_path = 1 [(required) = true]; + + repeated Item items = 2; // Empty collection means import of all export objects + + // Optional description + string description = 3 [(length).le = 128]; + + // Number of retries for failed file operations + uint32 number_of_retries = 4; + + // Prevent importing of ACL and owner. If true, objects are created with empty ACL + // and their owner will be the user who started the import. + bool no_acl = 5; + + // Skip checksum validation during import + bool skip_checksum_validation = 6; +} + +message ImportFromNfsResult { +} + +message ImportFromNfsMetadata { + ImportFromNfsSettings settings = 1; + ImportProgress.Progress progress = 2; + repeated ImportItemProgress items_progress = 3; +} + +message ImportFromNfsRequest { + Ydb.Operations.OperationParams operation_params = 1; + ImportFromNfsSettings settings = 2 [(required) = true]; +} + +message ImportFromNfsResponse { + // operation.result = ImportFromNfsResult + // operation.metadata = ImportFromNfsMetadata + Ydb.Operations.Operation operation = 1; +} + message ListObjectsInS3ExportSettings { message Item { // Database object path From 2b82caaa0b33a7248f30bbb837279417ab1a7d74 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Sat, 1 Nov 2025 11:24:08 +0300 Subject: [PATCH 2/2] nfs -> fs --- ydb/public/api/grpc/ydb_export_v1.proto | 4 ++-- ydb/public/api/grpc/ydb_import_v1.proto | 4 ++-- ydb/public/api/protos/ydb_export.proto | 25 ++++++++++++----------- ydb/public/api/protos/ydb_import.proto | 27 +++++++++++++------------ 4 files changed, 31 insertions(+), 29 deletions(-) diff --git a/ydb/public/api/grpc/ydb_export_v1.proto b/ydb/public/api/grpc/ydb_export_v1.proto index e4cf609acf5f..966fa5010030 100644 --- a/ydb/public/api/grpc/ydb_export_v1.proto +++ b/ydb/public/api/grpc/ydb_export_v1.proto @@ -15,7 +15,7 @@ service ExportService { // Method starts an asynchronous operation that can be cancelled while it is in progress. rpc ExportToS3(Export.ExportToS3Request) returns (Export.ExportToS3Response); - // Exports data to NFS. + // Exports data to file system. // Method starts an asynchronous operation that can be cancelled while it is in progress. - rpc ExportToNfs(ExportToNfsRequest) returns (ExportToNfsResponse); + rpc ExportToFs(ExportToFsRequest) returns (ExportToFsResponse); } diff --git a/ydb/public/api/grpc/ydb_import_v1.proto b/ydb/public/api/grpc/ydb_import_v1.proto index df8d42f69a09..3ac2d0ca53a9 100644 --- a/ydb/public/api/grpc/ydb_import_v1.proto +++ b/ydb/public/api/grpc/ydb_import_v1.proto @@ -11,9 +11,9 @@ service ImportService { // Method starts an asynchronous operation that can be cancelled while it is in progress. rpc ImportFromS3(Import.ImportFromS3Request) returns (Import.ImportFromS3Response); - // Imports data from NFS. + // Imports data from file system. // Method starts an asynchronous operation that can be cancelled while it is in progress. - rpc ImportFromNfs(Import.ImportFromNfsRequest) returns (Import.ImportFromNfsResponse); + rpc ImportFromFs(Import.ImportFromFsRequest) returns (Import.ImportFromFsResponse); // List objects from existing export stored in S3 bucket rpc ListObjectsInS3Export(Import.ListObjectsInS3ExportRequest) returns (Import.ListObjectsInS3ExportResponse); diff --git a/ydb/public/api/protos/ydb_export.proto b/ydb/public/api/protos/ydb_export.proto index 1a87dfda81b0..7c22e9f57e8f 100644 --- a/ydb/public/api/protos/ydb_export.proto +++ b/ydb/public/api/protos/ydb_export.proto @@ -182,18 +182,19 @@ message EncryptionSettings { } } -/// NFS -message ExportToNfsSettings { +/// File system (FS) +message ExportToFsSettings { message Item { // Database path to a table to be exported string source_path = 1 [(required) = true]; - /* The tables are exported to one or more files in NFS. + /* The tables are exported to one or more files in FS. The files are saved to the destination_path directory. */ string destination_path = 2 [(required) = true]; } - // Base path on mounted NFS where to write export + // Base path on FS where to write export + // Path to the mounted directory in the case of NFS // Example: /mnt/exports string base_path = 1 [(required) = true]; @@ -212,22 +213,22 @@ message ExportToNfsSettings { string compression = 5; } -message ExportToNfsResult { +message ExportToFsResult { } -message ExportToNfsMetadata { - ExportToNfsSettings settings = 1; +message ExportToFsMetadata { + ExportToFsSettings settings = 1; ExportProgress.Progress progress = 2; repeated ExportItemProgress items_progress = 3; } -message ExportToNfsRequest { +message ExportToFsRequest { Ydb.Operations.OperationParams operation_params = 1; - ExportToNfsSettings settings = 2 [(required) = true]; + ExportToFsSettings settings = 2 [(required) = true]; } -message ExportToNfsResponse { - // operation.result = ExportToNfsResult - // operation.metadata = ExportToNfsMetadata +message ExportToFsResponse { + // operation.result = ExportToFsResult + // operation.metadata = ExportToFsMetadata Ydb.Operations.Operation operation = 1; } diff --git a/ydb/public/api/protos/ydb_import.proto b/ydb/public/api/protos/ydb_import.proto index 95f5b2703837..cad19e6bd9e5 100644 --- a/ydb/public/api/protos/ydb_import.proto +++ b/ydb/public/api/protos/ydb_import.proto @@ -122,16 +122,16 @@ message ImportFromS3Response { Ydb.Operations.Operation operation = 1; } -/// NFS -message ImportFromNfsSettings { +/// File system (FS) +message ImportFromFsSettings { message Item { - /* YDB tables in NFS are stored in a directory structure (see ydb_export.proto). + /* YDB tables in FS are stored in a directory structure (see ydb_export.proto). The directory contains: * '/data_PartNumber', where 'PartNumber' represents the index of the part, starting at zero; * '/scheme.pb' - object with information about scheme, indexes, etc; * '/permissions.pb' - object with information about ACL and owner; * '/metadata.json' - object with metadata about the backup. - The path in NFS is specified relative to base_path. + The path in FS is specified relative to base_path. Example: "my_export/table1" */ string source_path = 1 [(required) = true]; @@ -140,7 +140,8 @@ message ImportFromNfsSettings { string destination_path = 2 [(required) = true]; } - // Base path on mounted NFS where the export is located + // Base path on FS where the export is located + // Path to the mounted directory in the case of NFS // Example: /mnt/exports string base_path = 1 [(required) = true]; @@ -160,23 +161,23 @@ message ImportFromNfsSettings { bool skip_checksum_validation = 6; } -message ImportFromNfsResult { +message ImportFromFsResult { } -message ImportFromNfsMetadata { - ImportFromNfsSettings settings = 1; +message ImportFromFsMetadata { + ImportFromFsSettings settings = 1; ImportProgress.Progress progress = 2; repeated ImportItemProgress items_progress = 3; } -message ImportFromNfsRequest { +message ImportFromFsRequest { Ydb.Operations.OperationParams operation_params = 1; - ImportFromNfsSettings settings = 2 [(required) = true]; + ImportFromFsSettings settings = 2 [(required) = true]; } -message ImportFromNfsResponse { - // operation.result = ImportFromNfsResult - // operation.metadata = ImportFromNfsMetadata +message ImportFromFsResponse { + // operation.result = ImportFromFsResult + // operation.metadata = ImportFromFsMetadata Ydb.Operations.Operation operation = 1; }