Backport to branch(3) : Add support for administrative operations in Blob adapter

.github/workflows/ci.yaml

@@ -2122,6 +2122,76 @@ jobs:
           name: tidb_8_5_integration_test_reports_${{ matrix.mode.label }}
           path: core/build/reports/tests/integrationTestJdbc
 
+  integration-test-for-blob-storage:
+    name: Blob Storage integration test (${{ matrix.mode.label }})
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        mode:
+          - label: default
+            group_commit_enabled: false
+          - label: with_group_commit
+            group_commit_enabled: true
+
+    services:
+      postgres:
+        image: mcr.microsoft.com/azure-storage/azurite
+        env:
+          AZURITE_ACCOUNTS: "test:test"
+        ports:
+          - 10000:10000
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Set up JDK ${{ env.JAVA_VERSION }} (${{ env.JAVA_VENDOR }})
+        uses: actions/setup-java@v5
+        with:
+          java-version: ${{ env.JAVA_VERSION }}
+          distribution: ${{ env.JAVA_VENDOR }}
+
+      - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}) to run integration test
+        uses: actions/setup-java@v5
+        if: ${{ env.SET_UP_INT_TEST_RUNTIME_NON_ORACLE_JDK == 'true'}}
+        with:
+          java-version: ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}
+          distribution: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}
+
+      - name: Login to Oracle container registry
+        uses: docker/login-action@v3
+        if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
+        with:
+          registry: container-registry.oracle.com
+          username: ${{ secrets.OCR_USERNAME }}
+          password: ${{ secrets.OCR_TOKEN }}
+
+      - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (oracle) to run the integration test
+        if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
+        run: |
+          container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}")
+          docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id"
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v5
+
+      - name: Create Blob Storage container
+        run: |
+          az storage container create \
+            --name test-container \
+            --connection-string "DefaultEndpointsProtocol=http;AccountName=test;AccountKey=test;BlobEndpoint=http://localhost:10000/test;"
+
+      - name: Execute Gradle 'integrationTestObjectStorage' task
+        run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.endpoint=http://localhost:10000/test/test-container -Dscalardb.object_storage.username=test -Dscalardb.object_storage.password=test ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }}
+
+      - name: Upload Gradle test reports
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: blob_storage_integration_test_reports_${{ matrix.mode.label }}
+          path: core/build/reports/tests/integrationTestObjectStorage
+
   integration-test-for-multi-storage:
     name: Multi-storage integration test (${{ matrix.mode.label }})
     runs-on: ubuntu-latest

build.gradle

@@ -28,6 +28,7 @@ subprojects {
         slf4jVersion = '1.7.36'
         cassandraDriverVersion = '3.11.5'
         azureCosmosVersion = '4.75.0'
+        azureBlobStorageVersion = '12.31.3'
         jooqVersion = '3.14.16'
         awssdkVersion = '2.37.3'
         commonsDbcp2Version = '2.13.0'

core/build.gradle

@@ -64,6 +64,16 @@ sourceSets {
         }
         resources.srcDir file('src/integration-test/resources')
     }
+    integrationTestObjectStorage {
+        java {
+            compileClasspath += main.output + test.output
+            runtimeClasspath += main.output + test.output
+            srcDir file('src/integration-test/java')
+            include '**/com/scalar/db/common/*.java'
+            include '**/com/scalar/db/storage/objectstorage/*.java'
+        }
+        resources.srcDir file('src/integration-test/resources')
+    }
     integrationTestMultiStorage {
         java {
             compileClasspath += main.output + test.output
@@ -136,6 +146,9 @@ configurations {
     integrationTestJdbcImplementation.extendsFrom testImplementation
     integrationTestJdbcRuntimeOnly.extendsFrom testRuntimeOnly
     integrationTestJdbcCompileOnly.extendsFrom testCompileOnly
+    integrationTestObjectStorageImplementation.extendsFrom testImplementation
+    integrationTestObjectStorageRuntimeOnly.extendsFrom testRuntimeOnly
+    integrationTestObjectStorageCompileOnly.extendsFrom testCompileOnly
     integrationTestMultiStorageImplementation.extendsFrom testImplementation
     integrationTestMultiStorageRuntimeOnly.extendsFrom testRuntimeOnly
     integrationTestMultiStorageCompileOnly.extendsFrom testCompileOnly
@@ -156,6 +169,7 @@ dependencies {
     implementation "org.slf4j:slf4j-api:${slf4jVersion}"
     implementation "com.datastax.cassandra:cassandra-driver-core:${cassandraDriverVersion}"
     implementation "com.azure:azure-cosmos:${azureCosmosVersion}"
+    implementation "com.azure:azure-storage-blob:${azureBlobStorageVersion}"
     implementation "org.jooq:jooq:${jooqVersion}"
     implementation platform("software.amazon.awssdk:bom:${awssdkVersion}")
     implementation 'software.amazon.awssdk:applicationautoscaling'
@@ -254,6 +268,17 @@ task integrationTestJdbc(type: Test) {
     maxHeapSize = "4g"
 }
 
+task integrationTestObjectStorage(type: Test) {
+    description = 'Runs the integration tests for Object Storages.'
+    group = 'verification'
+    testClassesDirs = sourceSets.integrationTestObjectStorage.output.classesDirs
+    classpath = sourceSets.integrationTestObjectStorage.runtimeClasspath
+    outputs.upToDateWhen { false }  // ensures integration tests are run every time when called
+    options {
+        systemProperties(System.getProperties().findAll{it.key.toString().startsWith("scalardb")})
+    }
+}
+
 task integrationTestMultiStorage(type: Test) {
     description = 'Runs the integration tests for multi-storage.'
     group = 'verification'

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitAdminIntegrationTestWithObjectStorage.java

@@ -0,0 +1,143 @@
+package com.scalar.db.storage.objectstorage;
+
+import com.scalar.db.config.DatabaseConfig;
+import com.scalar.db.transaction.consensuscommit.ConsensusCommitAdminIntegrationTestBase;
+import com.scalar.db.transaction.consensuscommit.ConsensusCommitConfig;
+import com.scalar.db.transaction.consensuscommit.Coordinator;
+import java.util.Properties;
+import org.junit.jupiter.api.Disabled;
+
+public class ConsensusCommitAdminIntegrationTestWithObjectStorage
+    extends ConsensusCommitAdminIntegrationTestBase {
+
+  @Override
+  protected Properties getProps(String testName) {
+    return ObjectStorageEnv.getProperties(testName);
+  }
+
+  @Override
+  protected String getSystemNamespaceName(Properties properties) {
+    return ObjectStorageUtils.getObjectStorageConfig(new DatabaseConfig(properties))
+        .getMetadataNamespace();
+  }
+
+  @Override
+  protected String getCoordinatorNamespaceName(String testName) {
+    return new ConsensusCommitConfig(new DatabaseConfig(getProperties(testName)))
+        .getCoordinatorNamespace()
+        .orElse(Coordinator.NAMESPACE);
+  }
+
+  @Override
+  @Disabled("Temporarily disabled because it includes DML operations")
+  public void truncateTable_ShouldTruncateProperly() {}
+
+  @Override
+  @Disabled("Object Storage does not support index-related operations")
+  public void createIndex_ForAllDataTypesWithExistingData_ShouldCreateIndexesCorrectly() {}
+
+  @Override
+  @Disabled("Object Storage does not support index-related operations")
+  public void createIndex_ForNonExistingTable_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage does not support index-related operations")
+  public void createIndex_ForNonExistingColumn_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage does not support index-related operations")
+  public void createIndex_ForAlreadyExistingIndex_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage does not support index-related operations")
+  public void createIndex_IfNotExists_ForAlreadyExistingIndex_ShouldNotThrowAnyException() {}
+
+  @Override
+  @Disabled("Object Storage does not support index-related operations")
+  public void dropIndex_ForAllDataTypesWithExistingData_ShouldDropIndexCorrectly() {}
+
+  @Override
+  @Disabled("Object Storage does not support index-related operations")
+  public void dropIndex_ForNonExistingTable_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage does not support index-related operations")
+  public void dropIndex_ForNonExistingIndex_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage does not support index-related operations")
+  public void dropIndex_IfExists_ForNonExistingIndex_ShouldNotThrowAnyException() {}
+
+  @Override
+  @Disabled("Object Storage does not support dropping columns")
+  public void dropColumnFromTable_DropColumnForEachExistingDataType_ShouldDropColumnsCorrectly() {}
+
+  @Override
+  @Disabled("Object Storage does not support dropping columns")
+  public void dropColumnFromTable_ForNonExistingTable_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage does not support dropping columns")
+  public void dropColumnFromTable_ForNonExistingColumn_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage does not support dropping columns")
+  public void dropColumnFromTable_ForPrimaryKeyColumn_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage does not support dropping columns")
+  public void dropColumnFromTable_ForIndexedColumn_ShouldDropColumnAndIndexCorrectly() {}
+
+  @Override
+  @Disabled("Object Storage does not support renaming columns")
+  public void renameColumn_ShouldRenameColumnCorrectly() {}
+
+  @Override
+  @Disabled("Object Storage does not support renaming columns")
+  public void renameColumn_ForNonExistingTable_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage does not support renaming columns")
+  public void renameColumn_ForNonExistingColumn_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage does not support renaming columns")
+  public void renameColumn_ForPrimaryKeyColumn_ShouldRenameColumnCorrectly() {}
+
+  @Override
+  @Disabled("Object Storage does not support renaming columns")
+  public void renameColumn_ForIndexKeyColumn_ShouldRenameColumnAndIndexCorrectly() {}
+
+  @Override
+  @Disabled("Object Storage does not support altering column types")
+  public void
+      alterColumnType_AlterColumnTypeFromEachExistingDataTypeToText_ShouldAlterColumnTypesCorrectly() {}
+
+  @Override
+  @Disabled("Object Storage does not support altering column types")
+  public void alterColumnType_WideningConversion_ShouldAlterColumnTypesCorrectly() {}
+
+  @Override
+  @Disabled("Object Storage does not support altering column types")
+  public void alterColumnType_ForPrimaryKeyOrIndexKeyColumn_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage does not support renaming tables")
+  public void renameTable_ForExistingTable_ShouldRenameTableCorrectly() {}
+
+  @Override
+  @Disabled("Object Storage does not support renaming tables")
+  public void renameTable_ForNonExistingTable_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage does not support renaming tables")
+  public void renameTable_IfNewTableNameAlreadyExists_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage does not support renaming tables")
+  public void renameTable_ForExistingTableWithIndexes_ShouldRenameTableAndIndexesCorrectly() {}
+
+  @Override
+  @Disabled("Object Storage does not support renaming tables")
+  public void renameTable_IfOnlyOneTableExists_ShouldRenameTableCorrectly() {}
+}

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitAdminRepairTableIntegrationTestWithObjectStorage.java

@@ -0,0 +1,29 @@
+package com.scalar.db.storage.objectstorage;
+
+import com.scalar.db.transaction.consensuscommit.ConsensusCommitAdminRepairTableIntegrationTestBase;
+import com.scalar.db.util.AdminTestUtils;
+import java.util.Properties;
+import org.junit.jupiter.api.Disabled;
+
+public class ConsensusCommitAdminRepairTableIntegrationTestWithObjectStorage
+    extends ConsensusCommitAdminRepairTableIntegrationTestBase {
+
+  @Override
+  protected Properties getProps(String testName) {
+    return ObjectStorageEnv.getProperties(testName);
+  }
+
+  @Override
+  protected AdminTestUtils getAdminTestUtils(String testName) {
+    return new ObjectStorageAdminTestUtils(getProperties(testName));
+  }
+
+  @Override
+  @Disabled("Object Storage recreates missing coordinator tables")
+  public void
+      repairTableAndCoordinatorTable_CoordinatorTablesDoNotExist_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage recreates missing coordinator tables")
+  public void repairTable_ForNonExistingTable_ShouldThrowIllegalArgument() {}
+}