Skip to content

Commit b3d44a1

Browse files
KodaiDbrfrn169
KodaiD
and
brfrn169
authored
Add support for administrative operations in Blob adapter (#3104)
Co-authored-by: Toshihiro Suzuki <brfrn169@gmail.com>
1 parent 69a70fa commit b3d44a1

File tree

35 files changed

+3689
-38
lines changed

35 files changed

+3689
-38
lines changed

‎.github/workflows/ci.yaml‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2122,6 +2122,76 @@ jobs:
21222122
name: tidb_8_5_integration_test_reports_${{ matrix.mode.label }}
21232123
path: core/build/reports/tests/integrationTestJdbc
21242124

2125+
integration-test-for-blob-storage:
2126+
name: Blob Storage integration test (${{ matrix.mode.label }})
2127+
runs-on: ubuntu-latest
2128+
2129+
strategy:
2130+
fail-fast: false
2131+
matrix:
2132+
mode:
2133+
- label: default
2134+
group_commit_enabled: false
2135+
- label: with_group_commit
2136+
group_commit_enabled: true
2137+
2138+
services:
2139+
postgres:
2140+
image: mcr.microsoft.com/azure-storage/azurite
2141+
env:
2142+
AZURITE_ACCOUNTS: "test:test"
2143+
ports:
2144+
- 10000:10000
2145+
2146+
steps:
2147+
- uses: actions/checkout@v5
2148+
2149+
- name: Set up JDK ${{ env.JAVA_VERSION }} (${{ env.JAVA_VENDOR }})
2150+
uses: actions/setup-java@v5
2151+
with:
2152+
java-version: ${{ env.JAVA_VERSION }}
2153+
distribution: ${{ env.JAVA_VENDOR }}
2154+
2155+
- name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}) to run integration test
2156+
uses: actions/setup-java@v5
2157+
if: ${{ env.SET_UP_INT_TEST_RUNTIME_NON_ORACLE_JDK == 'true'}}
2158+
with:
2159+
java-version: ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}
2160+
distribution: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}
2161+
2162+
- name: Login to Oracle container registry
2163+
uses: docker/login-action@v3
2164+
if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
2165+
with:
2166+
registry: container-registry.oracle.com
2167+
username: ${{ secrets.OCR_USERNAME }}
2168+
password: ${{ secrets.OCR_TOKEN }}
2169+
2170+
- name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (oracle) to run the integration test
2171+
if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
2172+
run: |
2173+
container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}")
2174+
docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id"
2175+
2176+
- name: Setup Gradle
2177+
uses: gradle/actions/setup-gradle@v5
2178+
2179+
- name: Create Blob Storage container
2180+
run: |
2181+
az storage container create \
2182+
--name test-container \
2183+
--connection-string "DefaultEndpointsProtocol=http;AccountName=test;AccountKey=test;BlobEndpoint=http://localhost:10000/test;"
2184+
2185+
- name: Execute Gradle 'integrationTestObjectStorage' task
2186+
run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.endpoint=http://localhost:10000/test/test-container -Dscalardb.object_storage.username=test -Dscalardb.object_storage.password=test ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }}
2187+
2188+
- name: Upload Gradle test reports
2189+
if: always()
2190+
uses: actions/upload-artifact@v4
2191+
with:
2192+
name: blob_storage_integration_test_reports_${{ matrix.mode.label }}
2193+
path: core/build/reports/tests/integrationTestObjectStorage
2194+
21252195
integration-test-for-multi-storage:
21262196
name: Multi-storage integration test (${{ matrix.mode.label }})
21272197
runs-on: ubuntu-latest

‎build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ subprojects {
2828
slf4jVersion = '1.7.36'
2929
cassandraDriverVersion = '3.11.5'
3030
azureCosmosVersion = '4.75.0'
31+
azureBlobStorageVersion = '12.31.3'
3132
jooqVersion = '3.14.16'
3233
awssdkVersion = '2.37.3'
3334
commonsDbcp2Version = '2.13.0'

‎core/build.gradle‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,16 @@ sourceSets {
6464
}
6565
resources.srcDir file('src/integration-test/resources')
6666
}
67+
integrationTestObjectStorage {
68+
java {
69+
compileClasspath += main.output + test.output
70+
runtimeClasspath += main.output + test.output
71+
srcDir file('src/integration-test/java')
72+
include '**/com/scalar/db/common/*.java'
73+
include '**/com/scalar/db/storage/objectstorage/*.java'
74+
}
75+
resources.srcDir file('src/integration-test/resources')
76+
}
6777
integrationTestMultiStorage {
6878
java {
6979
compileClasspath += main.output + test.output
@@ -136,6 +146,9 @@ configurations {
136146
integrationTestJdbcImplementation.extendsFrom testImplementation
137147
integrationTestJdbcRuntimeOnly.extendsFrom testRuntimeOnly
138148
integrationTestJdbcCompileOnly.extendsFrom testCompileOnly
149+
integrationTestObjectStorageImplementation.extendsFrom testImplementation
150+
integrationTestObjectStorageRuntimeOnly.extendsFrom testRuntimeOnly
151+
integrationTestObjectStorageCompileOnly.extendsFrom testCompileOnly
139152
integrationTestMultiStorageImplementation.extendsFrom testImplementation
140153
integrationTestMultiStorageRuntimeOnly.extendsFrom testRuntimeOnly
141154
integrationTestMultiStorageCompileOnly.extendsFrom testCompileOnly
@@ -156,6 +169,7 @@ dependencies {
156169
implementation "org.slf4j:slf4j-api:${slf4jVersion}"
157170
implementation "com.datastax.cassandra:cassandra-driver-core:${cassandraDriverVersion}"
158171
implementation "com.azure:azure-cosmos:${azureCosmosVersion}"
172+
implementation "com.azure:azure-storage-blob:${azureBlobStorageVersion}"
159173
implementation "org.jooq:jooq:${jooqVersion}"
160174
implementation platform("software.amazon.awssdk:bom:${awssdkVersion}")
161175
implementation 'software.amazon.awssdk:applicationautoscaling'
@@ -254,6 +268,17 @@ task integrationTestJdbc(type: Test) {
254268
maxHeapSize = "4g"
255269
}
256270

271+
task integrationTestObjectStorage(type: Test) {
272+
description = 'Runs the integration tests for Object Storages.'
273+
group = 'verification'
274+
testClassesDirs = sourceSets.integrationTestObjectStorage.output.classesDirs
275+
classpath = sourceSets.integrationTestObjectStorage.runtimeClasspath
276+
outputs.upToDateWhen { false } // ensures integration tests are run every time when called
277+
options {
278+
systemProperties(System.getProperties().findAll{it.key.toString().startsWith("scalardb")})
279+
}
280+
}
281+
257282
task integrationTestMultiStorage(type: Test) {
258283
description = 'Runs the integration tests for multi-storage.'
259284
group = 'verification'

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitAdminIntegrationTestWithObjectStorage.java‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
package com.scalar.db.storage.objectstorage;
2+
3+
import com.scalar.db.transaction.consensuscommit.ConsensusCommitAdminIntegrationTestBase;
4+
import com.scalar.db.util.AdminTestUtils;
5+
import java.util.Properties;
6+
import org.junit.jupiter.api.Disabled;
7+
8+
public class ConsensusCommitAdminIntegrationTestWithObjectStorage
9+
extends ConsensusCommitAdminIntegrationTestBase {
10+
11+
@Override
12+
protected Properties getProps(String testName) {
13+
return ObjectStorageEnv.getProperties(testName);
14+
}
15+
16+
@Override
17+
protected AdminTestUtils getAdminTestUtils(String testName) {
18+
return new ObjectStorageAdminTestUtils(getProperties(testName));
19+
}
20+
21+
@Override
22+
@Disabled("Temporarily disabled because it includes DML operations")
23+
public void truncateTable_ShouldTruncateProperly() {}
24+
25+
@Override
26+
@Disabled("Object Storage does not support index-related operations")
27+
public void createIndex_ForAllDataTypesWithExistingData_ShouldCreateIndexesCorrectly() {}
28+
29+
@Override
30+
@Disabled("Object Storage does not support index-related operations")
31+
public void createIndex_ForNonExistingTable_ShouldThrowIllegalArgumentException() {}
32+
33+
@Override
34+
@Disabled("Object Storage does not support index-related operations")
35+
public void createIndex_ForNonExistingColumn_ShouldThrowIllegalArgumentException() {}
36+
37+
@Override
38+
@Disabled("Object Storage does not support index-related operations")
39+
public void createIndex_ForAlreadyExistingIndex_ShouldThrowIllegalArgumentException() {}
40+
41+
@Override
42+
@Disabled("Object Storage does not support index-related operations")
43+
public void createIndex_IfNotExists_ForAlreadyExistingIndex_ShouldNotThrowAnyException() {}
44+
45+
@Override
46+
@Disabled("Object Storage does not support index-related operations")
47+
public void dropIndex_ForAllDataTypesWithExistingData_ShouldDropIndexCorrectly() {}
48+
49+
@Override
50+
@Disabled("Object Storage does not support index-related operations")
51+
public void dropIndex_ForNonExistingTable_ShouldThrowIllegalArgumentException() {}
52+
53+
@Override
54+
@Disabled("Object Storage does not support index-related operations")
55+
public void dropIndex_ForNonExistingIndex_ShouldThrowIllegalArgumentException() {}
56+
57+
@Override
58+
@Disabled("Object Storage does not support index-related operations")
59+
public void dropIndex_IfExists_ForNonExistingIndex_ShouldNotThrowAnyException() {}
60+
61+
@Override
62+
@Disabled("Object Storage does not support dropping columns")
63+
public void dropColumnFromTable_DropColumnForEachExistingDataType_ShouldDropColumnsCorrectly() {}
64+
65+
@Override
66+
@Disabled("Object Storage does not support dropping columns")
67+
public void dropColumnFromTable_ForNonExistingTable_ShouldThrowIllegalArgumentException() {}
68+
69+
@Override
70+
@Disabled("Object Storage does not support dropping columns")
71+
public void dropColumnFromTable_ForNonExistingColumn_ShouldThrowIllegalArgumentException() {}
72+
73+
@Override
74+
@Disabled("Object Storage does not support dropping columns")
75+
public void dropColumnFromTable_ForPrimaryKeyColumn_ShouldThrowIllegalArgumentException() {}
76+
77+
@Override
78+
@Disabled("Object Storage does not support dropping columns")
79+
public void dropColumnFromTable_ForIndexedColumn_ShouldDropColumnAndIndexCorrectly() {}
80+
81+
@Override
82+
@Disabled("Object Storage does not support renaming columns")
83+
public void renameColumn_ShouldRenameColumnCorrectly() {}
84+
85+
@Override
86+
@Disabled("Object Storage does not support renaming columns")
87+
public void renameColumn_ForNonExistingTable_ShouldThrowIllegalArgumentException() {}
88+
89+
@Override
90+
@Disabled("Object Storage does not support renaming columns")
91+
public void renameColumn_ForNonExistingColumn_ShouldThrowIllegalArgumentException() {}
92+
93+
@Override
94+
@Disabled("Object Storage does not support renaming columns")
95+
public void renameColumn_ForPrimaryKeyColumn_ShouldRenameColumnCorrectly() {}
96+
97+
@Override
98+
@Disabled("Object Storage does not support renaming columns")
99+
public void renameColumn_ForIndexKeyColumn_ShouldRenameColumnAndIndexCorrectly() {}
100+
101+
@Override
102+
@Disabled("Object Storage does not support altering column types")
103+
public void
104+
alterColumnType_AlterColumnTypeFromEachExistingDataTypeToText_ShouldAlterColumnTypesCorrectly() {}
105+
106+
@Override
107+
@Disabled("Object Storage does not support altering column types")
108+
public void alterColumnType_WideningConversion_ShouldAlterColumnTypesCorrectly() {}
109+
110+
@Override
111+
@Disabled("Object Storage does not support altering column types")
112+
public void alterColumnType_ForPrimaryKeyOrIndexKeyColumn_ShouldThrowIllegalArgumentException() {}
113+
114+
@Override
115+
@Disabled("Object Storage does not support renaming tables")
116+
public void renameTable_ForExistingTable_ShouldRenameTableCorrectly() {}
117+
118+
@Override
119+
@Disabled("Object Storage does not support renaming tables")
120+
public void renameTable_ForNonExistingTable_ShouldThrowIllegalArgumentException() {}
121+
122+
@Override
123+
@Disabled("Object Storage does not support renaming tables")
124+
public void renameTable_IfNewTableNameAlreadyExists_ShouldThrowIllegalArgumentException() {}
125+
126+
@Override
127+
@Disabled("Object Storage does not support renaming tables")
128+
public void renameTable_ForExistingTableWithIndexes_ShouldRenameTableAndIndexesCorrectly() {}
129+
130+
@Override
131+
@Disabled("Object Storage does not support renaming tables")
132+
public void renameTable_IfOnlyOneTableExists_ShouldRenameTableCorrectly() {}
133+
134+
@Override
135+
@Disabled("The ScalarDB environment does not need to be upgraded with Object Storage")
136+
public void
137+
upgrade_WhenMetadataTableExistsButNotNamespacesTable_ShouldCreateNamespacesTableAndImportExistingNamespaces() {}
138+
}

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitAdminRepairIntegrationTestWithObjectStorage.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
package com.scalar.db.storage.objectstorage;
2+
3+
import com.scalar.db.transaction.consensuscommit.ConsensusCommitAdminRepairIntegrationTestBase;
4+
import java.util.Properties;
5+
6+
public class ConsensusCommitAdminRepairIntegrationTestWithObjectStorage
7+
extends ConsensusCommitAdminRepairIntegrationTestBase {
8+
9+
@Override
10+
protected Properties getProps(String testName) {
11+
return ObjectStorageEnv.getProperties(testName);
12+
}
13+
14+
@Override
15+
protected void initialize(String testName) throws Exception {
16+
super.initialize(testName);
17+
adminTestUtils = new ObjectStorageAdminTestUtils(getProperties(testName));
18+
}
19+
}

0 commit comments

Comments
 (0)