Skip to content

Commit f339b87

Browse files
SylvainSenechalSylvainSenechal
committed
add tests for crr existing objects
Issue : ZENKO-5030
1 parent 1709d43 commit f339b87

File tree

14 files changed

+334
-3
lines changed

14 files changed

+334
-3
lines changed

‎.github/scripts/end2end/configure-e2e.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ kubectl run ${POD_NAME} \
9696
--env=AWS_SECRET_KEY=${AWS_SECRET_KEY} \
9797
--env=AWS_ENDPOINT=${AWS_ENDPOINT} \
9898
--env=AWS_FAIL_BUCKET_NAME=${AWS_FAIL_BUCKET_NAME} \
99+
--env=AWS_REPLICATION_CTST_BUCKET_NAME=${AWS_REPLICATION_CTST_BUCKET_NAME} \
99100
--env=AZURE_BACKEND_DESTINATION_LOCATION=${AZURE_BACKEND_DESTINATION_LOCATION} \
100101
--env=AZURE_BACKEND_ENDPOINT=${AZURE_BACKEND_ENDPOINT} \
101102
--env=AZURE_BACKEND_QUEUE_ENDPOINT=${AZURE_BACKEND_QUEUE_ENDPOINT} \

‎.github/scripts/end2end/patch-coredns.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ corefile="
1414
rewrite name exact ci-zenko-aws-crr-target-bucket.aws-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
1515
rewrite name exact ci-zenko-aws-fail-target-bucket.aws-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
1616
rewrite name exact ci-zenko-aws-target-bucket.aws-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
17+
rewrite name exact ci-zenko-aws-replication-ctst-bucket.aws-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
1718
rewrite name exact aws-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
1819
rewrite name exact azure-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
1920
rewrite name exact blob.azure-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local

‎.github/scripts/end2end/run-e2e-ctst.sh

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,20 @@ SORBETD_RESTORE_TIMEOUT=$(kubectl get zenko ${ZENKO_NAME} -o jsonpath='{.spec.so
7878
UTILIZATION_SERVICE_HOST=$(kubectl get zenko ${ZENKO_NAME} -o jsonpath='{.spec.scuba.api.ingress.hostname}')
7979
UTILIZATION_SERVICE_PORT="80"
8080

81+
MONGODB_REPLICASET=$(kubectl get secrets -l app.kubernetes.io/name=connector-cloudserver-config -o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq -r .mongodb.replicaSetHosts)
82+
MONGODB_AUTH_USERNAME=$(kubectl get secrets -l app.kubernetes.io/name=connector-cloudserver-config -o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq -r .mongodb.authCredentials.username)
83+
MONGODB_AUTH_PASSWORD=$(kubectl get secrets -l app.kubernetes.io/name=connector-cloudserver-config -o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq -r .mongodb.authCredentials.password)
84+
MONGODB_DATABASE=$(kubectl get secrets -l app.kubernetes.io/name=connector-cloudserver-config -o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq -r .mongodb.database)
85+
86+
S3_UTILS_TAG=$(yq eval ".s3utils.tag" ../../../solution/deps.yaml)
87+
88+
LOCATION_CONFIGS=$(kubectl get secrets -l app.kubernetes.io/name=connector-cloudserver-config -o jsonpath='{.items[0].data.locationConfig\.json}' | base64 -di | jq -r .)
89+
AWS_REPLICATION_ENDPOINT="http://$(echo "$LOCATION_CONFIGS" | jq -r --arg loc "$AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION" '.[$loc].details.awsEndpoint')"
90+
AWS_REPLICATION_ACCESS_KEY=$(echo "$LOCATION_CONFIGS" | jq -r --arg loc "$AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION" '.[$loc].details.credentials.accessKey')
91+
AWS_REPLICATION_SECRET_KEY=$(echo "$LOCATION_CONFIGS" | jq -r --arg loc "$AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION" '.[$loc].details.credentials.secretKey')
92+
AWS_REPLICATION_LOCATION_TYPE=$(echo "$LOCATION_CONFIGS" | jq -r --arg loc "$AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION" '.[$loc].type')
93+
AWS_REPLICATION_REGION=$(echo "$LOCATION_CONFIGS" | jq -r --arg loc "$AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION" '.[$loc].details.region')
94+
8195
# Setting CTST world params
8296
WORLD_PARAMETERS="$(jq -c <<EOF
8397
{
@@ -128,7 +142,19 @@ WORLD_PARAMETERS="$(jq -c <<EOF
128142
"DRAdminAccessKey":"${DR_ADMIN_ACCESS_KEY_ID}",
129143
"DRAdminSecretKey":"${DR_ADMIN_SECRET_ACCESS_KEY}",
130144
"UtilizationServiceHost":"${UTILIZATION_SERVICE_HOST}",
131-
"UtilizationServicePort":"${UTILIZATION_SERVICE_PORT}"
145+
"UtilizationServicePort":"${UTILIZATION_SERVICE_PORT}",
146+
"AwsBackendDestinationReplicationLocation":"${AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION}",
147+
"AwsReplicationBucketName":"${AWS_REPLICATION_CTST_BUCKET_NAME}",
148+
"AwsReplicationEndpoint":"${AWS_REPLICATION_ENDPOINT}",
149+
"AwsReplicationAccessKey":"${AWS_REPLICATION_ACCESS_KEY}",
150+
"AwsReplicationSecretKey":"${AWS_REPLICATION_SECRET_KEY}",
151+
"AwsReplicationLocationType":"${AWS_REPLICATION_LOCATION_TYPE}",
152+
"AwsReplicationRegion":"${AWS_REPLICATION_REGION}",
153+
"MongodbReplicaSet":"${MONGODB_REPLICASET}",
154+
"MongodbAuthUsername":"${MONGODB_AUTH_USERNAME}",
155+
"MongodbAuthPassword":"${MONGODB_AUTH_PASSWORD}",
156+
"MongodbDatabase":"${MONGODB_DATABASE}",
157+
"S3UtilsTag":"${S3_UTILS_TAG}"
132158
}
133159
EOF
134160
)"

‎.github/workflows/end2end.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ env:
5757
AWS_BACKEND_SOURCE_LOCATION: awsbackend
5858
AWS_BACKEND_DESTINATION_LOCATION: awsbackendmismatch
5959
AWS_BACKEND_DESTINATION_FAIL_LOCATION: awsbackendfail
60+
AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION: awsbackendreplicationctst
6061
GCP_BACKEND_DESTINATION_LOCATION: gcpbackendmismatch
6162
AZURE_BACKEND_DESTINATION_LOCATION: azurebackendmismatch
6263
COLD_BACKEND_DESTINATION_LOCATION: e2e-cold
@@ -66,6 +67,7 @@ env:
6667
AWS_BUCKET_NAME: ci-zenko-aws-target-bucket
6768
AWS_CRR_BUCKET_NAME: ci-zenko-aws-crr-target-bucket
6869
AWS_FAIL_BUCKET_NAME: ci-zenko-aws-fail-target-bucket
70+
AWS_REPLICATION_CTST_BUCKET_NAME: ci-zenko-aws-replication-ctst-bucket
6971
AZURE_CRR_BUCKET_NAME: ci-zenko-azure-crr-target-bucket
7072
AZURE_ARCHIVE_BUCKET_NAME: ci-zenko-azure-archive-target-bucket
7173
AZURE_ARCHIVE_BUCKET_NAME_2: ci-zenko-azure-archive-target-bucket-2

‎tests/ctst/common/common.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import {
1515
verifyObjectLocation,
1616
restoreObject,
1717
addTransitionWorkflow,
18+
putBucketReplication,
1819
} from 'steps/utils/utils';
1920
import { ActionPermissionsType } from 'steps/bucket-policies/utils';
2021
import constants from './constants';
@@ -259,6 +260,11 @@ Given('a transition workflow to {string} location', async function (this: Zenko,
259260
await addTransitionWorkflow.call(this, location);
260261
});
261262

263+
Given('a replication configuration from {string} bucket to {string} location',
264+
async function (this: Zenko, srcBucket: string, replicationLocation: string) {
265+
await putBucketReplication.call(this, srcBucket, replicationLocation);
266+
});
267+
262268
When('i restore object {string} for {int} days', async function (this: Zenko, objectName: string, days: number) {
263269
await restoreObject.call(this, objectName, days);
264270
});

‎tests/ctst/features/replicatateExistingObjects.feature

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
Feature: Replicate existing objects created before enabling replication
2+
This feature tests the script crrExistingObjects.js from S3utils that replicates
3+
objects created before the replication was enabled on source the bucket.
4+
5+
@2.7.0
6+
@PreMerge
7+
@ReplicationTest
8+
Scenario Outline: Object created before adding the replication setting is replicated with the script
9+
Given an existing bucket "<sourceBucket>" "with" versioning, "without" ObjectLock "without" retention mode
10+
And an object "<sourceObject>" that "exists"
11+
And a replication configuration from "<sourceBucket>" bucket to "<replicationLocation>" location
12+
When I run the job to replicate existing objects with status "<sourceObjStatus>" from bucket "<sourceBucket>" to location "<replicationLocation>"
13+
Then the object "<sourceObject>" should be replicated within 60 seconds from "<sourceBucket>" bucket
14+
And the replicated object "<sourceObject>" from "<sourceBucket>" bucket to "<replicationBucket>" bucket with location "<replicationLocation>" should be the same as its source object
15+
16+
Examples:
17+
| sourceBucket | sourceObjStatus | replicationBucket | sourceObject | replicationLocation |
18+
| source-bucket | NEW | ci-zenko-aws-replication-ctst-bucket | source-object-1 | awsbackendreplicationctst |

‎tests/ctst/package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@
1818
"prometheus-query": "^3.4.0",
1919
"proper-lockfile": "^4.1.2",
2020
"qs": "^6.13.0",
21-
"scubaclient": "git+https://github.com/scality/scubaclient#^1.1.1"
21+
"scubaclient": "git+https://github.com/scality/scubaclient#^1.1.1",
22+
"uuid": "^11.1.0"
2223
},
2324
"devDependencies": {
2425
"@aws-sdk/client-iam": "^3.582.0",

‎tests/ctst/steps/replication.ts

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
import { When, Then } from '@cucumber/cucumber';
2+
import Zenko from '../world/Zenko';
3+
import { createAndRunS3UtilsPod } from 'steps/utils/kubernetes';
4+
import assert from 'assert';
5+
import { GetObjectCommand } from '@aws-sdk/client-s3';
6+
import { v4 as uuidv4 } from 'uuid';
7+
import { getObject, headObject, getReplicationLocationConfig } from 'steps/utils/utils';
8+
import { safeJsonParse } from 'common/utils';
9+
10+
When('I run the job to replicate existing objects with status {string} from bucket {string} to location {string}',
11+
{ timeout: 600000 },
12+
async function (
13+
this: Zenko,
14+
sourceObjectStatus: string,
15+
sourceBucket: string,
16+
replicationLocation: string,
17+
) {
18+
const { locationType } = getReplicationLocationConfig(this, replicationLocation);
19+
const podManifest = {
20+
apiVersion: 'v1',
21+
kind: 'Pod',
22+
metadata: {
23+
name: `s3utils-crr-existing-${uuidv4()}`,
24+
namespace: 'default',
25+
labels: {
26+
app: 's3utils',
27+
script: 'crrExistingObjects.js'
28+
}
29+
},
30+
spec: {
31+
restartPolicy: 'Never',
32+
containers: [
33+
{
34+
name: 's3utils',
35+
image: `ghcr.io/scality/s3utils:${this.parameters.S3UtilsTag}`,
36+
command: ['node'],
37+
args: ['crrExistingObjects.js', sourceBucket],
38+
env: [
39+
{ name: 'MONGODB_REPLICASET', value: this.parameters.MongodbReplicaSet },
40+
{ name: 'MONGODB_AUTH_USERNAME', value: this.parameters.MongodbAuthUsername },
41+
{ name: 'MONGODB_AUTH_PASSWORD', value: this.parameters.MongodbAuthPassword },
42+
{ name: 'MONGODB_DATABASE', value: this.parameters.MongodbDatabase },
43+
{ name: 'MONGODB_SHARD_COLLECTIONS', value: 'true' },
44+
{ name: 'STORAGE_TYPE', value: locationType },
45+
{ name: 'TARGET_REPLICATION_STATUS', value: sourceObjectStatus },
46+
{ name: 'SITE_NAME', value: replicationLocation },
47+
]
48+
}
49+
]
50+
}
51+
};
52+
53+
await createAndRunS3UtilsPod(this, podManifest);
54+
});
55+
56+
Then('the object {string} should be replicated within {int} seconds from {string} bucket',
57+
async function (
58+
this: Zenko,
59+
objectName: string,
60+
replicationTimeout: number,
61+
bucketSource: string,
62+
) {
63+
const startTime = Date.now();
64+
65+
while (Date.now() - startTime < replicationTimeout * 1000) {
66+
await new Promise(resolve => setTimeout(resolve, 3000));
67+
68+
const response = await headObject(this, objectName, bucketSource);
69+
assert(response.stdout);
70+
assert.strictEqual(response.statusCode, 200, `failed to headobject, ${response.statusCode}`);
71+
const parsed = safeJsonParse<{
72+
ReplicationStatus?: string;
73+
LastModified?: string;
74+
ETag?: string;
75+
ContentLength?: number;
76+
VersionId?: string;
77+
Metadata?: Record<string, string>;
78+
}>(response.stdout || '{}');
79+
assert(parsed.ok);
80+
const replicationStatus = parsed.result?.ReplicationStatus;
81+
assert.notStrictEqual(replicationStatus, 'FAILED', `replication failed for object ${objectName}`);
82+
if (replicationStatus === 'COMPLETED') {
83+
return;
84+
}
85+
if (replicationStatus === 'PENDING' || replicationStatus === 'PROCESSING') {
86+
continue;
87+
}
88+
}
89+
assert.fail(`Timeout: Object '${objectName}' was not replicated successfully until timeout`);
90+
});
91+
92+
Then(
93+
'the replicated object {string} from {string} bucket to {string} bucket ' +
94+
'with location {string} should be the same as its source object',
95+
async function (
96+
this: Zenko,
97+
objectName: string,
98+
bucketSource: string,
99+
bucketDestination: string,
100+
replicationLocation: string,
101+
) {
102+
const { storageClass, awsS3Client } = getReplicationLocationConfig(this, replicationLocation);
103+
const command = new GetObjectCommand({
104+
Bucket: bucketDestination,
105+
// bucketMatch is disabled on the destination bucket, so replicated objects
106+
// are named sourceBucket/objectName
107+
Key: `${bucketSource}/${objectName}`,
108+
});
109+
const replicaObj = await awsS3Client.send(command);
110+
const sourceResponse = await getObject(this, objectName, bucketSource);
111+
assert.strictEqual(sourceResponse.statusCode, 200, `failed to getObject, ${sourceResponse.statusCode}`);
112+
const sourceObj = safeJsonParse<{
113+
ReplicationStatus?: string;
114+
LastModified?: string;
115+
ETag?: string;
116+
ContentLength?: number;
117+
VersionId?: string;
118+
Metadata?: Record<string, string>;
119+
}>(sourceResponse.stdout || '{}');
120+
assert(sourceObj.ok);
121+
122+
assert.strictEqual(sourceObj.result?.ReplicationStatus, 'COMPLETED');
123+
assert.strictEqual(
124+
sourceObj.result?.ContentLength,
125+
replicaObj.ContentLength
126+
);
127+
assert.strictEqual(
128+
sourceObj.result?.Metadata?.[`${storageClass}-version-id`],
129+
replicaObj.VersionId
130+
);
131+
assert.strictEqual(
132+
sourceObj.result?.Metadata?.[`${storageClass}-replication-status`],
133+
'COMPLETED'
134+
);
135+
assert.strictEqual(
136+
sourceObj.result?.VersionId,
137+
replicaObj.Metadata?.['scal-version-id']
138+
);
139+
assert.strictEqual(
140+
replicaObj.Metadata?.['scal-replication-status'],
141+
'REPLICA'
142+
);
143+
});

‎tests/ctst/steps/utils/kubernetes.ts

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import {
1313
V1PersistentVolumeClaim,
1414
CoreV1Api,
1515
BatchV1Api,
16+
V1Pod,
1617
} from '@kubernetes/client-node';
1718

1819
type ZenkoStatusValue = {
@@ -165,6 +166,20 @@ export async function createJobAndWaitForCompletion(
165166
}
166167
}
167168

169+
export async function createAndRunS3UtilsPod(
170+
world: Zenko,
171+
podManifest: V1Pod,
172+
) {
173+
const clientCore = createKubeCoreClient(world);
174+
175+
try {
176+
const response = await clientCore.createNamespacedPod('default', podManifest);
177+
return response.body;
178+
} catch (err: unknown) {
179+
world.logger.error('Failed to create pod, API server response:', { err });
180+
throw new Error(`Failed to create pod: ${err}`);
181+
}
182+
}
168183

169184
export async function waitForZenkoToStabilize(
170185
world: Zenko, needsReconciliation = false, timeout = 15 * 60 * 1000, namespace = 'default') {

0 commit comments

Comments
 (0)