Skip to content

Commit f463777

Browse files
Merge pull request #13 from contentstack/staging
Staging
2 parents 2fb81db + c73162b commit f463777

File tree

7 files changed

+128
-12
lines changed

7 files changed

+128
-12
lines changed

.github/workflows/release.yml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@ jobs:
3333
token: ${{ secrets.NPM_TOKEN }}
3434
package: ./package.json
3535
access: public
36-
tag: beta
3736
- name: Create Release
3837
if: ${{ steps.publish-plugin.conclusion == 'success' }}
3938
id: create_release

.talismanrc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
fileignoreconfig:
2-
- filename: .env-example
3-
checksum: 591f1e672d4df287107092b8fd37c27913e09225c6ced55293e1d459b1119d05
2+
- filename: src/core/query-executor.ts
3+
checksum: 3d0b5cf07f5a87256f132f85a5556d193ce5a1fa6d92df2c7c50514071d592b7
44
version: '1.0'

package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"name": "@contentstack/cli-cm-export-query",
33
"description": "Contentstack CLI plugin to export content from stack",
4-
"version": "1.0.0-beta.2",
4+
"version": "1.0.0-beta.3",
55
"author": "Contentstack",
66
"bugs": "https://github.com/contentstack/cli/issues",
77
"dependencies": {

snyk_output.log

Whitespace-only changes.

src/core/query-executor.ts

Lines changed: 117 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -286,23 +286,132 @@ export class QueryExporter {
286286
log(this.exportQueryConfig, 'Starting export of referenced assets...', 'info');
287287

288288
try {
289+
const assetsDir = path.join(
290+
sanitizePath(this.exportQueryConfig.exportDir),
291+
sanitizePath(this.exportQueryConfig.branchName || ''),
292+
'assets',
293+
);
294+
295+
const metadataFilePath = path.join(assetsDir, 'metadata.json');
296+
const assetFilePath = path.join(assetsDir, 'assets.json');
297+
298+
// Define temp file paths
299+
const tempMetadataFilePath = path.join(assetsDir, 'metadata_temp.json');
300+
const tempAssetFilePath = path.join(assetsDir, 'assets_temp.json');
301+
289302
const assetHandler = new AssetReferenceHandler(this.exportQueryConfig);
290303

291304
// Extract referenced asset UIDs from all entries
292305
const assetUIDs = assetHandler.extractReferencedAssets();
293306

294307
if (assetUIDs.length > 0) {
295-
log(this.exportQueryConfig, `Exporting ${assetUIDs.length} referenced assets...`, 'info');
308+
log(this.exportQueryConfig, `Found ${assetUIDs.length} referenced assets to export`, 'info');
296309

297-
const query = {
298-
modules: {
299-
assets: {
300-
uid: { $in: assetUIDs },
310+
// Define batch size - can be configurable through exportQueryConfig
311+
const batchSize = this.exportQueryConfig.assetBatchSize || 100;
312+
313+
if (assetUIDs.length <= batchSize) {
314+
const query = {
315+
modules: {
316+
assets: {
317+
uid: { $in: assetUIDs },
318+
},
301319
},
302-
},
303-
};
320+
};
321+
322+
await this.moduleExporter.exportModule('assets', { query });
323+
}
324+
325+
// if asset size is bigger than batch size, then we need to export in batches
326+
// Calculate number of batches
327+
const totalBatches = Math.ceil(assetUIDs.length / batchSize);
328+
log(this.exportQueryConfig, `Processing assets in ${totalBatches} batches of ${batchSize}`, 'info');
329+
330+
// Process assets in batches
331+
for (let i = 0; i < totalBatches; i++) {
332+
const start = i * batchSize;
333+
const end = Math.min(start + batchSize, assetUIDs.length);
334+
const batchAssetUIDs = assetUIDs.slice(start, end);
335+
336+
log(
337+
this.exportQueryConfig,
338+
`Exporting batch ${i + 1}/${totalBatches} (${batchAssetUIDs.length} assets)...`,
339+
'info',
340+
);
341+
342+
const query = {
343+
modules: {
344+
assets: {
345+
uid: { $in: batchAssetUIDs },
346+
},
347+
},
348+
};
349+
350+
await this.moduleExporter.exportModule('assets', { query });
351+
352+
// Read the current batch's metadata.json and assets.json files
353+
const currentMetadata: any = fsUtil.readFile(sanitizePath(metadataFilePath));
354+
const currentAssets: any = fsUtil.readFile(sanitizePath(assetFilePath));
355+
356+
// Check if this is the first batch
357+
if (i === 0) {
358+
// For first batch, initialize temp files with current content
359+
fsUtil.writeFile(sanitizePath(tempMetadataFilePath), currentMetadata);
360+
fsUtil.writeFile(sanitizePath(tempAssetFilePath), currentAssets);
361+
log(this.exportQueryConfig, `Initialized temporary files with first batch data`, 'info');
362+
} else {
363+
// For subsequent batches, append to temp files with incremented keys
364+
365+
// Handle metadata (which contains arrays of asset info)
366+
const tempMetadata: any = fsUtil.readFile(sanitizePath(tempMetadataFilePath)) || {};
367+
368+
// Merge metadata by combining arrays
369+
if (currentMetadata) {
370+
Object.keys(currentMetadata).forEach((key: string) => {
371+
if (!tempMetadata[key]) {
372+
tempMetadata[key] = currentMetadata[key];
373+
}
374+
});
375+
}
376+
377+
// Write updated metadata back to temp file
378+
fsUtil.writeFile(sanitizePath(tempMetadataFilePath), tempMetadata);
379+
380+
// Handle assets (which is an object with numeric keys)
381+
const tempAssets: any = fsUtil.readFile(sanitizePath(tempAssetFilePath)) || {};
382+
let nextIndex = Object.keys(tempAssets).length + 1;
383+
384+
// Add current assets with incremented keys
385+
Object.values(currentAssets).forEach((value: any) => {
386+
tempAssets[nextIndex.toString()] = value;
387+
nextIndex++;
388+
});
389+
390+
fsUtil.writeFile(sanitizePath(tempAssetFilePath), tempAssets);
391+
392+
log(this.exportQueryConfig, `Updated temporary files with batch ${i + 1} data`, 'info');
393+
}
394+
395+
// Optional: Add delay between batches to avoid rate limiting
396+
if (i < totalBatches - 1 && this.exportQueryConfig.batchDelayMs) {
397+
await new Promise((resolve) => setTimeout(resolve, this.exportQueryConfig.batchDelayMs));
398+
}
399+
}
400+
401+
// After all batches are processed, copy temp files back to original files
402+
const finalMetadata = fsUtil.readFile(sanitizePath(tempMetadataFilePath));
403+
const finalAssets = fsUtil.readFile(sanitizePath(tempAssetFilePath));
404+
405+
fsUtil.writeFile(sanitizePath(metadataFilePath), finalMetadata);
406+
fsUtil.writeFile(sanitizePath(assetFilePath), finalAssets);
407+
408+
log(this.exportQueryConfig, `Final data written back to original files`, 'info');
409+
410+
// Clean up temp files
411+
fsUtil.removeFile(sanitizePath(tempMetadataFilePath));
412+
fsUtil.removeFile(sanitizePath(tempAssetFilePath));
304413

305-
await this.moduleExporter.exportModule('assets', { query });
414+
log(this.exportQueryConfig, `Temporary files cleaned up`, 'info');
306415
log(this.exportQueryConfig, 'Referenced assets exported successfully', 'success');
307416
} else {
308417
log(this.exportQueryConfig, 'No referenced assets found in entries', 'info');

src/types/index.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,9 @@ export interface QueryExportConfig extends DefaultConfig {
185185
logsPath: string;
186186
dataPath: string;
187187
exportDelayMs?: number;
188+
batchDelayMs?: number;
189+
assetBatchSize?: number;
190+
assetBatchDelayMs?: number;
188191
}
189192

190193
export interface QueryMetadata {

src/utils/config-handler.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,11 @@ export async function setupQueryExportConfig(flags: any): Promise<QueryExportCon
2424
externalConfigPath: path.join(__dirname, '../config/export-config.json'),
2525
};
2626

27+
// override the external config path if the user provides a config file
28+
if (flags.config) {
29+
exportQueryConfig.externalConfigPath = sanitizePath(flags['config']);
30+
}
31+
2732
// Handle authentication
2833
if (flags.alias) {
2934
const { token, apiKey } = configHandler.get(`tokens.${flags.alias}`) || {};

0 commit comments

Comments
 (0)