@@ -286,23 +286,132 @@ export class QueryExporter {
286
286
log ( this . exportQueryConfig , 'Starting export of referenced assets...' , 'info' ) ;
287
287
288
288
try {
289
+ const assetsDir = path . join (
290
+ sanitizePath ( this . exportQueryConfig . exportDir ) ,
291
+ sanitizePath ( this . exportQueryConfig . branchName || '' ) ,
292
+ 'assets' ,
293
+ ) ;
294
+
295
+ const metadataFilePath = path . join ( assetsDir , 'metadata.json' ) ;
296
+ const assetFilePath = path . join ( assetsDir , 'assets.json' ) ;
297
+
298
+ // Define temp file paths
299
+ const tempMetadataFilePath = path . join ( assetsDir , 'metadata_temp.json' ) ;
300
+ const tempAssetFilePath = path . join ( assetsDir , 'assets_temp.json' ) ;
301
+
289
302
const assetHandler = new AssetReferenceHandler ( this . exportQueryConfig ) ;
290
303
291
304
// Extract referenced asset UIDs from all entries
292
305
const assetUIDs = assetHandler . extractReferencedAssets ( ) ;
293
306
294
307
if ( assetUIDs . length > 0 ) {
295
- log ( this . exportQueryConfig , `Exporting ${ assetUIDs . length } referenced assets... ` , 'info' ) ;
308
+ log ( this . exportQueryConfig , `Found ${ assetUIDs . length } referenced assets to export ` , 'info' ) ;
296
309
297
- const query = {
298
- modules : {
299
- assets : {
300
- uid : { $in : assetUIDs } ,
310
+ // Define batch size - can be configurable through exportQueryConfig
311
+ const batchSize = this . exportQueryConfig . assetBatchSize || 100 ;
312
+
313
+ if ( assetUIDs . length <= batchSize ) {
314
+ const query = {
315
+ modules : {
316
+ assets : {
317
+ uid : { $in : assetUIDs } ,
318
+ } ,
301
319
} ,
302
- } ,
303
- } ;
320
+ } ;
321
+
322
+ await this . moduleExporter . exportModule ( 'assets' , { query } ) ;
323
+ }
324
+
325
+ // if asset size is bigger than batch size, then we need to export in batches
326
+ // Calculate number of batches
327
+ const totalBatches = Math . ceil ( assetUIDs . length / batchSize ) ;
328
+ log ( this . exportQueryConfig , `Processing assets in ${ totalBatches } batches of ${ batchSize } ` , 'info' ) ;
329
+
330
+ // Process assets in batches
331
+ for ( let i = 0 ; i < totalBatches ; i ++ ) {
332
+ const start = i * batchSize ;
333
+ const end = Math . min ( start + batchSize , assetUIDs . length ) ;
334
+ const batchAssetUIDs = assetUIDs . slice ( start , end ) ;
335
+
336
+ log (
337
+ this . exportQueryConfig ,
338
+ `Exporting batch ${ i + 1 } /${ totalBatches } (${ batchAssetUIDs . length } assets)...` ,
339
+ 'info' ,
340
+ ) ;
341
+
342
+ const query = {
343
+ modules : {
344
+ assets : {
345
+ uid : { $in : batchAssetUIDs } ,
346
+ } ,
347
+ } ,
348
+ } ;
349
+
350
+ await this . moduleExporter . exportModule ( 'assets' , { query } ) ;
351
+
352
+ // Read the current batch's metadata.json and assets.json files
353
+ const currentMetadata : any = fsUtil . readFile ( sanitizePath ( metadataFilePath ) ) ;
354
+ const currentAssets : any = fsUtil . readFile ( sanitizePath ( assetFilePath ) ) ;
355
+
356
+ // Check if this is the first batch
357
+ if ( i === 0 ) {
358
+ // For first batch, initialize temp files with current content
359
+ fsUtil . writeFile ( sanitizePath ( tempMetadataFilePath ) , currentMetadata ) ;
360
+ fsUtil . writeFile ( sanitizePath ( tempAssetFilePath ) , currentAssets ) ;
361
+ log ( this . exportQueryConfig , `Initialized temporary files with first batch data` , 'info' ) ;
362
+ } else {
363
+ // For subsequent batches, append to temp files with incremented keys
364
+
365
+ // Handle metadata (which contains arrays of asset info)
366
+ const tempMetadata : any = fsUtil . readFile ( sanitizePath ( tempMetadataFilePath ) ) || { } ;
367
+
368
+ // Merge metadata by combining arrays
369
+ if ( currentMetadata ) {
370
+ Object . keys ( currentMetadata ) . forEach ( ( key : string ) => {
371
+ if ( ! tempMetadata [ key ] ) {
372
+ tempMetadata [ key ] = currentMetadata [ key ] ;
373
+ }
374
+ } ) ;
375
+ }
376
+
377
+ // Write updated metadata back to temp file
378
+ fsUtil . writeFile ( sanitizePath ( tempMetadataFilePath ) , tempMetadata ) ;
379
+
380
+ // Handle assets (which is an object with numeric keys)
381
+ const tempAssets : any = fsUtil . readFile ( sanitizePath ( tempAssetFilePath ) ) || { } ;
382
+ let nextIndex = Object . keys ( tempAssets ) . length + 1 ;
383
+
384
+ // Add current assets with incremented keys
385
+ Object . values ( currentAssets ) . forEach ( ( value : any ) => {
386
+ tempAssets [ nextIndex . toString ( ) ] = value ;
387
+ nextIndex ++ ;
388
+ } ) ;
389
+
390
+ fsUtil . writeFile ( sanitizePath ( tempAssetFilePath ) , tempAssets ) ;
391
+
392
+ log ( this . exportQueryConfig , `Updated temporary files with batch ${ i + 1 } data` , 'info' ) ;
393
+ }
394
+
395
+ // Optional: Add delay between batches to avoid rate limiting
396
+ if ( i < totalBatches - 1 && this . exportQueryConfig . batchDelayMs ) {
397
+ await new Promise ( ( resolve ) => setTimeout ( resolve , this . exportQueryConfig . batchDelayMs ) ) ;
398
+ }
399
+ }
400
+
401
+ // After all batches are processed, copy temp files back to original files
402
+ const finalMetadata = fsUtil . readFile ( sanitizePath ( tempMetadataFilePath ) ) ;
403
+ const finalAssets = fsUtil . readFile ( sanitizePath ( tempAssetFilePath ) ) ;
404
+
405
+ fsUtil . writeFile ( sanitizePath ( metadataFilePath ) , finalMetadata ) ;
406
+ fsUtil . writeFile ( sanitizePath ( assetFilePath ) , finalAssets ) ;
407
+
408
+ log ( this . exportQueryConfig , `Final data written back to original files` , 'info' ) ;
409
+
410
+ // Clean up temp files
411
+ fsUtil . removeFile ( sanitizePath ( tempMetadataFilePath ) ) ;
412
+ fsUtil . removeFile ( sanitizePath ( tempAssetFilePath ) ) ;
304
413
305
- await this . moduleExporter . exportModule ( 'assets' , { query } ) ;
414
+ log ( this . exportQueryConfig , `Temporary files cleaned up` , 'info' ) ;
306
415
log ( this . exportQueryConfig , 'Referenced assets exported successfully' , 'success' ) ;
307
416
} else {
308
417
log ( this . exportQueryConfig , 'No referenced assets found in entries' , 'info' ) ;
0 commit comments