Improve, xoxo

Author: jleeothon

dups.js

@@ -29,37 +29,55 @@ function minDate(file) {
 async function run() {
 	const folderId = Number.parseInt(process.env.FOLDER_ID, 10);
 	const allFiles = await client.listfolder(folderId, {recursive: true});
-	const goodFolders = allFiles.contents.filter(f => f.name.match(/^[\w-]+ \d+$/));
+	const goodFolders = allFiles.contents.filter((f) =>
+		f.name.match(/^[\w-]+ \d+$/)
+	);
 
-	const dupFilesPerFolderUnfiltered = await pMap(goodFolders, async folder => {
-		const name = folder.name;
-		const imageFiles = folder.contents.filter(file => file.contenttype === 'image/jpeg');
-		const hashToFilesMap = groupByHash(imageFiles);
-		const fileGroups = [...hashToFilesMap.values()].filter(files => files.length > 1);
-		fileGroups.sort((a, b) => {
-			return minDate(a) - minDate(b);
-		});
-		return {name, fileGroups};
-	});
+	const dupFilesPerFolderUnfiltered = await pMap(
+		goodFolders,
+		async (folder) => {
+			const name = folder.name;
+			const imageFiles = folder.contents.filter(
+				(file) => file.contenttype === 'image/jpeg'
+			);
+			const hashToFilesMap = groupByHash(imageFiles);
+			const fileGroups = [...hashToFilesMap.values()].filter(
+				(files) => files.length > 1
+			);
+			fileGroups.sort((a, b) => {
+				return minDate(a) - minDate(b);
+			});
+			return {name, fileGroups};
+		}
+	);
 
-	const dupFilesPerFolder = dupFilesPerFolderUnfiltered.filter(({fileGroups}) => fileGroups.length);
+	const dupFilesPerFolder = dupFilesPerFolderUnfiltered.filter(
+		({fileGroups}) => fileGroups.length
+	);
 
-	const lineFormat = f => `${f.name} (${new Date(minDate(f)).toISOString()})`;
+	const deletion = dupFilesPerFolder.map(({name, fileGroups}) => {
+		const keepDelGroups = fileGroups.map((g) => ({
+			keep: g[0],
+			del: g.slice(1)
+		}));
+		return {name, keepDelGroups};
+	});
 
-	// Delete dups
-	await pMap(dupFilesPerFolder, async ({name, fileGroups}) => {
-		console.log(name);
-		await pMap(fileGroups, async group => {
-			const first = group[0];
-			console.log(`  Keep ${lineFormat(first)}`);
-			await pMap(group.slice(1), file => {
-				console.log(`    Del ${lineFormat(file)}`);
-				client.deletefile(file.fileid);
-			});
-		}, {concurrency: 1});
-	}, {concurrency: 1});
+	console.log(JSON.stringify(deletion, null, 4));
+
+	const allToDelete = deletion
+		.flatMap(({keepDelGroups}) => keepDelGroups.map(({del}) => del))
+		.flat();
+	await pMap(
+		allToDelete,
+		async (file) => {
+			console.log(`Deleting ${file.name}`);
+			await client.deletefile(file.fileid);
+		},
+		{concurrency: 10}
+	);
 }
 
-run().catch(error => {
+run().catch((error) => {
 	console.error(error);
 });

intersects.js

@@ -16,14 +16,14 @@ function intersects(a, b) {
 function mergeNames(hashesPerName) {
 	const result = [];
 	for (const {name, hashes} of hashesPerName) {
-		const resultItem = result.find(i => intersects(hashes, i.hashUnion));
+		const resultItem = result.find((i) => intersects(hashes, i.hashUnion));
 		if (!resultItem) {
 			result.push({names: [name], hashUnion: hashes});
 			continue;
 		}
 
 		resultItem.names.push(name);
-		hashes.forEach(h => resultItem.hashUnion.add(h));
+		hashes.forEach((h) => resultItem.hashUnion.add(h));
 	}
 
 	return result;
@@ -32,25 +32,31 @@ function mergeNames(hashesPerName) {
 async function run() {
 	const folderId = Number.parseInt(process.env.FOLDER_ID, 10);
 	const allFiles = await client.listfolder(folderId, {recursive: true});
-	const goodFolders = allFiles.contents.filter(f => f.name.match(/^[\w-]+ \d+$/));
+	const goodFolders = allFiles.contents.filter((f) =>
+		f.name.match(/^[\w-]+ \d+$/)
+	);
 
-	const hashesPerName = await pMap(goodFolders, async f => {
+	const hashesPerName = await pMap(goodFolders, async (f) => {
 		const name = f.name;
-		const hashList = f.contents.filter(f => f.contenttype === 'image/jpeg').map(f => f.hash);
+		const hashList = f.contents
+			.filter((f) => f.contenttype === 'image/jpeg')
+			.map((f) => f.hash);
 		const hashes = new Set(hashList);
 		return {name, hashes};
 	});
 
 	// [{names: [], hashUnion: Set}, ...]
 	const nonSingleMergedNames = mergeNames(hashesPerName);
-	const mergedNames = nonSingleMergedNames.filter(({names}) => names.length > 1);
+	const mergedNames = nonSingleMergedNames.filter(
+		({names}) => names.length > 1
+	);
 
 	// [[name1, name2, name3], ...]
 	const onlyNames = mergedNames.map(({names}) => names);
 
 	console.log(onlyNames);
 }
 
-run().catch(error => {
+run().catch((error) => {
 	console.error(error);
 });

package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "description": "",
   "scripts": {
-    "lint": "xo --pretier"
+    "lint": "xo --prettier"
   },
   "author": "",
   "license": "ISC",