feat: Create uploads module

Author: Blckbrry-Pi

modules/uploads/config.ts

@@ -0,0 +1,19 @@
+export interface Config {
+	maxUploadSize: UploadSize;
+	maxFilesPerUpload: number;
+	s3: S3Config;
+}
+
+type Units = "b" | "kb" | "mb" | "gb" | "tb" | "kib" | "mib" | "gib" | "tib";
+
+export type UploadSize = {
+	[unit in Units]: Record<unit, number>;
+}[Units];
+
+export interface S3Config {
+	bucket: string;
+	region: string;
+	accessKeyId: string;
+	secretAccessKey: string;
+	endpoint: string;
+}

modules/uploads/db/migrations/20240328220608_create_uploads_module/migration.sql

@@ -0,0 +1,27 @@
+-- CreateTable
+CREATE TABLE "Upload" (
+    "id" UUID NOT NULL,
+    "userId" UUID,
+    "bucket" TEXT NOT NULL,
+    "contentLength" BIGINT NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+    "completedAt" TIMESTAMP(3),
+    "deletedAt" TIMESTAMP(3),
+
+    CONSTRAINT "Upload_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "Files" (
+    "path" TEXT NOT NULL,
+    "mime" TEXT,
+    "contentLength" BIGINT NOT NULL,
+    "nsfwScoreThreshold" DOUBLE PRECISION,
+    "uploadId" UUID NOT NULL,
+
+    CONSTRAINT "Files_pkey" PRIMARY KEY ("uploadId","path")
+);
+
+-- AddForeignKey
+ALTER TABLE "Files" ADD CONSTRAINT "Files_uploadId_fkey" FOREIGN KEY ("uploadId") REFERENCES "Upload"("id") ON DELETE RESTRICT ON UPDATE CASCADE;

modules/uploads/db/migrations/migration_lock.toml

@@ -0,0 +1,3 @@
+# Please do not edit this file manually
+# It should be added in your version-control system (i.e. Git)
+provider = "postgresql"

modules/uploads/db/schema.prisma

@@ -0,0 +1,32 @@
+// Do not modify this `datasource` block
+datasource db {
+	provider = "postgresql"
+	url      = env("DATABASE_URL")
+}
+
+model Upload {
+	id						String		@id @default(uuid()) @db.Uuid
+	userId					String?		@db.Uuid
+
+	bucket					String
+	contentLength			BigInt
+
+	createdAt				DateTime	@default(now())
+	updatedAt				DateTime	@updatedAt
+	completedAt				DateTime?
+	deletedAt				DateTime?
+
+	files					Files[]		@relation("Files")
+}
+
+model Files {
+	path					String
+	mime					String?
+	contentLength			BigInt
+	nsfwScoreThreshold		Float?
+
+	uploadId				String		@db.Uuid
+	upload					Upload		@relation("Files", fields: [uploadId], references: [id])
+
+	@@id([uploadId, path])
+}

modules/uploads/module.yaml

@@ -0,0 +1,40 @@
+scripts:
+  prepare:
+    name: Prepare Upload
+    description: Prepare an upload batch for data transfer
+  complete:
+    name: Complete Upload
+    description: Alert the module that the upload has been completed
+  get:
+    name: Get Upload Metadata
+    description: Get the metadata (including contained files) for specified upload IDs
+  get_file_links:
+    name: Get File Link
+    description: Get presigned download links for each of the specified files
+  list_for_user:
+    name: List Uploads for Users
+    description: Get a list of upload IDs associated with the specified user IDs
+  delete:
+    name: Delete Upload
+    description: Removes the upload and deletes the files from the bucket
+errors:
+  no_files:
+    name: No Files Provided
+    description: An upload must have at least 1 file
+  too_many_files:
+    name: Too Many Files Provided
+    description: There is a limit to how many files can be put into a single upload (see config)
+  duplicate_paths:
+    name: Duplicate Paths Provided
+    description: An upload cannot contain 2 files with the same paths (see `cause` for offending paths)
+  size_limit_exceeded:
+    name: Combined Size Limit Exceeded
+    description: There is a maximum total size per upload (see config)
+  upload_not_found:
+    name: Upload Not Found
+    description: The provided upload ID didn't match any known existing uploads
+  upload_already_completed:
+    name: Upload Already completed
+    description: \`complete\` was already called on this upload
+dependencies:
+  users: {}

modules/uploads/scripts/complete.ts

@@ -0,0 +1,77 @@
+import { RuntimeError, ScriptContext } from "../_gen/scripts/complete.ts";
+import { prismaToOutput } from "../utils/types.ts";
+import { Upload } from "../utils/types.ts";
+
+export interface Request {
+	uploadId: string;
+}
+
+export interface Response {
+	upload: Upload;
+}
+
+export async function run(
+	ctx: ScriptContext,
+	req: Request,
+): Promise<Response> {
+	const newUpload = await ctx.db.$transaction(async (db) => {
+		// Find the upload by ID
+		const upload = await db.upload.findFirst({
+			where: {
+				id: req.uploadId,
+			},
+			select: {
+				id: true,
+				userId: true,
+				bucket: true,
+				contentLength: true,
+				files: true,
+				createdAt: true,
+				updatedAt: true,
+				completedAt: true,
+			},
+		});
+
+		// Error if the upload wasn't prepared
+		if (!upload) {
+			throw new RuntimeError(
+				"upload_not_found",
+				{ cause: `Upload with ID ${req.uploadId} not found` },
+			);
+		}
+
+		// Error if `complete` was already called with this ID
+		if (upload.completedAt !== null) {
+			throw new RuntimeError(
+				"upload_already_completed",
+				{ cause: `Upload with ID ${req.uploadId} has already been completed` },
+			);
+		}
+
+		// Update the upload to mark it as completed
+		const completedUpload = await db.upload.update({
+			where: {
+				id: req.uploadId,
+			},
+			data: {
+				completedAt: new Date().toISOString(),
+			},
+			select: {
+				id: true,
+				userId: true,
+				bucket: true,
+				contentLength: true,
+				files: true,
+				createdAt: true,
+				updatedAt: true,
+				completedAt: true,
+			},
+		});
+
+		return completedUpload;
+	});
+
+	return {
+		upload: prismaToOutput(newUpload),
+	};
+}

modules/uploads/scripts/delete.ts

@@ -0,0 +1,78 @@
+import { RuntimeError, ScriptContext } from "../_gen/scripts/prepare.ts";
+import { getKey } from "../utils/types.ts";
+import { deleteKeys } from "../utils/bucket.ts";
+
+export interface Request {
+	uploadId: string;
+}
+
+export interface Response {
+	bytesDeleted: string;
+}
+
+export async function run(
+	ctx: ScriptContext,
+	req: Request,
+): Promise<Response> {
+	const bytesDeleted = await ctx.db.$transaction(async (db) => {
+		const upload = await db.upload.findFirst({
+			where: {
+				id: req.uploadId,
+				completedAt: { not: null },
+				deletedAt: null,
+			},
+			select: {
+				id: true,
+				userId: true,
+				bucket: true,
+				contentLength: true,
+				files: true,
+				createdAt: true,
+				updatedAt: true,
+				completedAt: true,
+			},
+		});
+		if (!upload) {
+			throw new RuntimeError(
+				"upload_not_found",
+				{
+					cause: `Upload with ID ${req.uploadId} not found`,
+					meta: { modified: false },
+				},
+			);
+		}
+
+		const filesToDelete = upload.files.map((file) =>
+			getKey(file.uploadId, file.path)
+		);
+		const deleteResults = await deleteKeys(ctx.userConfig.s3, filesToDelete);
+
+		const failures = upload.files
+			.map((file, i) => [file, deleteResults[i]] as const)
+			.filter(([, successfullyDeleted]) => !successfullyDeleted)
+			.map(([file]) => file);
+
+		if (failures.length) {
+			const failedPaths = JSON.stringify(failures.map((file) => file.path));
+			throw new RuntimeError(
+				"failed_to_delete",
+				{
+					cause: `Failed to delete files with paths ${failedPaths}`,
+					meta: { modified: failures.length !== filesToDelete.length },
+				},
+			);
+		}
+
+		await db.upload.update({
+			where: {
+				id: req.uploadId,
+			},
+			data: {
+				deletedAt: new Date().toISOString(),
+			},
+		});
+
+		return upload.contentLength.toString();
+	});
+	return { bytesDeleted };
+}

modules/uploads/scripts/get.ts

@@ -0,0 +1,49 @@
+import { ScriptContext } from "../_gen/scripts/get.ts";
+import { prismaToOutput } from "../utils/types.ts";
+import { Upload } from "../utils/types.ts";
+
+export interface Request {
+	uploadIds: string[];
+}
+
+export interface Response {
+	uploads: (Upload | null)[];
+}
+
+export async function run(
+	ctx: ScriptContext,
+	req: Request,
+): Promise<Response> {
+	// Find uploads that match the IDs in the request
+	const dbUploads = await ctx.db.upload.findMany({
+		where: {
+			id: {
+				in: req.uploadIds,
+			},
+			completedAt: { not: null },
+			deletedAt: null,
+		},
+		select: {
+			id: true,
+			userId: true,
+			bucket: true,
+			contentLength: true,
+			files: true,
+			createdAt: true,
+			updatedAt: true,
+			completedAt: true,
+		},
+	});
+
+	// Create a map of uploads by ID
+	const uploadMap = new Map(dbUploads.map((upload) => [upload.id, upload]));
+
+	// Reorder uploads to match the order of the request
+	const uploads = req.uploadIds.map((uploadId) => {
+		const upload = uploadMap.get(uploadId);
+		// If the upload wasn't found, return null
+		return upload ? prismaToOutput(upload) : null;
+	});
+
+	return { uploads };
+}