Skip to content
This repository was archived by the owner on Sep 17, 2024. It is now read-only.

Commit 2efca85

Browse files
Blckbrry-PiBlckbrry-Pi
committed
feat: Create uploads module
1 parent b5c0823 commit 2efca85

File tree

18 files changed

+1352
-0
lines changed

18 files changed

+1352
-0
lines changed

‎modules/uploads/config.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
import { UploadSize } from "./utils/data_size.ts";
2+
3+
export interface Config {
4+
maxUploadSize: UploadSize;
5+
maxMultipartUploadSize: UploadSize;
6+
maxFilesPerUpload?: number;
7+
defaultMultipartChunkSize?: UploadSize;
8+
9+
s3: {
10+
bucketName: string;
11+
region: string;
12+
endpoint: string;
13+
};
14+
}
15+
16+
export const DEFAULT_MAX_FILES_PER_UPLOAD = 10;
17+
18+
export const DEFAULT_MAX_UPLOAD_SIZE: UploadSize = "30mib";
19+
export const DEFAULT_MAX_MULTIPART_UPLOAD_SIZE: UploadSize = "10gib";
20+
export const DEFAULT_MULTIPART_CHUNK_SIZE: UploadSize = "10mib";

‎modules/uploads/db/migrations/20240419133543_initial_setup/migration.sql‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
-- CreateTable
2+
CREATE TABLE "Upload" (
3+
"id" UUID NOT NULL,
4+
"userId" UUID,
5+
"bucket" TEXT NOT NULL,
6+
"contentLength" BIGINT NOT NULL,
7+
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
8+
"updatedAt" TIMESTAMP(3) NOT NULL,
9+
"completedAt" TIMESTAMP(3),
10+
"deletedAt" TIMESTAMP(3),
11+
12+
CONSTRAINT "Upload_pkey" PRIMARY KEY ("id")
13+
);
14+
15+
-- CreateTable
16+
CREATE TABLE "Files" (
17+
"uploadId" UUID NOT NULL,
18+
"multipartUploadId" TEXT,
19+
"path" TEXT NOT NULL,
20+
"mime" TEXT,
21+
"contentLength" BIGINT NOT NULL,
22+
23+
CONSTRAINT "Files_pkey" PRIMARY KEY ("uploadId","path")
24+
);
25+
26+
-- AddForeignKey
27+
ALTER TABLE "Files" ADD CONSTRAINT "Files_uploadId_fkey" FOREIGN KEY ("uploadId") REFERENCES "Upload"("id") ON DELETE RESTRICT ON UPDATE CASCADE;

‎modules/uploads/db/migrations/migration_lock.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Please do not edit this file manually
2+
# It should be added in your version-control system (i.e. Git)
3+
provider = "postgresql"

‎modules/uploads/db/schema.prisma‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
// Do not modify this `datasource` block
2+
datasource db {
3+
provider = "postgresql"
4+
url = env("DATABASE_URL")
5+
}
6+
7+
model Upload {
8+
id String @id @default(uuid()) @db.Uuid
9+
userId String? @db.Uuid
10+
11+
bucket String
12+
contentLength BigInt
13+
14+
createdAt DateTime @default(now())
15+
updatedAt DateTime @updatedAt
16+
completedAt DateTime?
17+
deletedAt DateTime?
18+
19+
files Files[] @relation("Files")
20+
}
21+
22+
model Files {
23+
uploadId String @db.Uuid
24+
upload Upload @relation("Files", fields: [uploadId], references: [id])
25+
26+
multipartUploadId String?
27+
28+
path String
29+
mime String?
30+
contentLength BigInt
31+
32+
@@id([uploadId, path])
33+
}

‎modules/uploads/module.yaml‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
scripts:
2+
prepare:
3+
name: Prepare Upload
4+
description: Prepare an upload batch for data transfer
5+
complete:
6+
name: Complete Upload
7+
description: Alert the module that the upload has been completed
8+
get:
9+
name: Get Upload Metadata
10+
description: Get the metadata (including contained files) for specified upload IDs
11+
get_public_file_urls:
12+
name: Get File Link
13+
description: Get presigned download links for each of the specified files
14+
list_for_user:
15+
name: List Uploads for Users
16+
description: Get a list of upload IDs associated with the specified user IDs
17+
delete:
18+
name: Delete Upload
19+
description: Removes the upload and deletes the files from the bucket
20+
errors:
21+
no_files:
22+
name: No Files Provided
23+
description: An upload must have at least 1 file
24+
too_many_files:
25+
name: Too Many Files Provided
26+
description: There is a limit to how many files can be put into a single upload (see config)
27+
duplicate_paths:
28+
name: Duplicate Paths Provided
29+
description: An upload cannot contain 2 files with the same paths (see `cause` for offending paths)
30+
size_limit_exceeded:
31+
name: Combined Size Limit Exceeded
32+
description: There is a maximum total size per upload (see config)
33+
upload_not_found:
34+
name: Upload Not Found
35+
description: The provided upload ID didn't match any known existing uploads
36+
upload_already_completed:
37+
name: Upload Already completed
38+
description: \`complete\` was already called on this upload
39+
s3_not_configured:
40+
name: S3 Not Configured
41+
description: The S3 bucket is not configured (missing env variables)
42+
multipart_upload_completion_fail:
43+
name: Multipart Upload Completion Failure
44+
description: The multipart upload failed to complete (see `cause` for more information)
45+
dependencies: {}

‎modules/uploads/scripts/complete.ts‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
import { RuntimeError, ScriptContext } from "../_gen/scripts/complete.ts";
2+
import { keyExists, getMultipartUploadParts, completeMultipartUpload } from "../utils/bucket.ts";
3+
import { getS3EnvConfig } from "../utils/env.ts";
4+
import { getKey } from "../utils/types.ts";
5+
import { prismaToOutput } from "../utils/types.ts";
6+
import { Upload } from "../utils/types.ts";
7+
8+
export interface Request {
9+
uploadId: string;
10+
}
11+
12+
export interface Response {
13+
upload: Upload;
14+
}
15+
16+
export async function run(
17+
ctx: ScriptContext,
18+
req: Request,
19+
): Promise<Response> {
20+
const s3 = getS3EnvConfig(ctx.userConfig.s3);
21+
if (!s3) throw new RuntimeError("s3_not_configured");
22+
23+
const newUpload = await ctx.db.$transaction(async (db) => {
24+
// Find the upload by ID
25+
const upload = await db.upload.findFirst({
26+
where: {
27+
id: req.uploadId,
28+
},
29+
select: {
30+
id: true,
31+
userId: true,
32+
bucket: true,
33+
contentLength: true,
34+
files: true,
35+
createdAt: true,
36+
updatedAt: true,
37+
completedAt: true,
38+
},
39+
});
40+
41+
// Error if the upload wasn't prepared
42+
if (!upload) {
43+
throw new RuntimeError(
44+
"upload_not_found",
45+
{
46+
meta: { uploadId: req.uploadId },
47+
},
48+
);
49+
}
50+
51+
// Check with S3 to see if the files were uploaded
52+
const fileExistencePromises = upload.files.map(
53+
async file => {
54+
// If the file was uploaded in parts, complete the multipart upload
55+
if (file.multipartUploadId) {
56+
try {
57+
const parts = await getMultipartUploadParts(
58+
s3,
59+
getKey(upload.id, file.path),
60+
file.multipartUploadId,
61+
);
62+
if (parts.length === 0) return false;
63+
64+
await completeMultipartUpload(
65+
s3,
66+
getKey(upload.id, file.path),
67+
file.multipartUploadId,
68+
parts,
69+
);
70+
71+
72+
} catch (e) {
73+
throw new RuntimeError(
74+
"multipart_upload_completion_fail",
75+
{ cause: e },
76+
)
77+
}
78+
}
79+
80+
// Check if the file exists
81+
return await keyExists(s3, getKey(upload.id, file.path))
82+
},
83+
);
84+
const fileExistence = await Promise.all(fileExistencePromises);
85+
const filesAllExist = fileExistence.every(Boolean);
86+
if (!filesAllExist) {
87+
const missingFiles = upload.files.filter((_, i) => !fileExistence[i]);
88+
throw new RuntimeError(
89+
"files_not_uploaded",
90+
{
91+
meta: {
92+
uploadId: req.uploadId,
93+
missingFiles: missingFiles.map((file) => file.path),
94+
},
95+
},
96+
);
97+
}
98+
99+
// Error if `complete` was already called with this ID
100+
if (upload.completedAt !== null) {
101+
throw new RuntimeError(
102+
"upload_already_completed",
103+
{
104+
meta: { uploadId: req.uploadId },
105+
},
106+
);
107+
}
108+
109+
// Update the upload to mark it as completed
110+
const completedUpload = await db.upload.update({
111+
where: {
112+
id: req.uploadId,
113+
},
114+
data: {
115+
completedAt: new Date().toISOString(),
116+
},
117+
select: {
118+
id: true,
119+
userId: true,
120+
bucket: true,
121+
contentLength: true,
122+
files: true,
123+
createdAt: true,
124+
updatedAt: true,
125+
completedAt: true,
126+
},
127+
});
128+
129+
return completedUpload;
130+
});
131+
132+
return {
133+
upload: prismaToOutput(newUpload, true),
134+
};
135+
}

‎modules/uploads/scripts/delete.ts‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
import { RuntimeError, ScriptContext } from "../_gen/scripts/delete.ts";
2+
import { getKey } from "../utils/types.ts";
3+
import { deleteKeys } from "../utils/bucket.ts";
4+
import { getS3EnvConfig } from "../utils/env.ts";
5+
6+
export interface Request {
7+
uploadId: string;
8+
}
9+
10+
export interface Response {
11+
bytesDeleted: string;
12+
}
13+
14+
export async function run(
15+
ctx: ScriptContext,
16+
req: Request,
17+
): Promise<Response> {
18+
const s3 = getS3EnvConfig(ctx.userConfig.s3);
19+
if (!s3) throw new RuntimeError("s3_not_configured");
20+
21+
const bytesDeleted = await ctx.db.$transaction(async (db) => {
22+
const upload = await db.upload.findFirst({
23+
where: {
24+
id: req.uploadId,
25+
completedAt: { not: null },
26+
deletedAt: null,
27+
},
28+
select: {
29+
id: true,
30+
userId: true,
31+
bucket: true,
32+
contentLength: true,
33+
files: true,
34+
createdAt: true,
35+
updatedAt: true,
36+
completedAt: true,
37+
},
38+
});
39+
if (!upload) {
40+
throw new RuntimeError(
41+
"upload_not_found",
42+
{
43+
meta: {
44+
modified: false,
45+
uploadId: req.uploadId,
46+
},
47+
},
48+
);
49+
}
50+
51+
const filesToDelete = upload.files.map((file) =>
52+
getKey(file.uploadId, file.path)
53+
);
54+
const deleteResults = await deleteKeys(s3, filesToDelete);
55+
56+
const failures = upload.files
57+
.map((file, i) => [file, deleteResults[i]] as const)
58+
.filter(([, successfullyDeleted]) => !successfullyDeleted)
59+
.map(([file]) => file);
60+
61+
if (failures.length) {
62+
const failedPaths = JSON.stringify(failures.map((file) => file.path));
63+
throw new RuntimeError(
64+
"failed_to_delete",
65+
{
66+
meta: {
67+
modified: failures.length !== filesToDelete.length,
68+
reason:`Failed to delete files with paths ${failedPaths}`,
69+
},
70+
},
71+
);
72+
}
73+
74+
await db.upload.update({
75+
where: {
76+
id: req.uploadId,
77+
},
78+
data: {
79+
deletedAt: new Date().toISOString(),
80+
},
81+
});
82+
83+
return upload.contentLength.toString();
84+
});
85+
return { bytesDeleted };
86+
}

0 commit comments

Comments
 (0)