Skip to content

Commit 5786fb5

Browse files
d4mrd4mr
authored
Add webhook metadata cleanup script and enhance Redis cleanup logic (#66)
- Introduced a new script for cleaning up old webhook metadata in Redis, allowing for batch processing and dry run options. - Updated existing Redis cleanup script to handle additional patterns and results for user operations, EIP-7702, webhooks, and external bundler sends. - Improved error handling and statistics tracking for various cleanup operations. - Added tests to verify pruning behavior with randomly generated job IDs to ensure correctness in job metadata management.
1 parent 029bc40 commit 5786fb5

File tree

4 files changed

+644
-8
lines changed

4 files changed

+644
-8
lines changed

‎scripts/cleanup-webhook-meta.ts‎

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
#!/usr/bin/env bun
2+
3+
import Redis from "ioredis";
4+
5+
if (!process.env.REDIS_URL) {
6+
throw new Error("REDIS_URL is not set");
7+
}
8+
9+
// Configuration
10+
const CONFIG = {
11+
redisUrl: process.env.REDIS_URL,
12+
batchSize: 5000,
13+
dryRun: false, // Set to false to actually delete
14+
maxAgeHours: 3, // Delete jobs finished more than 3 hours ago
15+
} as const;
16+
17+
class WebhookMetaCleanup {
18+
private redis: Redis;
19+
private stats = {
20+
totalScanned: 0,
21+
totalDeleted: 0,
22+
totalSkipped: 0,
23+
errors: 0,
24+
invalidTimestamps: 0,
25+
};
26+
27+
constructor() {
28+
this.redis = new Redis(CONFIG.redisUrl);
29+
}
30+
31+
async run(): Promise<void> {
32+
console.log(`🚀 Starting cleanup (DRY_RUN: ${CONFIG.dryRun})`);
33+
console.log("🎯 Target pattern:");
34+
console.log(" - twmq:engine-cloud_webhook:job:*:meta");
35+
console.log(` - Max age: ${CONFIG.maxAgeHours} hours`);
36+
console.log("");
37+
38+
try {
39+
await this.cleanOldJobMeta();
40+
this.printFinalStats();
41+
} catch (error) {
42+
console.error(`💥 Fatal error: ${error}`);
43+
throw error;
44+
} finally {
45+
await this.redis.quit();
46+
}
47+
}
48+
49+
private async cleanOldJobMeta(): Promise<void> {
50+
const pattern = "twmq:engine-cloud_webhook:job:*:meta";
51+
console.log(`🔍 Scanning pattern: ${pattern}`);
52+
53+
let cursor = "0";
54+
// Unix timestamps are always in UTC (seconds since Jan 1, 1970 00:00:00 UTC)
55+
const now = Math.floor(Date.now() / 1000);
56+
const cutoffTimestamp = now - (CONFIG.maxAgeHours * 60 * 60);
57+
58+
console.log(` Current time (UTC): ${now} (${new Date(now * 1000).toISOString()})`);
59+
console.log(` Cutoff time (UTC): ${cutoffTimestamp} (${new Date(cutoffTimestamp * 1000).toISOString()})`);
60+
console.log("");
61+
62+
do {
63+
const [newCursor, keys] = await this.redis.scan(
64+
cursor,
65+
"MATCH",
66+
pattern,
67+
"COUNT",
68+
CONFIG.batchSize
69+
);
70+
cursor = newCursor;
71+
72+
if (keys.length > 0) {
73+
this.stats.totalScanned += keys.length;
74+
console.log(` Scanned ${keys.length} keys (total: ${this.stats.totalScanned})`);
75+
76+
await this.processKeyBatch(keys, cutoffTimestamp);
77+
}
78+
} while (cursor !== "0");
79+
80+
console.log(`✅ Scan complete: ${pattern} (scanned ${this.stats.totalScanned} keys)`);
81+
console.log("");
82+
}
83+
84+
private async processKeyBatch(keys: string[], cutoffTimestamp: number): Promise<void> {
85+
const keysToDelete: string[] = [];
86+
87+
// Batch fetch all finished_at timestamps using pipeline
88+
const pipeline = this.redis.pipeline();
89+
for (const key of keys) {
90+
pipeline.hget(key, "finished_at");
91+
}
92+
93+
let results;
94+
try {
95+
results = await pipeline.exec();
96+
} catch (error) {
97+
console.error(` 💥 Error fetching timestamps batch: ${error}`);
98+
this.stats.errors += keys.length;
99+
return;
100+
}
101+
102+
// Process results
103+
for (let i = 0; i < keys.length; i++) {
104+
const key = keys[i];
105+
if (!key) continue;
106+
107+
const [err, finishedAt] = results?.[i] ?? [null, null];
108+
109+
if (err) {
110+
console.error(` 💥 Error processing key ${key}: ${err}`);
111+
this.stats.errors += 1;
112+
continue;
113+
}
114+
115+
if (!finishedAt) {
116+
this.stats.totalSkipped += 1;
117+
continue;
118+
}
119+
120+
const finishedAtTimestamp = parseInt(finishedAt as string, 10);
121+
122+
if (isNaN(finishedAtTimestamp)) {
123+
this.stats.invalidTimestamps += 1;
124+
continue;
125+
}
126+
127+
if (finishedAtTimestamp < cutoffTimestamp) {
128+
const age = Math.floor((Date.now() / 1000 - finishedAtTimestamp) / 3600);
129+
if (keysToDelete.length < 10) {
130+
// Only log first 10 to avoid spam
131+
console.log(` 🗑️ Marking for deletion: ${key} (finished ${age}h ago)`);
132+
}
133+
keysToDelete.push(key);
134+
} else {
135+
this.stats.totalSkipped += 1;
136+
}
137+
}
138+
139+
// Delete the marked keys
140+
if (keysToDelete.length > 0) {
141+
console.log(` Found ${keysToDelete.length} keys to delete in this batch`);
142+
if (CONFIG.dryRun) {
143+
console.log(` [DRY RUN] Would delete ${keysToDelete.length} keys`);
144+
this.stats.totalDeleted += keysToDelete.length;
145+
} else {
146+
await this.deleteKeys(keysToDelete);
147+
}
148+
}
149+
}
150+
151+
private async deleteKeys(keys: string[]): Promise<void> {
152+
try {
153+
const pipeline = this.redis.pipeline();
154+
for (const key of keys) {
155+
pipeline.del(key);
156+
}
157+
158+
const results = await pipeline.exec();
159+
const deletedCount = results?.filter(([err]) => err === null).length || 0;
160+
const failedCount = keys.length - deletedCount;
161+
162+
console.log(` ✅ Deleted ${deletedCount} keys`);
163+
if (failedCount > 0) {
164+
console.log(` ❌ Failed to delete ${failedCount} keys`);
165+
this.stats.errors += failedCount;
166+
}
167+
168+
this.stats.totalDeleted += deletedCount;
169+
} catch (error) {
170+
console.error(` 💥 Error deleting batch: ${error}`);
171+
this.stats.errors += keys.length;
172+
}
173+
}
174+
175+
private printFinalStats(): void {
176+
console.log("📈 Final Statistics:");
177+
console.log(` Total Scanned: ${this.stats.totalScanned.toLocaleString()}`);
178+
console.log(` Total ${CONFIG.dryRun ? 'Would Delete' : 'Deleted'}: ${this.stats.totalDeleted.toLocaleString()}`);
179+
console.log(` Total Skipped (not old enough): ${this.stats.totalSkipped.toLocaleString()}`);
180+
if (this.stats.invalidTimestamps > 0) {
181+
console.log(` Invalid Timestamps: ${this.stats.invalidTimestamps.toLocaleString()}`);
182+
}
183+
if (this.stats.errors > 0) {
184+
console.log(` Errors: ${this.stats.errors.toLocaleString()}`);
185+
}
186+
console.log("");
187+
188+
if (CONFIG.dryRun) {
189+
console.log("💡 This was a DRY RUN - no data was actually deleted");
190+
console.log("💡 Set CONFIG.dryRun = false to actually delete the keys");
191+
} else {
192+
console.log("✅ CLEANUP COMPLETED - Data has been permanently deleted");
193+
}
194+
}
195+
}
196+
197+
// Main execution
198+
async function main() {
199+
const cleaner = new WebhookMetaCleanup();
200+
await cleaner.run();
201+
}
202+
203+
if (import.meta.main) {
204+
main().catch(console.error);
205+
}
206+

‎scripts/simple-redis-cleanup.ts‎

Lines changed: 83 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,13 @@ class SimpleRedisCleanup {
1717
private redis: Redis;
1818
private stats = {
1919
useropErrors: 0,
20+
useropResults: 0,
2021
eip7702Errors: 0,
22+
eip7702Results: 0,
23+
webhookErrors: 0,
24+
webhookResults: 0,
25+
externalBundlerErrors: 0,
26+
externalBundlerResults: 0,
2127
totalDeleted: 0,
2228
errors: 0,
2329
};
@@ -30,15 +36,31 @@ class SimpleRedisCleanup {
3036
console.log(`🚀 Starting cleanup (DRY_RUN: ${CONFIG.dryRun})`);
3137
console.log("🎯 Target patterns:");
3238
console.log(" - twmq:engine-cloud_userop_confirm:job:*:errors");
39+
console.log(" - twmq:engine-cloud_userop_confirm:jobs:result (hash)");
3340
console.log(" - twmq:engine-cloud_eip7702_send:job:*:errors");
41+
console.log(" - twmq:engine-cloud_eip7702_send:jobs:result (hash)");
42+
console.log(" - twmq:engine-cloud_webhook:job:*:errors");
43+
console.log(" - twmq:engine-cloud_webhook:jobs:result (hash)");
44+
console.log(" - twmq:engine-cloud_external_bundler_send:job:*:errors");
45+
console.log(" - twmq:engine-cloud_external_bundler_send:jobs:result (hash)");
3446
console.log("");
3547

3648
try {
37-
// Clean userop confirm error keys
49+
// Clean userop confirm keys
3850
await this.cleanPattern("twmq:engine-cloud_userop_confirm:job:*:errors");
51+
await this.cleanHash("twmq:engine-cloud_userop_confirm:jobs:result", "userop_confirm");
3952

40-
// Clean eip7702 send error keys
53+
// Clean eip7702 send keys
4154
await this.cleanPattern("twmq:engine-cloud_eip7702_send:job:*:errors");
55+
await this.cleanHash("twmq:engine-cloud_eip7702_send:jobs:result", "eip7702_send");
56+
57+
// Clean webhook keys
58+
await this.cleanPattern("twmq:engine-cloud_webhook:job:*:errors");
59+
await this.cleanHash("twmq:engine-cloud_webhook:jobs:result", "webhook");
60+
61+
// Clean external bundler send keys
62+
await this.cleanPattern("twmq:engine-cloud_external_bundler_send:job:*:errors");
63+
await this.cleanHash("twmq:engine-cloud_external_bundler_send:jobs:result", "external_bundler_send");
4264

4365
this.printFinalStats();
4466
} catch (error) {
@@ -83,6 +105,37 @@ class SimpleRedisCleanup {
83105
console.log("");
84106
}
85107

108+
private async cleanHash(key: string, queueType: string): Promise<void> {
109+
console.log(`🔍 Checking hash: ${key}`);
110+
111+
try {
112+
const exists = await this.redis.exists(key);
113+
114+
if (exists) {
115+
const fieldCount = await this.redis.hlen(key);
116+
console.log(` Found hash with ${fieldCount} fields`);
117+
118+
if (CONFIG.dryRun) {
119+
console.log(` [DRY RUN] Would delete hash with ${fieldCount} fields`);
120+
this.updateStatsForHash(queueType, fieldCount);
121+
} else {
122+
await this.redis.del(key);
123+
console.log(` ✅ Deleted hash with ${fieldCount} fields`);
124+
this.updateStatsForHash(queueType, fieldCount);
125+
this.stats.totalDeleted += 1;
126+
}
127+
} else {
128+
console.log(` Hash does not exist`);
129+
}
130+
131+
console.log(`✅ Hash complete: ${key}`);
132+
console.log("");
133+
} catch (error) {
134+
console.error(` 💥 Error handling hash: ${error}`);
135+
this.stats.errors += 1;
136+
}
137+
}
138+
86139
private async deleteKeys(keys: string[]): Promise<void> {
87140
try {
88141
const pipeline = this.redis.pipeline();
@@ -112,13 +165,39 @@ class SimpleRedisCleanup {
112165
this.stats.useropErrors += count;
113166
} else if (pattern.includes("eip7702_send")) {
114167
this.stats.eip7702Errors += count;
168+
} else if (pattern.includes("webhook")) {
169+
this.stats.webhookErrors += count;
170+
} else if (pattern.includes("external_bundler_send")) {
171+
this.stats.externalBundlerErrors += count;
172+
}
173+
}
174+
175+
private updateStatsForHash(queueType: string, count: number): void {
176+
if (queueType === "userop_confirm") {
177+
this.stats.useropResults += count;
178+
} else if (queueType === "eip7702_send") {
179+
this.stats.eip7702Results += count;
180+
} else if (queueType === "webhook") {
181+
this.stats.webhookResults += count;
182+
} else if (queueType === "external_bundler_send") {
183+
this.stats.externalBundlerResults += count;
115184
}
116185
}
117186

118187
private printFinalStats(): void {
119188
console.log("📈 Final Statistics:");
120-
console.log(` Userop Confirm Errors: ${this.stats.useropErrors.toLocaleString()}`);
121-
console.log(` EIP-7702 Send Errors: ${this.stats.eip7702Errors.toLocaleString()}`);
189+
console.log(` Userop Confirm:`);
190+
console.log(` - Errors: ${this.stats.useropErrors.toLocaleString()}`);
191+
console.log(` - Result Hash Fields: ${this.stats.useropResults.toLocaleString()}`);
192+
console.log(` EIP-7702 Send:`);
193+
console.log(` - Errors: ${this.stats.eip7702Errors.toLocaleString()}`);
194+
console.log(` - Result Hash Fields: ${this.stats.eip7702Results.toLocaleString()}`);
195+
console.log(` Webhook:`);
196+
console.log(` - Errors: ${this.stats.webhookErrors.toLocaleString()}`);
197+
console.log(` - Result Hash Fields: ${this.stats.webhookResults.toLocaleString()}`);
198+
console.log(` External Bundler Send:`);
199+
console.log(` - Errors: ${this.stats.externalBundlerErrors.toLocaleString()}`);
200+
console.log(` - Result Hash Fields: ${this.stats.externalBundlerResults.toLocaleString()}`);
122201
console.log(` Total ${CONFIG.dryRun ? 'Would Delete' : 'Deleted'}: ${this.stats.totalDeleted.toLocaleString()}`);
123202
if (this.stats.errors > 0) {
124203
console.log(` Errors: ${this.stats.errors}`);

0 commit comments

Comments
 (0)