feat(fuzz): improve fuzzing effectiveness, better timeouts, and comprehensive seeds, reproduction tooling

.github/actions/fuzzing/action.yml

@@ -23,14 +23,10 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Install AFL++
+    - name: Verify AFL++ installation
       shell: bash
       run: |
-        echo "Installing AFL++..."
-        apt-get update -qq
-        apt-get install -y -qq afl++ lld-17 > /dev/null
-
-        echo "AFL++ installed successfully"
+        echo "Verifying AFL++ installation..."
         afl-fuzz -h | head -5 || true
 
         # Verify AFL++ compilers are available
@@ -99,6 +95,7 @@ runs:
         AFL_TESTCACHE_SIZE: ${{ inputs.mode == 'smoke' && '50' || '500' }}
         AFL_SKIP_CPUFREQ: ${{ inputs.mode == 'long' && '1' || '' }}
         AFL_FAST_CAL: ${{ inputs.mode == 'long' && '1' || '' }}
+        AFL_PERSISTENT_RECORD: 1000
 
     - name: Analyze fuzzing results
       shell: bash

.github/workflows/fuzz-long.yml

@@ -36,7 +36,7 @@ jobs:
         instance: [0]  # Can be expanded to [0, 1, 2, 3] for parallel fuzzing
 
     container:
-      image: ghcr.io/romange/ubuntu-dev:24
+      image: ghcr.io/romange/ubuntu-dev:24-afl
       options: --security-opt seccomp=unconfined --sysctl "net.ipv6.conf.all.disable_ipv6=0"
       credentials:
         username: ${{ github.repository_owner }}

.github/workflows/fuzz-smoke.yml

@@ -26,7 +26,7 @@ jobs:
     timeout-minutes: 45
 
     container:
-      image: ghcr.io/romange/ubuntu-dev:24
+      image: ghcr.io/romange/ubuntu-dev:24-afl
       options: --security-opt seccomp=unconfined --sysctl "net.ipv6.conf.all.disable_ipv6=0"
       credentials:
         username: ${{ github.repository_owner }}

fuzz/FUZZING.md

@@ -2,14 +2,20 @@
 
 ## Install AFL++
 
+For effective fuzzing with crash replay support, AFL++ must be built from source with `AFL_PERSISTENT_RECORD` enabled.
+
 ```bash
-# From package manager (Ubuntu/Debian)
+# Install dependencies
 sudo apt update
-sudo apt install afl++
+sudo apt install llvm-18-dev clang-18 lld-18 gcc-13-plugin-dev
 
-# Or build from source
-git clone https://github.com/AFLplusplus/AFLplusplus
+# Build AFL++ with AFL_PERSISTENT_RECORD support
+git clone --depth=1 --branch v4.34c https://github.com/AFLplusplus/AFLplusplus.git
 cd AFLplusplus
+
+# Enable AFL_PERSISTENT_RECORD (required for stateful crash replay)
+sed -i 's|// #define AFL_PERSISTENT_RECORD|#define AFL_PERSISTENT_RECORD|' include/config.h
+
 make distrib
 sudo make install
 ```
@@ -22,22 +28,74 @@ sudo afl-system-config
 
 Sets core_pattern and CPU governors for optimal AFL++ performance.
 
-## Build
+## Build Dragonfly
 
 ```bash
 cmake -B build-dbg -DUSE_AFL=ON -DCMAKE_BUILD_TYPE=Debug -GNinja
 ninja -C build-dbg dragonfly
 ```
 
-## Run
+## Run Fuzzer
 
 ```bash
 cd fuzz
 ./run_fuzzer.sh
 ```
 
-## Replay Crash on production dragonfly:
+## AFL_PERSISTENT_RECORD (Stateful Crash Replay)
+
+Dragonfly uses AFL++ persistent mode for performance. This means multiple fuzzing iterations run within the same process, and the server accumulates state between iterations.
+
+**Problem:** When a crash occurs, AFL++ only saves the last input. But the crash may depend on state accumulated from previous inputs.
+
+**Solution:** `AFL_PERSISTENT_RECORD` saves the last N inputs before a crash, enabling replay of the full sequence.
+
+### Enable Recording
+
+```bash
+# Set number of inputs to record before crash (e.g., last 100 inputs)
+AFL_PERSISTENT_RECORD=100 ./run_fuzzer.sh
+```
+
+When a crash occurs, AFL++ saves files in the crashes directory:
+```
+crashes/RECORD:000000,cnt:000000  (input N-99)
+crashes/RECORD:000000,cnt:000001  (input N-98)
+...
+crashes/RECORD:000000,cnt:000099  (crashing input)
+```
+
+### Replay Recorded Crash
+
+```bash
+# Set directory containing RECORD files
+export AFL_PERSISTENT_DIR=./artifacts/resp/default/crashes
+
+# Replay specific record (e.g., record 000000)
+AFL_PERSISTENT_REPLAY=000000 ./build-dbg/dragonfly --port=6379
+```
+
+This replays all recorded inputs in sequence, reproducing the exact state that led to the crash.
+
+### Manual Replay (Alternative)
+
+If AFL_PERSISTENT_REPLAY doesn't work, replay manually:
+
+```bash
+# Start dragonfly
+./build-dbg/dragonfly --port=6379 &
+
+# Send each recorded input in order
+for f in $(ls crashes/RECORD:000000,cnt:* | sort); do
+    nc localhost 6379 < "$f"
+done
+```
+
+## Replay Simple Crash
+
+For crashes that don't depend on accumulated state:
+
 ```bash
-./dragonfly --port=6379 &
+./build-dbg/dragonfly --port=6379 &
 nc localhost 6379 < artifacts/resp/default/crashes/id:000000,...
 ```

fuzz/dict/resp.dict

@@ -34,8 +34,6 @@
 "LINDEX"
 "LSET"
 "LTRIM"
-"BLPOP"
-"BRPOP"
 
 # Hash operations
 "HSET"
@@ -186,11 +184,93 @@
 "LIMIT"
 "COUNT"
 "MATCH"
-"BLOCK"
 
 # Common RESP command patterns
 "*1\x0d\x0a$4\x0d\x0aPING\x0d\x0a"
 "*2\x0d\x0a$3\x0d\x0aGET\x0d\x0a$3\x0d\x0akey\x0d\x0a"
 "*3\x0d\x0a$3\x0d\x0aSET\x0d\x0a$3\x0d\x0akey\x0d\x0a$5\x0d\x0avalue\x0d\x0a"
 "*2\x0d\x0a$3\x0d\x0aDEL\x0d\x0a$3\x0d\x0akey\x0d\x0a"
 "*2\x0d\x0a$6\x0d\x0aEXISTS\x0d\x0a$3\x0d\x0akey\x0d\x0a"
+
+# Scripting commands
+"EVAL"
+"EVALSHA"
+"EVAL_RO"
+"EVALSHA_RO"
+"SCRIPT"
+
+# Bitfield commands
+"BITFIELD"
+"BITFIELD_RO"
+"BITOP"
+"BITCOUNT"
+"BITPOS"
+"GETBIT"
+"SETBIT"
+
+# More sorted set operations
+"ZINTER"
+"ZUNION"
+"ZINTERSTORE"
+"ZUNIONSTORE"
+"ZPOPMIN"
+"ZPOPMAX"
+"ZMPOP"
+
+# Edge case numbers
+"9223372036854775807"
+"-9223372036854775808"
+"2147483647"
+"-2147483648"
+"0.0"
+"-0.0"
+"inf"
+"-inf"
+"+inf"
+"nan"
+
+# Stream IDs and patterns
+"0-0"
+"0-*"
+"$"
+">"
+"*"
+"MAXLEN"
+"MINID"
+
+# JSON paths
+"$.."
+"$[*]"
+"$[-1]"
+"$.name"
+"$..name"
+
+# RESP protocol edge cases
+"*-1\x0d\x0a"
+"$-1\x0d\x0a"
+"*0\x0d\x0a"
+"$0\x0d\x0a\x0d\x0a"
+
+# Lua scripting patterns
+"return redis.call"
+"redis.pcall"
+"KEYS[1]"
+"ARGV[1]"
+
+# Bitfield subcommands
+"OVERFLOW"
+"WRAP"
+"SAT"
+"FAIL"
+
+# Aggregate options
+"AGGREGATE"
+"SUM"
+"MIN"
+"MAX"
+"WEIGHTS"
+
+# Binary edge cases
+"\x00"
+"\xff"
+"\x00\x00\x00\x00"

fuzz/run_fuzzer.sh

@@ -17,7 +17,7 @@ OUTPUT_DIR="${OUTPUT_DIR:-$FUZZ_DIR/artifacts/$TARGET}"
 CORPUS_DIR="${CORPUS_DIR:-$FUZZ_DIR/corpus/$TARGET}"
 SEEDS_DIR="${SEEDS_DIR:-$FUZZ_DIR/seeds/$TARGET}"
 DICT_FILE="${DICT_FILE:-$FUZZ_DIR/dict/$TARGET.dict}"
-TIMEOUT="10000+"
+TIMEOUT="500"
 FUZZ_TARGET="$BUILD_DIR/dragonfly"
 AFL_PROACTOR_THREADS="${AFL_PROACTOR_THREADS:-2}"
 
@@ -79,9 +79,10 @@ run_fuzzer() {
 
     AFL_CMD=(
         afl-fuzz -D
+        -l 2
         -o "${OUTPUT_DIR}"
         -t "${TIMEOUT}"
-        -m none
+        -m 4096
         -i "${CORPUS_DIR}"
     )
 
@@ -99,6 +100,10 @@ run_fuzzer() {
         --bind=::
         --dbfilename=""
         --omit_basic_usage
+        --rename_command=SHUTDOWN=
+        --rename_command=DEBUG=
+        --rename_command=FLUSHALL=
+        --rename_command=FLUSHDB=
     )
 
     print_info "Running: ${AFL_CMD[*]}"
@@ -107,6 +112,9 @@ run_fuzzer() {
     cd "${OUTPUT_DIR}"
 
     # Run AFL++ - fuzzing integrated in dragonfly via USE_AFL
+    # AFL_HANG_TMOUT: Only consider it a hang if no response for 60 seconds
+    # This prevents false positives from slow but legitimate operations
+    export AFL_HANG_TMOUT=60000
     exec "${AFL_CMD[@]}"
 }
 

fuzz/seeds/resp/bitfield.resp

@@ -0,0 +1,11 @@
+*6
+$8
+BITFIELD
+$3
+key
+$3
+GET
+$2
+u8
+$1
+0

fuzz/seeds/resp/eval.resp

@@ -0,0 +1,6 @@
+*3
+$4
+EVAL
+$26
+return redis.call("PING")
+$0

fuzz/seeds/resp/hset.resp

@@ -0,0 +1,9 @@
+*4
+$4
+HSET
+$4
+hash
+$5
+field
+$5
+value

fuzz/seeds/resp/json.resp

@@ -0,0 +1,9 @@
+*4
+$8
+JSON.SET
+$3
+doc
+$1
+$
+$15
+{"name":"test"}

fuzz/seeds/resp/pipeline.resp

@@ -0,0 +1,25 @@
+*1
+$4
+PING
+*3
+$3
+SET
+$1
+a
+$1
+1
+*2
+$4
+INCR
+$1
+a
+*2
+$3
+GET
+$1
+a
+*2
+$3
+DEL
+$1
+a

fuzz/seeds/resp/sadd.resp

@@ -0,0 +1,7 @@
+*3
+$4
+SADD
+$3
+set
+$6
+member

fuzz/seeds/resp/watch_multi.resp

@@ -0,0 +1,18 @@
+*2
+$5
+WATCH
+$1
+k
+*1
+$5
+MULTI
+*3
+$3
+SET
+$1
+k
+$1
+1
+*1
+$4
+EXEC

fuzz/seeds/resp/xadd.resp

@@ -0,0 +1,11 @@
+*5
+$4
+XADD
+$6
+stream
+$1
+*
+$5
+field
+$5
+value