From ed0fe3c3cd80f308ada668f6e5ee903caba806fe Mon Sep 17 00:00:00 2001 From: Tim Waugh Date: Fri, 19 Sep 2025 09:23:22 +0100 Subject: [PATCH] Fix fuzz testing Assisted-by: Cursor --- Makefile.am | 68 ++++++++++++++++++++++++++++++++++++++++- Makefile.fuzz | 43 ++++++++++++++++---------- configure.ac | 48 +++++++++++++++++++++++++++++ fuzz/README.md | 33 ++++++++++++++++++-- fuzz/analyze_crashes.sh | 19 ++++++++++-- fuzz/run_fuzz.sh | 53 +++++++++++++++++--------------- 6 files changed, 217 insertions(+), 47 deletions(-) diff --git a/Makefile.am b/Makefile.am index e9859e33..bf32ab3b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -348,9 +348,73 @@ distclean-local: -rm -rf $(top_builddir)/test-arena -rm -f lib-built -# Fuzzing targets - delegate to separate makefile +if ENABLE_FUZZING +# Fuzzing-specific instrumented binaries +noinst_PROGRAMS = src/fuzz-filterdiff src/fuzz-interdiff src/fuzz-rediff + +src_fuzz_filterdiff_SOURCES = $(src_filterdiff_SOURCES) +src_fuzz_filterdiff_LDADD = $(src_filterdiff_LDADD) +src_fuzz_filterdiff_CFLAGS = $(AM_CFLAGS) $(FUZZ_CFLAGS) +src_fuzz_filterdiff_CPPFLAGS = $(AM_CPPFLAGS) $(FUZZ_CPPFLAGS) + +src_fuzz_interdiff_SOURCES = $(src_interdiff_SOURCES) +src_fuzz_interdiff_LDADD = $(src_interdiff_LDADD) +src_fuzz_interdiff_CFLAGS = $(AM_CFLAGS) $(FUZZ_CFLAGS) +src_fuzz_interdiff_CPPFLAGS = $(AM_CPPFLAGS) $(FUZZ_CPPFLAGS) + +src_fuzz_rediff_SOURCES = $(src_rediff_SOURCES) +src_fuzz_rediff_LDADD = $(src_rediff_LDADD) +src_fuzz_rediff_CFLAGS = $(AM_CFLAGS) $(FUZZ_CFLAGS) +src_fuzz_rediff_CPPFLAGS = $(AM_CPPFLAGS) $(FUZZ_CPPFLAGS) + +# Override CC and LINK for fuzzing targets to use AFL++ for both compilation and linking +$(src_fuzz_filterdiff_OBJECTS): CC = $(AFL_CC) +$(src_fuzz_interdiff_OBJECTS): CC = $(AFL_CC) +$(src_fuzz_rediff_OBJECTS): CC = $(AFL_CC) + +# Use AFL++ compiler for linking to include AFL++ runtime +src_fuzz_filterdiff_LINK = $(AFL_CC) $(AM_CFLAGS) $(CFLAGS) $(FUZZ_CFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@ +src_fuzz_interdiff_LINK = $(AFL_CC) $(AM_CFLAGS) $(CFLAGS) $(FUZZ_CFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@ +src_fuzz_rediff_LINK = $(AFL_CC) $(AM_CFLAGS) $(CFLAGS) $(FUZZ_CFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@ + +# Create fuzzing symlinks (like regular tools) +src/fuzz-combinediff$(EXEEXT): src/fuzz-interdiff$(EXEEXT) + ln -sf $(notdir $<) $@ + +src/fuzz-flipdiff$(EXEEXT): src/fuzz-interdiff$(EXEEXT) + ln -sf $(notdir $<) $@ + +src/fuzz-lsdiff$(EXEEXT): src/fuzz-filterdiff$(EXEEXT) + ln -sf $(notdir $<) $@ + +src/fuzz-grepdiff$(EXEEXT): src/fuzz-filterdiff$(EXEEXT) + ln -sf $(notdir $<) $@ + +src/fuzz-patchview$(EXEEXT): src/fuzz-filterdiff$(EXEEXT) + ln -sf $(notdir $<) $@ + +CLEANFILES += src/fuzz-combinediff src/fuzz-flipdiff src/fuzz-lsdiff src/fuzz-grepdiff src/fuzz-patchview + +# Fuzzing targets - use instrumented binaries when available fuzz-help fuzz-corpus fuzz-test fuzz-analyze fuzz-clean fuzz-check fuzz-ci fuzz-stats: @if [ -f Makefile.fuzz ]; then \ + $(MAKE) -f Makefile.fuzz FUZZ_BINDIR="$(top_builddir)/src" $@; \ + else \ + echo "Fuzzing not available - Makefile.fuzz not found"; \ + fi + +fuzz-filterdiff fuzz-interdiff fuzz-rediff fuzz-grepdiff fuzz-lsdiff: src/fuzz-filterdiff src/fuzz-interdiff src/fuzz-rediff src/fuzz-lsdiff$(EXEEXT) src/fuzz-grepdiff$(EXEEXT) + @if [ -f Makefile.fuzz ]; then \ + $(MAKE) -f Makefile.fuzz FUZZ_BINDIR="$(top_builddir)/src" $@; \ + else \ + echo "Fuzzing not available - Makefile.fuzz not found"; \ + fi + +else +# Fuzzing targets - delegate to separate makefile (non-instrumented fallback) +fuzz-help fuzz-corpus fuzz-test fuzz-analyze fuzz-clean fuzz-check fuzz-ci fuzz-stats: + @if [ -f Makefile.fuzz ]; then \ + echo "Warning: Using non-instrumented binaries - configure with --enable-fuzzing for better results"; \ $(MAKE) -f Makefile.fuzz $@; \ else \ echo "Fuzzing not available - Makefile.fuzz not found"; \ @@ -358,10 +422,12 @@ fuzz-help fuzz-corpus fuzz-test fuzz-analyze fuzz-clean fuzz-check fuzz-ci fuzz- fuzz-filterdiff fuzz-interdiff fuzz-rediff fuzz-grepdiff fuzz-lsdiff: @if [ -f Makefile.fuzz ]; then \ + echo "Warning: Using non-instrumented binaries - configure with --enable-fuzzing for better results"; \ $(MAKE) -f Makefile.fuzz $@; \ else \ echo "Fuzzing not available - Makefile.fuzz not found"; \ fi +endif EXTRA_DIST = $(man_MANS) \ tests/common.sh tests/soak-test \ diff --git a/Makefile.fuzz b/Makefile.fuzz index 2ca0add5..a9c539fd 100644 --- a/Makefile.fuzz +++ b/Makefile.fuzz @@ -5,6 +5,9 @@ CORPUS_DIR = $(FUZZ_DIR)/corpus RESULTS_DIR = $(FUZZ_DIR)/results TOOLS = filterdiff interdiff rediff grepdiff lsdiff +# Allow overriding binary directory (for instrumented builds) +FUZZ_BINDIR ?= src + .PHONY: fuzz-help fuzz-corpus fuzz-clean fuzz-test fuzz-all fuzz-analyze .PHONY: $(addprefix fuzz-, $(TOOLS)) @@ -29,6 +32,9 @@ fuzz-help: @echo " - AFL++ installed (american-fuzzy-lop package)" @echo " - Tools built (run 'make' first)" @echo "" + @echo "For best results (instrumented binaries):" + @echo " ./configure --enable-fuzzing && make" + @echo "" @echo "Quick start: make fuzz-corpus && make fuzz-test" # Generate or update the fuzzing corpus @@ -45,10 +51,15 @@ fuzz-corpus: $(FUZZ_DIR)/generate_corpus.sh @echo "Corpus ready: $$(ls -1 $(CORPUS_DIR) | wc -l) files" # Quick fuzzing test (60 seconds) -fuzz-test: fuzz-corpus src/filterdiff +fuzz-test: fuzz-corpus $(FUZZ_BINDIR)/filterdiff @echo "Running quick fuzz test (60 seconds)..." @echo "This will test basic fuzzing functionality" - timeout 60s $(FUZZ_DIR)/run_fuzz.sh filterdiff || true + @if [ -f "$(FUZZ_BINDIR)/fuzz-filterdiff" ]; then \ + echo "Using instrumented binary: $(FUZZ_BINDIR)/fuzz-filterdiff"; \ + else \ + echo "Warning: Using non-instrumented binary: $(FUZZ_BINDIR)/filterdiff"; \ + fi + FUZZ_BINDIR="$(FUZZ_BINDIR)" timeout 60s $(FUZZ_DIR)/run_fuzz.sh filterdiff || true @echo "" @echo "Quick test completed. Check fuzz/results/filterdiff/ for results" @if [ -d "$(RESULTS_DIR)/filterdiff/crashes" ] && [ -n "$$(ls -A $(RESULTS_DIR)/filterdiff/crashes 2>/dev/null)" ]; then \ @@ -58,16 +69,16 @@ fuzz-test: fuzz-corpus src/filterdiff fi # Extended fuzzing on all tools (runs in background) -fuzz-all: fuzz-corpus $(addprefix src/, $(TOOLS)) +fuzz-all: fuzz-corpus @echo "Starting extended fuzzing on all tools..." @echo "This will run fuzzing sessions in the background" @echo "Monitor with: ps aux | grep afl-fuzz" @echo "Stop with: pkill afl-fuzz" @echo "" @for tool in $(TOOLS); do \ - if [ -f "src/$$tool" ]; then \ + if [ -f "$(FUZZ_BINDIR)/$$tool" ] || [ -f "$(FUZZ_BINDIR)/fuzz-$$tool" ]; then \ echo "Starting $$tool fuzzing in background..."; \ - nohup $(FUZZ_DIR)/run_fuzz.sh $$tool > $(RESULTS_DIR)/$$tool.log 2>&1 & \ + FUZZ_BINDIR="$(FUZZ_BINDIR)" nohup $(FUZZ_DIR)/run_fuzz.sh $$tool > $(RESULTS_DIR)/$$tool.log 2>&1 & \ sleep 2; \ fi; \ done @@ -75,26 +86,26 @@ fuzz-all: fuzz-corpus $(addprefix src/, $(TOOLS)) @echo "All fuzzing sessions started. Logs in $(RESULTS_DIR)/*.log" @echo "Run 'make fuzz-analyze' periodically to check for crashes" -# Individual tool fuzzing targets -fuzz-filterdiff: fuzz-corpus src/filterdiff +# Individual tool fuzzing targets - depend on instrumented binaries when available +fuzz-filterdiff: fuzz-corpus @echo "Starting filterdiff fuzzing (interactive)..." - $(FUZZ_DIR)/run_fuzz.sh filterdiff + FUZZ_BINDIR="$(FUZZ_BINDIR)" $(FUZZ_DIR)/run_fuzz.sh filterdiff -fuzz-interdiff: fuzz-corpus src/interdiff +fuzz-interdiff: fuzz-corpus @echo "Starting interdiff fuzzing (interactive)..." - $(FUZZ_DIR)/run_fuzz.sh interdiff + FUZZ_BINDIR="$(FUZZ_BINDIR)" $(FUZZ_DIR)/run_fuzz.sh interdiff -fuzz-rediff: fuzz-corpus src/rediff +fuzz-rediff: fuzz-corpus @echo "Starting rediff fuzzing (interactive)..." - $(FUZZ_DIR)/run_fuzz.sh rediff + FUZZ_BINDIR="$(FUZZ_BINDIR)" $(FUZZ_DIR)/run_fuzz.sh rediff -fuzz-grepdiff: fuzz-corpus src/grepdiff +fuzz-grepdiff: fuzz-corpus @echo "Starting grepdiff fuzzing (interactive)..." - $(FUZZ_DIR)/run_fuzz.sh grepdiff + FUZZ_BINDIR="$(FUZZ_BINDIR)" $(FUZZ_DIR)/run_fuzz.sh grepdiff -fuzz-lsdiff: fuzz-corpus src/lsdiff +fuzz-lsdiff: fuzz-corpus @echo "Starting lsdiff fuzzing (interactive)..." - $(FUZZ_DIR)/run_fuzz.sh lsdiff + FUZZ_BINDIR="$(FUZZ_BINDIR)" $(FUZZ_DIR)/run_fuzz.sh lsdiff # Analyze fuzzing results fuzz-analyze: $(FUZZ_DIR)/analyze_crashes.sh diff --git a/configure.ac b/configure.ac index 15f6aa6c..7a87b18b 100644 --- a/configure.ac +++ b/configure.ac @@ -103,6 +103,54 @@ AS_IF([test "x$with_pcre2" != xno], AC_MSG_RESULT(no) ) +dnl Check for fuzzing support +AC_MSG_CHECKING([whether to enable fuzzing support]) +AC_ARG_ENABLE([fuzzing], + [AS_HELP_STRING([--enable-fuzzing], + [enable AFL++ fuzzing instrumentation @<:@default=no@:>@])], + [], [enable_fuzzing=no]) +AC_MSG_RESULT($enable_fuzzing) + +AM_CONDITIONAL([ENABLE_FUZZING], [test "x$enable_fuzzing" = "xyes"]) + +if test "x$enable_fuzzing" = "xyes"; then + # Check for AFL++ compilers (prefer modern versions) + AC_CHECK_PROGS([AFL_CC], [afl-clang-fast afl-gcc-fast afl-clang afl-gcc], []) + AC_CHECK_PROGS([AFL_CXX], [afl-clang-fast++ afl-g++-fast afl-clang++ afl-g++], []) + + if test "x$AFL_CC" = "x"; then + AC_MSG_ERROR([AFL++ compiler not found. Install AFL++ or disable fuzzing with --disable-fuzzing]) + fi + + # Test if AFL++ compiler works (version compatibility check) + AC_MSG_CHECKING([if $AFL_CC works]) + echo 'int main(){return 0;}' > conftest.c + if $AFL_CC conftest.c -o conftest >/dev/null 2>&1; then + AC_MSG_RESULT([yes]) + rm -f conftest.c conftest + else + AC_MSG_RESULT([no - version incompatibility detected]) + AC_MSG_WARN([AFL++ compiler has version issues, falling back to regular compiler with fuzzing flags]) + AFL_CC="$CC" + if test "x$AFL_CXX" != "x"; then + AFL_CXX="$CXX" + fi + rm -f conftest.c conftest + fi + + # Store AFL++ compilers for use in Makefile, but don't override CC during configure + AC_SUBST([AFL_CC]) + AC_SUBST([AFL_CXX]) + + # Add fuzzing-specific flags for the Makefile + FUZZ_CFLAGS="-g -O1 -fno-omit-frame-pointer" + FUZZ_CPPFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION" + AC_SUBST([FUZZ_CFLAGS]) + AC_SUBST([FUZZ_CPPFLAGS]) + + AC_MSG_NOTICE([Fuzzing enabled - instrumented binaries will be built with $AFL_CC]) +fi + dnl Check diff/patch programs AC_MSG_CHECKING(for diff program) DIFF=diff diff --git a/fuzz/README.md b/fuzz/README.md index 58ee4303..430c1546 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -40,6 +40,21 @@ The fuzz testing setup targets the main patch processing tools: ## Usage +### Building with Instrumentation (Recommended) + +For effective fuzzing, build patchutils with AFL++ instrumentation: + +```bash +# Configure with fuzzing support +./configure --enable-fuzzing + +# Build instrumented binaries +make + +# Now fuzzing will use instrumented binaries automatically +make fuzz-test +``` + ### Via Makefile (Recommended) ```bash @@ -49,7 +64,7 @@ make fuzz-help # Generate/update corpus (includes latest git diffs) make fuzz-corpus -# Quick 60-second test +# Quick 60-second test (uses instrumented binaries if available) make fuzz-test # Fuzz specific tools @@ -91,7 +106,21 @@ make fuzz-ci ## Integration -The fuzzer integrates with the existing testing infrastructure: +The fuzzer integrates with the autotools build system and existing testing infrastructure: + +### Autotools Integration +- `--enable-fuzzing` configure option enables AFL++ instrumentation +- Creates separate `fuzz-*` instrumented binaries alongside regular tools +- Automatic detection of instrumented vs. non-instrumented binaries +- Graceful fallback to regular binaries when instrumentation not available + +### Testing Integration - Uses existing test cases as seed corpus - Validates discovered issues against invariant tests - Generates regression tests for fixed bugs + +### Build System Features +- Conditional compilation based on `--enable-fuzzing` +- Proper compiler detection for AFL++ (afl-clang-fast, afl-gcc) +- Fuzzing-specific CFLAGS and preprocessor definitions +- Integration with existing Makefile targets diff --git a/fuzz/analyze_crashes.sh b/fuzz/analyze_crashes.sh index 05d13883..7e924cc2 100755 --- a/fuzz/analyze_crashes.sh +++ b/fuzz/analyze_crashes.sh @@ -7,6 +7,9 @@ SCRIPT_DIR="$(dirname "$0")" RESULTS_DIR="$SCRIPT_DIR/results" PROJECT_DIR="$(dirname "$SCRIPT_DIR")" +# Allow overriding binary directory (for instrumented builds) +FUZZ_BINDIR="${FUZZ_BINDIR:-$PROJECT_DIR/src}" + echo "Analyzing AFL++ fuzzing results..." if [ ! -d "$RESULTS_DIR" ]; then @@ -39,7 +42,12 @@ analyze_tool_crashes() { # Try to reproduce the crash echo " Reproducing crash..." - if timeout 10s "$PROJECT_DIR/src/$tool" < "$crash_file" >/dev/null 2>&1; then + # Use instrumented binary if available + local binary_path="$FUZZ_BINDIR/$tool" + if [ -f "$FUZZ_BINDIR/fuzz-$tool" ]; then + binary_path="$FUZZ_BINDIR/fuzz-$tool" + fi + if timeout 10s "$binary_path" < "$crash_file" >/dev/null 2>&1; then echo " ⚠ Crash not reproduced (may be timing-dependent)" else local exit_code=$? @@ -49,7 +57,7 @@ analyze_tool_crashes() { if command -v gdb >/dev/null 2>&1; then echo " Getting stack trace..." timeout 30s gdb -batch -ex run -ex bt -ex quit \ - --args "$PROJECT_DIR/src/$tool" < "$crash_file" 2>/dev/null | \ + --args "$binary_path" < "$crash_file" 2>/dev/null | \ grep -A 20 "Program received signal" | head -20 || true fi fi @@ -78,7 +86,12 @@ analyze_tool_crashes() { echo " ⚠ Large input file - may cause memory exhaustion" else echo " Testing with timeout..." - if timeout 5s "$PROJECT_DIR/src/$tool" < "$hang_file" >/dev/null 2>&1; then + # Use instrumented binary if available + local binary_path="$FUZZ_BINDIR/$tool" + if [ -f "$FUZZ_BINDIR/fuzz-$tool" ]; then + binary_path="$FUZZ_BINDIR/fuzz-$tool" + fi + if timeout 5s "$binary_path" < "$hang_file" >/dev/null 2>&1; then echo " ✓ Completed within timeout" else echo " ⚠ Still hangs or crashes" diff --git a/fuzz/run_fuzz.sh b/fuzz/run_fuzz.sh index ed7d4671..a89a8698 100755 --- a/fuzz/run_fuzz.sh +++ b/fuzz/run_fuzz.sh @@ -9,6 +9,9 @@ CORPUS_DIR="$SCRIPT_DIR/corpus" RESULTS_DIR="$SCRIPT_DIR/results" DICT_FILE="$SCRIPT_DIR/patch.dict" +# Allow overriding binary directory (for instrumented builds) +FUZZ_BINDIR="${FUZZ_BINDIR:-$PROJECT_DIR/src}" + # Default tool to fuzz TOOL="${1:-filterdiff}" @@ -23,22 +26,22 @@ fi # Ensure results directory exists mkdir -p "$RESULTS_DIR/$TOOL" -# Check if we have the normal binaries built -if [ ! -f "$PROJECT_DIR/src/$TOOL" ]; then - echo "Building patchutils tools..." - cd "$PROJECT_DIR" - make clean >/dev/null 2>&1 || true - - # Try to configure if not already done - if [ ! -f Makefile ]; then - if [ ! -f configure ]; then - ./bootstrap - fi - ./configure - fi +# Determine which binary to use (instrumented if available) +BINARY_PATH="" +INSTRUMENTED=false - make -j$(nproc) - cd - >/dev/null +if [ -f "$FUZZ_BINDIR/fuzz-$TOOL" ]; then + BINARY_PATH="$FUZZ_BINDIR/fuzz-$TOOL" + INSTRUMENTED=true + echo "Using instrumented binary: $BINARY_PATH" +elif [ -f "$FUZZ_BINDIR/$TOOL" ]; then + BINARY_PATH="$FUZZ_BINDIR/$TOOL" + echo "Warning: Using non-instrumented binary: $BINARY_PATH" + echo "For better fuzzing results, configure with --enable-fuzzing and rebuild" +else + echo "Binary not found: $TOOL" + echo "Please build patchutils first with 'make' or 'make --enable-fuzzing'" + exit 1 fi # AFL++ system configuration check @@ -72,8 +75,8 @@ case "$TOOL" in -x "$DICT_FILE" \ -t 5000 \ -m none \ - -n \ - -- "$PROJECT_DIR/src/filterdiff" --list + $( [ "$INSTRUMENTED" = "true" ] || echo "-n" ) \ + -- "$BINARY_PATH" --list ;; interdiff) # For interdiff, we need two patch files - use a simple approach @@ -86,8 +89,8 @@ case "$TOOL" in -x "$DICT_FILE" \ -t 5000 \ -m none \ - -n \ - -- "$PROJECT_DIR/src/interdiff" "$CORPUS_DIR/$PATCH1" @@ + $( [ "$INSTRUMENTED" = "true" ] || echo "-n" ) \ + -- "$BINARY_PATH" "$CORPUS_DIR/$PATCH1" @@ ;; rediff) # Test rediff (patch correction) @@ -97,8 +100,8 @@ case "$TOOL" in -x "$DICT_FILE" \ -t 5000 \ -m none \ - -n \ - -- "$PROJECT_DIR/src/rediff" + $( [ "$INSTRUMENTED" = "true" ] || echo "-n" ) \ + -- "$BINARY_PATH" ;; grepdiff) # Test grepdiff with a simple pattern @@ -108,8 +111,8 @@ case "$TOOL" in -x "$DICT_FILE" \ -t 5000 \ -m none \ - -n \ - -- "$PROJECT_DIR/src/grepdiff" ".*" + $( [ "$INSTRUMENTED" = "true" ] || echo "-n" ) \ + -- "$BINARY_PATH" ".*" ;; lsdiff) # Test lsdiff (list files in patch) @@ -119,8 +122,8 @@ case "$TOOL" in -x "$DICT_FILE" \ -t 5000 \ -m none \ - -n \ - -- "$PROJECT_DIR/src/lsdiff" + $( [ "$INSTRUMENTED" = "true" ] || echo "-n" ) \ + -- "$BINARY_PATH" ;; *) echo "Unknown tool: $TOOL"