diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..3e21407 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,17 @@ +name: test + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + - uses: actions/setup-go@v5 + with: + go-version: "1.22" + - name: test + run: go test -v ./... + diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..ff42552 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,29 @@ +name: Build and Release + +on: + release: + types: [created] + workflow_dispatch: + +jobs: + release: + runs-on: ubuntu-latest + strategy: + matrix: + goos: [linux] + goarch: ["386", amd64, arm64] + steps: + - uses: actions/checkout@v4 + - uses: wangyoucao577/go-release-action@v1.50 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + project_path: "./" + binary_name: "sandfly-entropyscan" + compress_assets: OFF + build_flags: -trimpath + pre_command: export CGO_ENABLED=0 + ldflags: -s -w + md5sum: FALSE + sha256sum: TRUE diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e6fa1d4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/sandfly-entropyscan +.idea/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..772dade --- /dev/null +++ b/Makefile @@ -0,0 +1,13 @@ +all: check fmt build + +build : + go build -x -trimpath ./ + +fmt : + gofmt -w *.go + +check : + go vet ./... + +clean : + rm sandfly-entropyscan || true diff --git a/build.sh b/build.sh deleted file mode 100755 index ee631c4..0000000 --- a/build.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -# Build script for sandfly-entropyscan -# -# sandfly-entropyscan is an entropy scanner to spot packed/encrypted binaries and processes on Linux and other platforms. -# -# MIT Licensed (c) 2019-2022 Sandfly Security -# https://www.sandflysecurity.com -# @SandflySecurity - -echo "Building for current OS." -go build -ldflags="-s -w" \ No newline at end of file diff --git a/fileutils/fileutils.go b/fileutils.go similarity index 51% rename from fileutils/fileutils.go rename to fileutils.go index cd413f7..97e7103 100644 --- a/fileutils/fileutils.go +++ b/fileutils.go @@ -1,5 +1,5 @@ // Sandfly entropyscan file utilities to calculate entropy, crypto hashes, etc -package fileutils +package main /* This utility will help find packed or encrypted files or processes on a Linux system by calculating the entropy @@ -41,8 +41,8 @@ import ( "crypto/sha1" "crypto/sha256" "crypto/sha512" - "encoding/binary" "encoding/hex" + "errors" "fmt" "io" "math" @@ -60,99 +60,137 @@ const ( constMagicNumElf = "7f454c46" ) -// Pass in a path and we'll see if the magic number is Linux ELF type. -func IsElfType(path string) (isElf bool, err error) { - var hexData [constMagicNumRead]byte +type ErrNotRegularFile struct { + Path string +} + +func (e *ErrNotRegularFile) Error() string { + return fmt.Sprintf("file '%s' is not a regular file", e.Path) +} + +func NewErrNotRegularFile(path string) *ErrNotRegularFile { + return &ErrNotRegularFile{Path: path} +} + +type ErrFileTooLarge struct { + Path string + Size int64 + Max int64 +} + +func (e *ErrFileTooLarge) Error() string { + return fmt.Sprintf("file size of '%s' is too large (%d bytes) to calculate entropy (max allowed: %d bytes)", + e.Path, e.Size, e.Max) +} + +func NewErrFileTooLarge(path string, size int64) *ErrFileTooLarge { + return &ErrFileTooLarge{Path: path, Size: size, Max: constMaxFileSize} +} + +var ErrNoPath = fmt.Errorf("no path provided") +var elfType []byte + +func init() { + var err error + if elfType, err = hex.DecodeString(constMagicNumElf); err != nil { + // this should never happen + panic(fmt.Errorf("couldn't decode ELF magic number constant: %w", err)) + } + if len(elfType) > constMagicNumRead { + // this should never happen + panic(fmt.Errorf("elf magic number string is longer than magic number read bytes")) + } +} + +func preCheckFilepath(path string) (*os.File, int64, error) { if path == "" { - return false, fmt.Errorf("must provide a path to file to get ELF type") + return nil, 0, ErrNoPath } f, err := os.Open(path) if err != nil { - return false, err + return nil, 0, fmt.Errorf("couldn't open '%s': %w", path, err) } - defer f.Close() - fStat, err := f.Stat() if err != nil { - return false, err + if f != nil { + _ = f.Close() + } + return f, 0, err } - // Not a regular file, so can't be ELF if !fStat.Mode().IsRegular() { - return false, nil + _ = f.Close() + return f, 0, NewErrNotRegularFile(path) } - // Too small to be ELF - if fStat.Size() < constMagicNumRead { - return false, nil + if fStat.Size() == 0 { + _ = f.Close() + return f, fStat.Size(), fmt.Errorf("file '%s' is zero size", path) } - err = binary.Read(f, binary.LittleEndian, &hexData) - if err != nil { - return false, err - } + return f, fStat.Size(), nil +} - elfType, err := hex.DecodeString(constMagicNumElf) - if err != nil { +// IsElfType will reead the magic bytes from the passed file and determine if it is an ELF file. +func IsElfType(path string) (isElf bool, err error) { + var fSize int64 + var f io.ReadCloser + + if f, fSize, err = preCheckFilepath(path); err != nil { return false, err } - if len(elfType) > constMagicNumRead { - return false, fmt.Errorf("elf magic number string is longer than magic number read bytes") + + defer func() { + _ = f.Close() + }() + + // Too small to be ELF + if fSize < constMagicNumRead { + return false, fmt.Errorf("file '%s' is too small to be an ELF file", path) } - if bytes.Equal(hexData[:len(elfType)], elfType) { - return true, nil + var hexData [constMagicNumRead]byte + + var n int + if n, err = f.Read(hexData[:]); err != nil { + return false, fmt.Errorf("couldn't read from '%s': %w", path, err) + } + if n != constMagicNumRead { + return false, fmt.Errorf("couldn't read enough bytes from '%s'", path) } - return false, nil + return bytes.Equal(hexData[:len(elfType)], elfType), nil } -// Calculates entropy of a file. +// Entropy calculates entropy of a file. func Entropy(path string) (entropy float64, err error) { var size int64 + var f io.ReadCloser - if path == "" { - return entropy, fmt.Errorf("must provide a path to file to get entropy") - } - - f, err := os.Open(path) - if err != nil { - return 0, fmt.Errorf("couldn't open path (%s) to get entropy: %v", path, err) - } - defer f.Close() - - fStat, err := f.Stat() - if err != nil { + if f, size, err = preCheckFilepath(path); err != nil { return 0, err } - if !fStat.Mode().IsRegular() { - return 0, fmt.Errorf("file (%s) is not a regular file to calculate entropy", path) - } - - size = fStat.Size() - // Zero sized file is zero entropy. - if size == 0 { - return 0, nil - } + defer func() { + _ = f.Close() + }() if size > int64(constMaxFileSize) { - return 0, fmt.Errorf("file size (%d) is too large to calculate entropy (max allowed: %d)", - size, int64(constMaxFileSize)) + return 0, NewErrFileTooLarge(path, size) } dataBytes := make([]byte, constMaxEntropyChunk) byteCounts := make([]int, 256) for { - numBytesRead, err := f.Read(dataBytes) - if err == io.EOF { + numBytesRead, readErr := f.Read(dataBytes) + if errors.Is(readErr, io.EOF) { break } - if err != nil { - return 0, err + if readErr != nil { + return 0, readErr } // For each byte of the data that was read, increment the count @@ -174,41 +212,26 @@ func Entropy(path string) (entropy float64, err error) { return math.Round(entropy*100) / 100, nil } -// Generates MD5 hash of a file +// HashMD5 calculates the MD5 checksum of a file. func HashMD5(path string) (hash string, err error) { - if path == "" { - return hash, fmt.Errorf("must provide a path to file to hash") - } - - f, err := os.Open(path) - if err != nil { - return hash, fmt.Errorf("couldn't open path (%s): %v", path, err) - } - defer f.Close() - - fStat, err := f.Stat() - if err != nil { + var fSize int64 + var f io.ReadCloser + if f, fSize, err = preCheckFilepath(path); err != nil { return hash, err } - if !fStat.Mode().IsRegular() { - return hash, fmt.Errorf("file (%s) is not a regular file to calculate hash", path) - } - - // Zero sized file is no hash. - if fStat.Size() == 0 { - return hash, nil - } + defer func() { + _ = f.Close() + }() - if fStat.Size() > int64(constMaxFileSize) { - return hash, fmt.Errorf("file size (%d) is too large to calculate hash (max allowed: %d)", - fStat.Size(), int64(constMaxFileSize)) + if fSize > int64(constMaxFileSize) { + return hash, NewErrFileTooLarge(path, fSize) } hashMD5 := md5.New() _, err = io.Copy(hashMD5, f) if err != nil { - return hash, fmt.Errorf("couldn't read path (%s) to get MD5 hash: %v", path, err) + return hash, fmt.Errorf("couldn't read path (%s) to get MD5 hash: %w", path, err) } hash = hex.EncodeToString(hashMD5.Sum(nil)) @@ -216,41 +239,27 @@ func HashMD5(path string) (hash string, err error) { return hash, nil } -// Generates SHA1 hash of a file +// HashSHA1 calculates the SHA1 checksum of a file. func HashSHA1(path string) (hash string, err error) { - if path == "" { - return hash, fmt.Errorf("must provide a path to file to hash") - } + var fSize int64 + var f io.ReadCloser - f, err := os.Open(path) - if err != nil { - return hash, fmt.Errorf("couldn't open path (%s): %v", path, err) - } - defer f.Close() - - fStat, err := f.Stat() - if err != nil { + if f, fSize, err = preCheckFilepath(path); err != nil { return hash, err } - if !fStat.Mode().IsRegular() { - return hash, fmt.Errorf("file (%s) is not a regular file to calculate hash", path) - } - - // Zero sized file is no hash. - if fStat.Size() == 0 { - return hash, nil - } + defer func() { + _ = f.Close() + }() - if fStat.Size() > int64(constMaxFileSize) { - return hash, fmt.Errorf("file size (%d) is too large to calculate hash (max allowed: %d)", - fStat.Size(), int64(constMaxFileSize)) + if fSize > int64(constMaxFileSize) { + return hash, NewErrFileTooLarge(path, fSize) } hashSHA1 := sha1.New() _, err = io.Copy(hashSHA1, f) if err != nil { - return hash, fmt.Errorf("couldn't read path (%s) to get SHA1 hash: %v", path, err) + return hash, fmt.Errorf("couldn't read path (%s) to get SHA1 hash: %w", path, err) } hash = hex.EncodeToString(hashSHA1.Sum(nil)) @@ -258,41 +267,27 @@ func HashSHA1(path string) (hash string, err error) { return hash, nil } -// Generates SHA256 hash of a file +// HashSHA256 calculates the SHA256 checksum of a file. func HashSHA256(path string) (hash string, err error) { - if path == "" { - return hash, fmt.Errorf("must provide a path to file to hash") - } + var fSize int64 + var f io.ReadCloser - f, err := os.Open(path) - if err != nil { - return hash, fmt.Errorf("couldn't open path (%s): %v", path, err) - } - defer f.Close() - - fStat, err := f.Stat() - if err != nil { + if f, fSize, err = preCheckFilepath(path); err != nil { return hash, err } - if !fStat.Mode().IsRegular() { - return hash, fmt.Errorf("file (%s) is not a regular file to calculate hash", path) - } - - // Zero sized file is no hash. - if fStat.Size() == 0 { - return hash, nil - } + defer func() { + _ = f.Close() + }() - if fStat.Size() > int64(constMaxFileSize) { - return hash, fmt.Errorf("file size (%d) is too large to calculate hash (max allowed: %d)", - fStat.Size(), int64(constMaxFileSize)) + if fSize > int64(constMaxFileSize) { + return hash, NewErrFileTooLarge(path, fSize) } hashSHA256 := sha256.New() _, err = io.Copy(hashSHA256, f) if err != nil { - return hash, fmt.Errorf("couldn't read path (%s) to get SHA256 hash: %v", path, err) + return hash, fmt.Errorf("couldn't read '%s' to get SHA256 hash: %w", path, err) } hash = hex.EncodeToString(hashSHA256.Sum(nil)) @@ -300,41 +295,27 @@ func HashSHA256(path string) (hash string, err error) { return hash, nil } -// Generates SHA512 hash of a file +// HashSHA512 calculates the SHA512 checksum of a file. func HashSHA512(path string) (hash string, err error) { - if path == "" { - return hash, fmt.Errorf("must provide a path to file to hash") - } + var fSize int64 + var f io.ReadCloser - f, err := os.Open(path) - if err != nil { - return hash, fmt.Errorf("couldn't open path (%s): %v", path, err) - } - defer f.Close() - - fStat, err := f.Stat() - if err != nil { + if f, fSize, err = preCheckFilepath(path); err != nil { return hash, err } - if !fStat.Mode().IsRegular() { - return hash, fmt.Errorf("file (%s) is not a regular file to calculate hash", path) - } - - // Zero sized file is no hash. - if fStat.Size() == 0 { - return hash, nil - } + defer func() { + _ = f.Close() + }() - if fStat.Size() > int64(constMaxFileSize) { - return hash, fmt.Errorf("file size (%d) is too large to calculate hash (max allowed: %d)", - fStat.Size(), int64(constMaxFileSize)) + if fSize > int64(constMaxFileSize) { + return hash, NewErrFileTooLarge(path, fSize) } hashSHA512 := sha512.New() _, err = io.Copy(hashSHA512, f) if err != nil { - return hash, fmt.Errorf("couldn't read path (%s) to get SHA512 hash: %v", path, err) + return hash, fmt.Errorf("couldn't read path (%s) to get SHA512 hash: %w", path, err) } hash = hex.EncodeToString(hashSHA512.Sum(nil)) diff --git a/sandfly-entropyscan.go b/sandfly-entropyscan.go index 9d6f0d3..4cb0b6e 100644 --- a/sandfly-entropyscan.go +++ b/sandfly-entropyscan.go @@ -36,20 +36,24 @@ Author: @SandflySecurity */ import ( + "bytes" + "encoding/json" + "errors" "flag" "fmt" "log" "os" - "path" "path/filepath" + "reflect" + "runtime" "strconv" - - "github.com/sandflysecurity/sandfly-entropyscan/fileutils" + "strings" + "sync" ) const ( // constVersion Version - constVersion = "1.1.1" + constVersion = "1.2.0" // constProcDir default /proc dir for processes. constProcDir = "/proc" // constDelimeterDefault default delimiter for CSV output. @@ -60,214 +64,474 @@ const ( constMaxPID = 4194304 ) -type fileData struct { - path string - name string - entropy float64 - elf bool - hash hashes +type csvHeaderStructMapping struct { + header string // key in CSV header + structTag string // borrow JSON struct tag for CSV } -type hashes struct { - md5 string - sha1 string - sha256 string - sha512 string +type csvSchema struct { + keys map[int]csvHeaderStructMapping + delim string } -func main() { - var filePath string - var dirPath string - var delimChar string - var entropyMaxVal float64 - var elfOnly bool - var procOnly bool - var csvOutput bool - var version bool - - flag.StringVar(&filePath, "file", "", "full path to a single file to analyze") - flag.StringVar(&dirPath, "dir", "", "directory name to analyze") - flag.StringVar(&delimChar, "delim", constDelimeterDefault, "delimeter for CSV output") - flag.Float64Var(&entropyMaxVal, "entropy", 0, "show any file with entropy greater than or equal to this value (0.0 - 8.0 max 8.0, default is 0)") - flag.BoolVar(&elfOnly, "elf", false, "only check ELF executables") - flag.BoolVar(&procOnly, "proc", false, "check running processes") - flag.BoolVar(&csvOutput, "csv", false, "output results in CSV format (filename, path, entropy, elf_file [true|false], MD5, SHA1, SHA256, SHA512)") - flag.BoolVar(&version, "version", false, "show version and exit") - flag.Parse() - - if version { - fmt.Printf("sandfly-entropyscan Version %s\n", constVersion) - fmt.Printf("Copyright (c) 2019-2022 Sandlfy Security - www.sandflysecurity.com\n\n") - os.Exit(0) +func (csv csvSchema) header() []byte { + var buf = new(bytes.Buffer) + for i := 0; i < len(csv.keys); i++ { + _, _ = buf.WriteString(csv.keys[i].header) + if i < len(csv.keys)-1 { + _, _ = buf.WriteString(csv.delim) + } } + return buf.Bytes() +} - if entropyMaxVal > 8 { - log.Fatal("max entropy value is 8.0") +var ( + ErrUnsupportedType = errors.New("unsupported type") + ErrNilPointer = errors.New("nil pointer") +) + +func (csv csvSchema) parse(in any) ([]byte, error) { + var buf = new(bytes.Buffer) + write := func(s string) { _, _ = buf.WriteString(s) } + ref := reflect.ValueOf(in) + if ref.Kind() == reflect.Ptr && !ref.IsNil() { + ref = ref.Elem() } - if entropyMaxVal < 0 { - log.Fatal("min entropy value is 0.0") + if ref.Kind() == reflect.Ptr && ref.IsNil() { + return nil, ErrNilPointer } - if procOnly { - if os.Geteuid() != 0 { - log.Fatalf("process checking option requires UID/EUID 0 (root) to run") + var finErr error + +outerIter: + for i := 0; i < len(csv.keys); i++ { + var field = reflect.ValueOf(nil) + iter: + for j := 0; j < ref.NumField(); j++ { + structTag := ref.Type().Field(j).Tag.Get("json") + target := csv.keys[i].structTag + if strings.Contains(target, ".") { + target = strings.Split(target, ".")[0] + } + switch structTag { + case target: + field = ref.Field(j) + if field.Kind() == reflect.Ptr && !field.IsNil() { + field = field.Elem() + } + break iter + default: + } } - // This will do a PID bust of all PID range to help detect hidden PIDs. - pidPaths, err := genPIDExePaths() - if err != nil { - log.Fatalf("error generating PID list: %v\n", err) + + if (field.Kind() == reflect.Pointer || field.Kind() == reflect.Interface) && field.IsNil() { + continue } - for pid := 0; pid < len(pidPaths); pid++ { - // Only check elf files which should be all these will be anyway. - fileInfo, err := checkFilePath(pidPaths[pid], true, entropyMaxVal) - // anything that is not an error is a valid /proc/*/exe link we could see and process. We will analyze it. - if err == nil { - if fileInfo.entropy >= entropyMaxVal { - printResults(fileInfo, csvOutput, delimChar) - } + + switch field.Kind() { + case reflect.String: + write(field.String()) + case reflect.Float64: + write(strconv.FormatFloat(field.Float(), 'f', 2, 64)) + case reflect.Float32: + write(strconv.FormatFloat(field.Float(), 'f', 2, 32)) + case reflect.Bool: + write(strconv.FormatBool(field.Bool())) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + write(strconv.Itoa(int(field.Int()))) + case reflect.Struct: + targetTag := csv.keys[i].structTag + if strings.Contains(targetTag, ".") { + targetTag = strings.Split(targetTag, ".")[1] } + write(field.FieldByName(targetTag).String()) + case reflect.Ptr: + finErr = ErrUnsupportedType + default: + finErr = fmt.Errorf("csv: %w: %s", ErrUnsupportedType, field.Kind().String()) + } + + if i < len(csv.keys)-1 { + write(csv.delim) + } + + if i == len(csv.keys)-1 { + write("\n") + } + + if finErr != nil { + break outerIter } - os.Exit(0) } - if filePath != "" { - fileInfo, err := checkFilePath(filePath, elfOnly, entropyMaxVal) + return buf.Bytes(), finErr +} + +// (filename, path, entropy, elf_file [true|false], MD5, SHA1, SHA256, SHA512) +var defCSVHeader = csvSchema{ + keys: map[int]csvHeaderStructMapping{ + 0: {"filename", "name"}, + 1: {"path", "path"}, + 2: {"entropy", "entropy"}, + 3: {"elf_file", "elf"}, + 4: {"md5", "checksums.MD5"}, + 5: {"sha1", "checksums.SHA1"}, + 6: {"sha256", "checksums.SHA256"}, + 7: {"sha512", "checksums.SHA512"}, + }, + delim: constDelimeterDefault, +} + +type Results struct { + Files + csvSchema csvSchema +} + +func NewResults() *Results { + return &Results{Files: make(Files, 0), csvSchema: defCSVHeader} +} + +func (r *Results) WithDelimiter(delim string) *Results { + r.csvSchema.delim = delim + return r +} + +func (r *Results) Add(f *File) { + r.Files = append(r.Files, f) +} + +func (r *Results) MarshalCSV() ([]byte, error) { + buf := new(bytes.Buffer) + write := func(data []byte) { _, _ = buf.Write(data) } + write(r.csvSchema.header()) + write([]byte("\n")) + for _, file := range r.Files { + entry, err := r.csvSchema.parse(file) if err != nil { - log.Fatalf("error processing file (%s): %v\n", filePath, err) + return nil, err } + write(entry) + } + return buf.Bytes(), nil +} - if fileInfo.entropy >= entropyMaxVal { - printResults(fileInfo, csvOutput, delimChar) - } +type Files []*File + +type File struct { + Path string `json:"path"` + Name string `json:"name"` + Entropy float64 `json:"entropy"` + IsELF bool `json:"elf"` + Checksums *Checksums `json:"checksums"` +} + +type Checksums struct { + MD5 string `json:"md5"` + SHA1 string `json:"sha1"` + SHA256 string `json:"sha256"` + SHA512 string `json:"sha512"` +} + +type config struct { + filePath string + dirPath string + delimChar string + entropyMaxVal float64 + elfOnly bool + procOnly bool + csvOutput bool + jsonOutput bool + printInterimResults bool + outputFile string + version bool + sumMD5 bool + sumSHA1 bool + sumSHA256 bool + sumSHA512 bool + results *Results +} + +var cfgOnce sync.Once + +func newConfigFromFlags() *config { + cfg := new(config) + + cfgOnce.Do(func() { + flag.StringVar(&cfg.filePath, "file", "", "full path to a single file to analyze") + flag.StringVar(&cfg.dirPath, "dir", "", "directory name to analyze") + flag.StringVar(&cfg.delimChar, "delim", constDelimeterDefault, "delimeter for CSV output") + flag.StringVar(&cfg.outputFile, "output", "", "output file to write results to (default stdout) (only json and csv formats supported)") + flag.Float64Var(&cfg.entropyMaxVal, "entropy", 0, "show any file with entropy greater than or equal to this value (0.0 - 8.0 max 8.0, default is 0)") + + flag.BoolVar(&cfg.elfOnly, "elf", false, "only check ELF executables") + flag.BoolVar(&cfg.procOnly, "proc", false, "check running processes") + flag.BoolVar(&cfg.csvOutput, "csv", false, "output results in CSV format (filename, path, entropy, elf_file [true|false], MD5, SHA1, SHA256, SHA512)") + flag.BoolVar(&cfg.jsonOutput, "json", false, "output results in JSON format") + flag.BoolVar(&cfg.printInterimResults, "print", false, "print interim results to stdout even if output file is specified") + flag.BoolVar(&cfg.version, "version", false, "show version and exit") + flag.BoolVar(&cfg.sumMD5, "md5", true, "calculate and show MD5 checksum of file(s)") + flag.BoolVar(&cfg.sumSHA1, "sha1", true, "calculate and show SHA1 checksum of file(s)") + flag.BoolVar(&cfg.sumSHA256, "sha256", true, "calculate and show SHA256 checksum of file(s)") + flag.BoolVar(&cfg.sumSHA512, "sha512", true, "calculate and show SHA512 checksum of file(s)") + + flag.Parse() + }) + + switch { + case cfg.version: + fmt.Printf("sandfly-entropyscan Version %s\n", constVersion) + fmt.Printf("Copyright (c) 2019-2022 Sandlfy Security - www.sandflysecurity.com\n\n") os.Exit(0) + case cfg.entropyMaxVal > 8: + log.Fatal("max entropy value is 8.0") + case cfg.entropyMaxVal < 0: + log.Fatal("min entropy value is 0.0") + default: + // proceed } - if dirPath != "" { + return cfg +} + +func main() { + cfg := newConfigFromFlags() + + if cfg.csvOutput || cfg.jsonOutput { + cfg.results = NewResults() + if cfg.delimChar != constDelimeterDefault { + cfg.results = cfg.results.WithDelimiter(cfg.delimChar) + } + } + + if !cfg.csvOutput && !cfg.jsonOutput { + cfg.printInterimResults = true + } + + if cfg.csvOutput && cfg.jsonOutput { + log.Fatal("csv and json output options are mutually exclusive") + } + + defer func() { + var res []byte + switch { + case cfg.csvOutput: + var err error + if res, err = cfg.results.MarshalCSV(); err != nil { + log.Fatal(err.Error()) + } + case cfg.jsonOutput: + var err error + if res, err = json.Marshal(cfg.results); err != nil { + log.Fatal(err.Error()) + } + default: + } + if len(res) > 0 { + switch { + case cfg.outputFile != "": + if err := os.WriteFile(cfg.outputFile, res, 0644); err != nil { + log.Fatal(err.Error()) + } + default: + _, _ = os.Stdout.Write(res) + } + } + }() + + switch { + case cfg.procOnly: + if runtime.GOOS == "windows" { + log.Fatalf("process checking option is not supported on Windows") + } + if os.Geteuid() != 0 { + log.Fatalf("process checking option requires UID/EUID 0 (root) to run") + } + + results := NewResults() + + for pid := constMinPID; pid < constMaxPID; pid++ { + procfsTarget := filepath.Join(constProcDir, strconv.Itoa(pid), "/exe") + // Only check elf files which should be all these will be anyway. + file, err := cfg.checkFilePath(procfsTarget) + // anything that is not an error is a valid /proc/*/exe link we could see and process. We will analyze it. + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + log.Printf("(!) could not read /proc/%d/exe: %s", pid, err) + continue + } + if (file.Entropy < cfg.entropyMaxVal) || (!file.IsELF && cfg.elfOnly) { + continue + } + + results.Add(file) + cfg.printResults(file) + } + case cfg.filePath != "": + fileInfo, err := cfg.checkFilePath(cfg.filePath) + if err != nil { + log.Fatalf("error processing file (%s): %v\n", cfg.filePath, err) + } + if fileInfo.Entropy >= cfg.entropyMaxVal { + cfg.printResults(fileInfo) + } + case cfg.dirPath != "": var search = func(filePath string, info os.FileInfo, err error) error { + dir, _ := filepath.Split(filePath) if err != nil { - log.Fatalf("error walking directory (%s) inside search function: %v\n", filePath, err) + return fmt.Errorf("error walking directory (%s): %v\n", dir, err) } // If info comes back as nil we don't want to read it or we panic. - if info != nil { - // if not a directory, then check it for a file we want. - if !info.IsDir() { - // Only check regular files. Checking devices, etc. won't work. - if info.Mode().IsRegular() { - fileInfo, err := checkFilePath(filePath, elfOnly, entropyMaxVal) - if err != nil { - log.Fatalf("error processing file (%s): %v\n", filePath, err) - } - - if fileInfo.entropy >= entropyMaxVal { - printResults(fileInfo, csvOutput, delimChar) - } - } - } + if info == nil { + return nil + } + if info.IsDir() { + return nil } + // Only check regular files. Checking devices, etc. won't work. + if !info.Mode().IsRegular() { + return nil + } + fileInfo, err := cfg.checkFilePath(filePath) + if err != nil { + return fmt.Errorf("error processing file (%s): %v\n", filePath, err) + } + + if fileInfo.Entropy >= cfg.entropyMaxVal { + cfg.printResults(fileInfo) + } + return nil } - err := filepath.Walk(dirPath, search) + err := filepath.Walk(cfg.dirPath, search) if err != nil { - log.Fatalf("error walking directory (%s): %v\n", dirPath, err) + log.Fatalf("error walking directory (%s): %v\n", cfg.dirPath, err) } - os.Exit(0) } } -// Prints results -func printResults(fileInfo fileData, csvFormat bool, delimChar string) { - - if !csvFormat { - fmt.Printf("filename: %s\npath: %s\nentropy: %.2f\nelf: %v\nmd5: %s\nsha1: %s\nsha256: %s\nsha512: %s\n\n", - fileInfo.name, - fileInfo.path, - fileInfo.entropy, - fileInfo.elf, - fileInfo.hash.md5, - fileInfo.hash.sha1, - fileInfo.hash.sha256, - fileInfo.hash.sha512) - } else { - fmt.Printf("%s%s%s%s%.2f%s%v%s%s%s%s%s%s%s%s\n", - fileInfo.name, - delimChar, - fileInfo.path, - delimChar, - fileInfo.entropy, - delimChar, - fileInfo.elf, - delimChar, - fileInfo.hash.md5, - delimChar, - fileInfo.hash.sha1, - delimChar, - fileInfo.hash.sha256, - delimChar, - fileInfo.hash.sha512) +func (cfg *config) printResults(file *File) { + switch { + case (cfg.csvOutput || cfg.jsonOutput) && cfg.outputFile == "": + cfg.results.Add(file) + case (cfg.csvOutput || cfg.jsonOutput) && cfg.outputFile != "": + cfg.results.Add(file) + fallthrough + case cfg.printInterimResults: + format := "filename: %s\npath: %s\nentropy: %.2f\nelf: %v\n" + str := fmt.Sprintf(format, + file.Name, + file.Path, + file.Entropy, + file.IsELF, + ) + if cfg.sumMD5 { + str += "md5: " + file.Checksums.MD5 + "\n" + } + if cfg.sumSHA1 { + str += "sha1: " + file.Checksums.SHA1 + "\n" + } + if cfg.sumSHA256 { + str += "sha256: " + file.Checksums.SHA256 + "\n" + } + if cfg.sumSHA512 { + str += "sha512: " + file.Checksums.SHA512 + "\n" + } + fmt.Print(str + "\n") } } -func checkFilePath(filePath string, elfOnly bool, entropyMaxVal float64) (fileInfo fileData, err error) { - isElfType, err := fileutils.IsElfType(filePath) +type hasher struct { + enabled bool + target *string + f func(string) (string, error) +} + +func (h hasher) sum(file *File) error { + if !h.enabled { + return nil + } + var res string + var err error + if res, err = h.f(file.Path); err == nil { + *h.target = res + } if err != nil { - return fileInfo, err + return fmt.Errorf("error calculating checksum for file (%s): %w", file.Path, err) } - _, fileName := filepath.Split(filePath) + return nil +} - fileInfo.path = filePath - fileInfo.name = fileName - fileInfo.elf = isElfType - fileInfo.entropy = -1 +func (cfg *config) runEnabledHashers(file *File) error { + wg := new(sync.WaitGroup) - // If they only want Linux ELFs. - if elfOnly && isElfType { - entropy, err := fileutils.Entropy(filePath) - if err != nil { - log.Fatalf("error calculating entropy for file (%s): %v\n", filePath, err) - } - fileInfo.entropy = entropy + if file.Checksums == nil { + file.Checksums = new(Checksums) } - // They want entropy on all files. - if !elfOnly { - entropy, err := fileutils.Entropy(filePath) - if err != nil { - log.Fatalf("error calculating entropy for file (%s): %v\n", filePath, err) + + do := []hasher{ + {cfg.sumMD5, &file.Checksums.MD5, HashMD5}, + {cfg.sumSHA1, &file.Checksums.SHA1, HashSHA1}, + {cfg.sumSHA256, &file.Checksums.SHA256, HashSHA256}, + {cfg.sumSHA512, &file.Checksums.SHA512, HashSHA512}, + } + wg.Add(len(do)) + var errs = make(chan error, len(do)) + for _, v := range do { + go func(chk hasher, fi *File, vwg *sync.WaitGroup) { + errs <- chk.sum(fi) + vwg.Done() + }(v, file, wg) + } + wg.Wait() + close(errs) + var errsSlice = make([]error, 0, len(do)) + for e := range errs { + if e != nil { + errsSlice = append(errsSlice, e) } - fileInfo.entropy = entropy + } + return errors.Join(errsSlice...) +} + +func (cfg *config) checkFilePath(filePath string) (file *File, err error) { + file = new(File) + file.Checksums = new(Checksums) + + file.Path = filePath + + if file.IsELF, err = IsElfType(filePath); err != nil { + return file, err } - if fileInfo.entropy >= entropyMaxVal { - md5, err := fileutils.HashMD5(filePath) - if err != nil { - log.Fatalf("error calculating MD5 hash for file (%s): %v\n", filePath, err) - } - sha1, err := fileutils.HashSHA1(filePath) - if err != nil { - log.Fatalf("error calculating SHA1 hash for file (%s): %v\n", filePath, err) - } - sha256, err := fileutils.HashSHA256(filePath) - if err != nil { - log.Fatalf("error calculating SHA256 hash for file (%s): %v\n", filePath, err) + // handle procfs links + if _, file.Name = filepath.Split(filePath); file.Name == "exe" { + if file.Name, err = os.Readlink(filePath); err != nil { + log.Printf("(!) could not read link (%s): %s\n", filePath, err) + file.Name = "unknown" + } else { + file.Name = filepath.Base(file.Name) } - sha512, err := fileutils.HashSHA512(filePath) - if err != nil { - log.Fatalf("error calculating SHA512 hash for file (%s): %v\n", filePath, err) - } - fileInfo.hash.md5 = md5 - fileInfo.hash.sha1 = sha1 - fileInfo.hash.sha256 = sha256 - fileInfo.hash.sha512 = sha512 } - return fileInfo, nil -} - -func genPIDExePaths() (pidPaths []string, err error) { + switch { + case cfg.elfOnly && !file.IsELF: + return &File{}, nil + case !cfg.elfOnly || (cfg.elfOnly && file.IsELF): + var entropy float64 + if entropy, err = Entropy(filePath); err != nil { + log.Fatalf("error calculating entropy for file (%s): %v\n", filePath, err) + } + file.Entropy = entropy + } - for pid := constMinPID; pid < constMaxPID; pid++ { - pidPaths = append(pidPaths, path.Join(constProcDir, strconv.Itoa(pid), "/exe")) + if file.Entropy < cfg.entropyMaxVal { + return file, nil } - return pidPaths, nil + err = cfg.runEnabledHashers(file) + + return file, err } diff --git a/sandfly-entropyscan_test.go b/sandfly-entropyscan_test.go new file mode 100644 index 0000000..eba2b86 --- /dev/null +++ b/sandfly-entropyscan_test.go @@ -0,0 +1,359 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +func TestCsvSchemaHeader(t *testing.T) { + csv := csvSchema{ + keys: map[int]csvHeaderStructMapping{ + 0: {"filename", "name"}, + 1: {"path", "path"}, + }, + delim: ",", + } + + expected := []byte("filename,path") + result := csv.header() + + if !strings.EqualFold(string(result), string(expected)) { + t.Errorf("expected %s but got %s", string(expected), string(result)) + } +} + +func TestResultChecksums(t *testing.T) { + f, err := os.CreateTemp(t.TempDir(), "yeet") + if err != nil { + t.Errorf("\n\nunexpected error:\n %v", err) + } + if _, err = f.WriteString("yeeterson mcgee"); err != nil { + t.Errorf("\n\nunexpected error:\n %v", err) + } + path := f.Name() + if err = f.Close(); err != nil { + t.Errorf("\n\nunexpected error:\n %v", err) + } + + yeet := &File{ + Path: path, + Name: "yeet", + Entropy: 0.5, + IsELF: false, + Checksums: new(Checksums), + } + + results := NewResults() + + cfg := newConfigFromFlags() + cfg.sumMD5 = true + cfg.sumSHA1 = true + cfg.sumSHA256 = true + cfg.sumSHA512 = true + if err = cfg.runEnabledHashers(yeet); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + for i, h := range []string{yeet.Checksums.MD5, yeet.Checksums.SHA1, yeet.Checksums.SHA256, yeet.Checksums.SHA512} { + chkName := "md5" + switch i { + case 1: + chkName = "sha1" + case 2: + chkName = "sha256" + case 3: + chkName = "sha512" + } + if strings.TrimSpace(h) == "" { + t.Errorf("expected %s hash but got empty string", chkName) + } + t.Logf("%s: %s", chkName, h) + } + + results.Add(yeet) + + expected := []byte("filename,path,entropy,elf_file,md5,sha1,sha256,sha512\n" + + "yeet," + path + "," + "0.50,false," + yeet.Checksums.MD5 + "," + + yeet.Checksums.SHA1 + "," + yeet.Checksums.SHA256 + "," + + yeet.Checksums.SHA512 + "\n", + ) + + result, err := results.MarshalCSV() + + if err != nil { + t.Errorf("\n\nunexpected error:\n %v", err) + } + + if !strings.EqualFold(string(result), string(expected)) { + t.Errorf("\n\nexpected:\n"+ + "%s \n"+ + "got: \n"+ + "%s\n\n", + string(expected), + string(result), + ) + } +} + +func TestResultsCustomSchema(t *testing.T) { + results := NewResults() + results.Add(&File{ + Path: "test/path", + Name: "testfile", + Checksums: new(Checksums), + }) + results.csvSchema = csvSchema{ + keys: map[int]csvHeaderStructMapping{ + 0: {"filename", "name"}, + 1: {"path", "path"}, + }, + delim: ";", + } + + expected := []byte("filename;path\n" + + "testfile;test/path\n") + result, err := results.MarshalCSV() + + if err != nil { + t.Errorf("\n\nunexpected error:\n %v", err) + } + + if !strings.EqualFold(string(result), string(expected)) { + t.Errorf("\n\nexpected:\n"+ + "%s \n"+ + "got: \n"+ + "%s\n\n", string(expected), string(result)) + } +} + +func TestResultsAdd(t *testing.T) { + results := NewResults() + results.Add(&File{ + Path: "test/path", + Name: "testfile", + Checksums: new(Checksums), + }) + + if len(results.Files) != 1 { + t.Errorf("expected length of 1 but got %d", len(results.Files)) + } +} + +func TestParseHappyPath(t *testing.T) { + csv := csvSchema{ + keys: map[int]csvHeaderStructMapping{ + 0: {"filename", "name"}, + 1: {"path", "path"}, + }, + delim: ",", + } + + in := File{ + Path: "test/path", + Name: "testfile", + } + + expected := []byte("testfile,test/path\n") + result, err := csv.parse(in) + + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + + if !strings.EqualFold(string(result), string(expected)) { + t.Errorf("Expected %s but got %s", string(expected), string(result)) + } +} + +func TestParseUnsupportedType(t *testing.T) { + csv := csvSchema{ + keys: map[int]csvHeaderStructMapping{ + 0: {"filename", "name"}, + 1: {"path", "path"}, + }, + delim: ",", + } + + in := struct { + Path complex128 + Name string + }{ + Path: complex128(1 + 2i), + Name: "testfile", + } + + _, err := csv.parse(in) + + if !errors.Is(err, ErrUnsupportedType) { + t.Errorf("Expected ErrRecheck but got %v", err) + } +} + +func TestParseInlineStruct(t *testing.T) { + csv := csvSchema{ + keys: map[int]csvHeaderStructMapping{ + 0: {"filename", "name"}, + 1: {"path", "path"}, + }, + delim: ",", + } + + in := struct { + Yeeterson string `json:"path"` + Mcgee string `json:"name"` + }{ + Yeeterson: "test/path", + Mcgee: "testfile", + } + + expected := []byte("testfile,test/path\n") + result, err := csv.parse(in) + + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + + if !strings.EqualFold(string(result), string(expected)) { + t.Errorf("Expected %s but got %s", string(expected), string(result)) + } +} + +func TestParseNilPointer(t *testing.T) { + csv := csvSchema{ + keys: map[int]csvHeaderStructMapping{ + 0: {"filename", "name"}, + 1: {"path", "path"}, + }, + delim: ",", + } + + var in *File = nil + + _, err := csv.parse(in) + + if !errors.Is(err, ErrNilPointer) { + t.Errorf("Expected ErrNilPointer but got %v", err) + } +} + +func TestParseNonNilPointer(t *testing.T) { + csv := csvSchema{ + keys: map[int]csvHeaderStructMapping{ + 0: {"filename", "name"}, + 1: {"path", "path"}, + }, + delim: ",", + } + + in := &File{ + Path: "test/path", + Name: "testfile", + } + + expected := []byte("testfile,test/path\n") + result, err := csv.parse(in) + + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + + if !strings.EqualFold(string(result), string(expected)) { + t.Errorf("Expected %s but got %s", string(expected), string(result)) + } +} + +func TestJSONCSVParityAndCheckOwnPID(t *testing.T) { + csv := defCSVHeader + cfg := newConfigFromFlags() + cfg.sumMD5 = true + cfg.sumSHA1 = true + cfg.sumSHA256 = true + cfg.sumSHA512 = true + + myPID := os.Getpid() + procfsTarget := filepath.Join(constProcDir, strconv.Itoa(myPID), "/exe") + file, err := cfg.checkFilePath(procfsTarget) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + var jDat []byte + + if jDat, err = json.Marshal(file); err != nil { + t.Fatalf("unexpected json error: %v", err) + } + + t.Logf("my PID json data: \n %s", string(jDat)) + + expected := [][]byte{ + []byte("filename,path,entropy,elf_file,md5,sha1,sha256,sha512\n"), + []byte(file.Name + "," + file.Path + "," + strconv.FormatFloat(file.Entropy, 'f', 2, 64) + "," + + strconv.FormatBool(file.IsELF) + "," + file.Checksums.MD5 + "," + file.Checksums.SHA1 + "," + + file.Checksums.SHA256 + "," + file.Checksums.SHA512 + "\n"), + } + + result, err := csv.parse(file) + + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + if !strings.EqualFold(string(result), string(expected[1])) { + t.Errorf("Expected %s but got %s", string(expected[1]), string(result)) + } + + results := NewResults() + results.Add(file) + + result, err = results.MarshalCSV() + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + expectedJoined := bytes.Join(expected, []byte("")) + + if !strings.EqualFold(string(result), string(expectedJoined)) { + t.Errorf("Expected %s but got %s", string(expectedJoined), string(result)) + } +} + +func TestErroneous(t *testing.T) { + t.Run("IsElfType", func(t *testing.T) { + isElf, err := IsElfType("") + if err == nil { + t.Errorf("expected error on empty file passed, got nil") + } + if isElf { + t.Errorf("expected isElf == false on empty file passed, got true") + } + if isElf, err = IsElfType("/dev/nope"); err == nil { + t.Errorf("expected error on non-existent file passed, got nil") + } + if isElf { + t.Errorf("expected isElf == false on non-existent file passed, got true") + } + if isElf, err = IsElfType("/dev/null"); err == nil { + t.Errorf("expected error on non-regular file passed, got nil") + } + if isElf { + t.Errorf("expected isElf == false on non-regular file passed, got true") + } + smallFilePath := filepath.Join(t.TempDir(), "smol") + if err = os.WriteFile(smallFilePath, []byte{0x05}, 0644); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if isElf, err = IsElfType(smallFilePath); err == nil { + t.Errorf("expected error on small file passed, got nil") + } + if isElf { + t.Errorf("expected isElf == false on small file passed, got true") + } + + }) +}