Port mining optimizations for veblop (ethereum#1783)

* feat: disabling gogc override

* internal/cli: add GC variables in config

* core,eth,internal/cli,triedb/pathdb: add maxdifflayers to config

* consensus/bor: fix header timestamp when time is in the past

When the calculated header timestamp based on parent time and producer delay
falls in the past, ensure we add the full producer delay to the current time
rather than just setting it to current time. This prevents immediate timeout
issues in the commit interrupt mechanism where blocks would be interrupted
immediately due to negative time delays.

Previously, when header.Time < now, we would set header.Time = now, which
could still result in timing issues during block building. Now we properly
calculate header.Time = now + CalcProducerDelay() to ensure adequate time
for block building operations.

* consensus/bor,internal/cli,params: add header timestamp fix when in past

* perf(core/blockstm): optimize HasReadDep for small input sizes

Add performance optimization to HasReadDep function based on production metrics
showing ~3M calls with median size of 15. For small inputs (<=512 elements),
use direct nested loop comparison instead of map allocation to improve performance
due to better cache locality and avoiding allocation overhead.

Additionally:
- Replace map[Key]bool with map[Key]struct{} to reduce memory usage
- Add Prometheus metrics to track function calls and input sizes
- Add detailed comments explaining the optimization rationale

The cutoff of 512 was determined through benchmarking and captures >50% of
production calls, providing significant performance improvement for the
common case while maintaining O(n+m) complexity for larger inputs.

* core: add hasReadDepCallCounter and readsMapSizeHist metrics

* eth.internal/cli,triedb: fix lint

* core,eth,internal/cli,triedb/pathdb: rm maxdifflayers config

* core: add dagBuildTimer

* Revert "consensus/bor,internal/cli,params: add header timestamp fix when in past"

This reverts commit 76f6ac5afef4309ec18184cf07618fc25cefff55.

* triedb/pathdb: rm invalid comment

* internal/cli: rm unused function

---------

Co-authored-by: John Hilliard <praetoriansentry@gmail.com>

Author: Raneet10

consensus/bor/bor.go

@@ -988,7 +988,7 @@ func (c *Bor) Prepare(chain consensus.ChainHeaderReader, header *types.Header) e
 
 	header.Time = parent.Time + CalcProducerDelay(number, succession, c.config)
 	if header.Time < uint64(time.Now().Unix()) {
-		header.Time = uint64(time.Now().Unix())
+		header.Time = uint64(time.Now().Unix()) + CalcProducerDelay(number, succession, c.config)
 	} else {
 		// For primary validators, wait until the current block production window
 		// starts. This prevents bor from starting to build next block before time

core/blockstm/dag.go

@@ -8,6 +8,13 @@ import (
 	"github.com/heimdalr/dag"
 
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
+)
+
+var (
+	hasReadDepCallCounter = metrics.NewRegisteredCounter("blockstm/hasreaddep/calls", nil)
+	readsMapSizeHist      = metrics.NewRegisteredHistogram("blockstm/hasreaddep/reads/size", nil, metrics.NewExpDecaySample(1028, 0.015))
+	dagBuildTimer         = metrics.NewRegisteredTimer("blockstm/dag/build", nil)
 )
 
 type DAG struct {
@@ -20,13 +27,39 @@ type TxDep struct {
 	FullWriteList [][]WriteDescriptor
 }
 
+// HasReadDep checks if there are any read dependencies between two transactions.
+// Performance optimization: Based on production metrics showing ~3M calls with median
+// size of 15 and 95th percentile of 94, we avoid map allocation for small inputs.
+// This optimization leverages the fact that for small lists, linear search is faster
+// than map construction due to avoiding allocation overhead and better cache locality.
 func HasReadDep(txFrom TxnOutput, txTo TxnInput) bool {
-	reads := make(map[Key]bool)
+	hasReadDepCallCounter.Inc(1)
+
+	// Cutoff determined by benchmarking: below this size, nested loops outperform map lookup
+	// due to avoiding allocation overhead. With median=15, this captures >50% of calls.
+	const smallCutoff = 512
+	if len(txTo) <= smallCutoff {
+		// For small inputs, use direct comparison (O(n*m) but with better constants)
+		for _, rd := range txFrom {
+			for _, v := range txTo {
+				if rd.Path == v.Path {
+					return true
+				}
+			}
+		}
+		return false
+	}
 
+	// For larger inputs, use map for O(n+m) complexity
+	// Using struct{} instead of bool saves memory (0 bytes vs 1 byte per entry)
+	// since we only need set membership, not associated values
+	reads := make(map[Key]struct{})
 	for _, v := range txTo {
-		reads[v.Path] = true
+		reads[v.Path] = struct{}{}
 	}
 
+	readsMapSizeHist.Update(int64(len(reads)))
+
 	for _, rd := range txFrom {
 		if _, ok := reads[rd.Path]; ok {
 			return true
@@ -37,6 +70,8 @@ func HasReadDep(txFrom TxnOutput, txTo TxnInput) bool {
 }
 
 func BuildDAG(deps TxnInputOutput) (d DAG) {
+	defer dagBuildTimer.UpdateSince(time.Now())
+
 	d = DAG{dag.NewDAG()}
 	ids := make(map[int]string)
 

internal/cli/server/config.go

@@ -3,10 +3,12 @@ package server
 import (
 	"crypto/ecdsa"
 	"fmt"
-	"math"
+
+	// "math"
 	"math/big"
 	"os"
 	"path/filepath"
+	"regexp"
 	"runtime"
 	"strconv"
 	"strings"
@@ -614,6 +616,16 @@ type CacheConfig struct {
 
 	// Raise the open file descriptor resource limit (default = system fd limit)
 	FDLimit int `hcl:"fdlimit,optional" toml:"fdlimit,optional"`
+
+	// GC settings
+	// GoMemLimit sets the soft memory limit for the runtime
+	GoMemLimit string `hcl:"gomemlimit,optional" toml:"gomemlimit,optional"`
+
+	// GoGC sets the initial garbage collection target percentage
+	GoGC int `hcl:"gogc,optional" toml:"gogc,optional"`
+
+	// GoDebug sets debugging variables for the runtime
+	GoDebug string `hcl:"godebug,optional" toml:"godebug,optional"`
 }
 
 type ExtraDBConfig struct {
@@ -847,6 +859,9 @@ func DefaultConfig() *Config {
 			FilterLogCacheSize: ethconfig.Defaults.FilterLogCacheSize,
 			TrieTimeout:        60 * time.Minute,
 			FDLimit:            0,
+			GoMemLimit:         "",  // Empty means no limit
+			GoGC:               100, // Go default is 100%
+			GoDebug:            "",  // Empty means no debug flags
 		},
 		ExtraDB: &ExtraDBConfig{
 			// These are LevelDB defaults, specifying here for clarity in code and in logging.
@@ -1226,11 +1241,34 @@ func (c *Config) buildEth(stack *node.Node, accountManager *accounts.Manager) (*
 				cache = allowance
 			}
 		}
-		// Tune the garbage collector
-		gogc := math.Max(20, math.Min(100, 100/(float64(cache)/1024)))
+		// Apply configurable garbage collection settings with validation
+		if c.Cache.GoMemLimit != "" {
+			if err := validateGoMemLimit(c.Cache.GoMemLimit); err != nil {
+				log.Warn("Invalid GOMEMLIMIT value, skipping", "value", c.Cache.GoMemLimit, "error", err)
+			} else {
+				os.Setenv("GOMEMLIMIT", c.Cache.GoMemLimit)
+				log.Info("Set GOMEMLIMIT", "value", c.Cache.GoMemLimit)
+			}
+		}
+
+		// Sanitize GOGC value to reasonable bounds
+		sanitizedGoGC := sanitizeGoGC(c.Cache.GoGC)
+		if sanitizedGoGC != c.Cache.GoGC {
+			log.Warn("GOGC value sanitized", "original", c.Cache.GoGC, "sanitized", sanitizedGoGC)
+		}
+		if sanitizedGoGC != 100 { // Only set if different from default
+			godebug.SetGCPercent(sanitizedGoGC)
+			log.Info("Set GOGC", "percent", sanitizedGoGC)
+		}
 
-		log.Debug("Sanitizing Go's GC trigger", "percent", int(gogc))
-		godebug.SetGCPercent(int(gogc))
+		if c.Cache.GoDebug != "" {
+			if err := validateGoDebug(c.Cache.GoDebug); err != nil {
+				log.Warn("Invalid GODEBUG value, skipping", "value", c.Cache.GoDebug, "error", err)
+			} else {
+				os.Setenv("GODEBUG", c.Cache.GoDebug)
+				log.Info("Set GODEBUG", "value", c.Cache.GoDebug)
+			}
+		}
 
 		n.DatabaseCache = calcPerc(c.Cache.PercDatabase)
 		n.SnapshotCache = calcPerc(c.Cache.PercSnapshot)
@@ -1706,6 +1744,103 @@ func MakePasswordListFromFile(path string) ([]string, error) {
 	return lines, nil
 }
 
+// validateGoMemLimit validates GOMEMLIMIT values
+func validateGoMemLimit(value string) error {
+	// GOMEMLIMIT can be:
+	// - A number followed by optional unit (B, KB, MB, GB, TB, PB)
+	// - "off" to disable soft limit
+	if value == "off" {
+		return nil
+	}
+
+	// Parse the value to validate format
+	if matched, _ := regexp.MatchString(`^[0-9]+(\.[0-9]+)?[KMGTPE]?[iB]?$`, value); !matched {
+		return fmt.Errorf("invalid GOMEMLIMIT format, expected number with optional unit (e.g., '8GB', '1024MB')")
+	}
+
+	return nil
+}
+
+// sanitizeGoGC clamps GOGC values to reasonable bounds
+func sanitizeGoGC(value int) int {
+	const (
+		minGoGC = 10
+		maxGoGC = 2000
+	)
+
+	if value < minGoGC {
+		return minGoGC
+	}
+	if value > maxGoGC {
+		return maxGoGC
+	}
+	return value
+}
+
+// validateGoDebug validates GODEBUG values for known debug variables
+func validateGoDebug(value string) error {
+	// Known GODEBUG variables (not exhaustive, but covers common ones)
+	knownVars := map[string]bool{
+		"gctrace":            true, // GC trace
+		"gcpacertrace":       true, // GC pacer trace
+		"madvdontneed":       true, // Memory management
+		"scavenge":           true, // Scavenger debug
+		"asyncpreempt":       true, // Async preemption
+		"cgocheck":           true, // CGO check
+		"schedtrace":         true, // Scheduler trace
+		"scheddetail":        true, // Scheduler detail
+		"tracebackancestors": true, // Traceback ancestors
+		"httpmuxgo121":       true, // HTTP mux
+		"netdns":             true, // DNS resolution
+		"tls13":              true, // TLS 1.3
+		"panicnil":           true, // Panic on nil
+		"invalidptr":         true, // Invalid pointer checking
+		"sbrk":               true, // Memory allocation
+		"efence":             true, // Electric fence
+		"inittrace":          true, // Init trace
+		"cpu.all":            true, // CPU features
+	}
+
+	// Split by comma and validate each key=value pair
+	pairs := strings.Split(value, ",")
+	for _, pair := range pairs {
+		if strings.TrimSpace(pair) == "" {
+			continue
+		}
+
+		parts := strings.SplitN(pair, "=", 2)
+		if len(parts) != 2 {
+			return fmt.Errorf("invalid GODEBUG format: '%s', expected key=value", pair)
+		}
+
+		key := strings.TrimSpace(parts[0])
+		val := strings.TrimSpace(parts[1])
+
+		if key == "" {
+			return fmt.Errorf("empty GODEBUG variable name in: '%s'", pair)
+		}
+
+		// Validate value (most GODEBUG vars use 0/1 or numeric values)
+		if val == "" {
+			return fmt.Errorf("empty GODEBUG value for '%s'", key)
+		}
+
+		// Warn about unknown variables but don't fail (Go may add new ones)
+		if !knownVars[key] {
+			log.Warn("Unknown GODEBUG variable", "var", key, "value", val)
+		}
+
+		// Validate numeric values where appropriate
+		if key == "gctrace" || key == "gcpacertrace" || key == "schedtrace" || key == "cgocheck" {
+			if _, err := strconv.Atoi(val); err != nil {
+				return fmt.Errorf("GODEBUG variable '%s' expects numeric value, got '%s'", key, val)
+			}
+		}
+	}
+
+	return nil
+}
+
 // loadFilteredAddresses loads newline-separated addresses to filter from the specified file.
 func loadFilteredAddresses(filePath string) (map[common.Address]struct{}, error) {
 	if filePath == "" {

internal/cli/server/flags.go

@@ -489,6 +489,27 @@ func (c *Command) Flags(config *Config) *flagset.Flagset {
 		Default: c.cliConfig.Cache.FDLimit,
 		Group:   "Cache",
 	})
+	f.StringFlag(&flagset.StringFlag{
+		Name:    "cache.gomemlimit",
+		Usage:   "Set GOMEMLIMIT for the runtime (e.g. '34GB', '34359738368'). Empty means no limit",
+		Value:   &c.cliConfig.Cache.GoMemLimit,
+		Default: c.cliConfig.Cache.GoMemLimit,
+		Group:   "Cache",
+	})
+	f.IntFlag(&flagset.IntFlag{
+		Name:    "cache.gogc",
+		Usage:   "Set GOGC percentage for garbage collection trigger (default: 100)",
+		Value:   &c.cliConfig.Cache.GoGC,
+		Default: c.cliConfig.Cache.GoGC,
+		Group:   "Cache",
+	})
+	f.StringFlag(&flagset.StringFlag{
+		Name:    "cache.godebug",
+		Usage:   "Set GODEBUG variables for runtime debugging (e.g. 'gctrace=1,gcpacertrace=1')",
+		Value:   &c.cliConfig.Cache.GoDebug,
+		Default: c.cliConfig.Cache.GoDebug,
+		Group:   "Cache",
+	})
 
 	// LevelDB options
 	f.Uint64Flag(&flagset.Uint64Flag{

triedb/pathdb/database.go

@@ -134,6 +134,7 @@ func (c *Config) sanitize() *Config {
 		log.Warn("Sanitizing invalid node buffer size", "provided", common.StorageSize(conf.WriteBufferSize), "updated", common.StorageSize(maxBufferSize))
 		conf.WriteBufferSize = maxBufferSize
 	}
+
 	return &conf
 }
 

triedb/pathdb/database_test.go

@@ -121,6 +121,7 @@ type tester struct {
 	snapStorages map[common.Hash]map[common.Hash]map[common.Hash][]byte // Keyed by the hash of account address and the hash of storage key
 }
 
+//nolint:unused
 func newTester(t *testing.T, historyLimit uint64, isVerkle bool, layers int, enableIndex bool) *tester {
 	var (
 		disk, _ = rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{Ancient: t.TempDir()})
@@ -464,8 +465,6 @@ func TestDatabaseRollback(t *testing.T) {
 	defer func() {
 		maxDiffLayers = 128
 	}()
-
-	// Verify state histories
 	tester := newTester(t, 0, false, 32, false)
 	defer tester.release()
 
@@ -498,7 +497,6 @@ func TestDatabaseRecoverable(t *testing.T) {
 	defer func() {
 		maxDiffLayers = 128
 	}()
-
 	var (
 		tester = newTester(t, 0, false, 12, false)
 		index  = tester.bottomIndex()