From f651600bb378d9c457271006b149c840b92080b0 Mon Sep 17 00:00:00 2001 From: lightclient Date: Tue, 23 Sep 2025 19:53:04 -0600 Subject: [PATCH 1/6] trie: add inspector Co-authored-by: Fynn Co-authored-by: lightclient Co-authored-by: MariusVanDerWijden --- cmd/geth/dbcmd.go | 96 ++++++++++++++++ trie/inspect.go | 279 ++++++++++++++++++++++++++++++++++++++++++++++ trie/trie.go | 13 +++ 3 files changed, 388 insertions(+) create mode 100644 trie/inspect.go diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index c57add06563..f041658b723 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -19,9 +19,11 @@ package main import ( "bytes" "fmt" + "math" "os" "os/signal" "path/filepath" + "runtime" "slices" "strconv" "strings" @@ -75,6 +77,7 @@ Remove blockchain and state databases`, dbCompactCmd, dbGetCmd, dbDeleteCmd, + dbInspectTrieCmd, dbPutCmd, dbGetSlotsCmd, dbDumpFreezerIndex, @@ -93,6 +96,18 @@ Remove blockchain and state databases`, Usage: "Inspect the storage size for each type of data in the database", Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`, } + dbInspectTrieCmd = &cli.Command{ + Action: inspectTrie, + Name: "inspect-trie", + ArgsUsage: " ", + Flags: []cli.Flag{ + utils.DataDirFlag, + }, + Usage: "Print detailed trie information about the structure of account trie and storage tries.", + Description: `This commands iterates the entrie trie-backed state. If the 'blocknum' is not specified, +the latest block number will be used by default. 'jobnum' indicates the number of coroutines concurrently traversing +the account and storage trie.`, + } dbCheckStateContentCmd = &cli.Command{ Action: checkStateContent, Name: "check-state-content", @@ -386,6 +401,87 @@ func checkStateContent(ctx *cli.Context) error { return nil } +func inspectTrie(ctx *cli.Context) error { + if ctx.NArg() > 2 { + return fmt.Errorf("excessive number of arguments: %v", ctx.Command.ArgsUsage) + } + + var ( + blockNumber uint64 + trieRootHash common.Hash + jobnum uint64 + ) + + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + db := utils.MakeChainDatabase(ctx, stack, false) + defer db.Close() + + var headerBlockHash common.Hash + switch { + case ctx.NArg() == 0 || ctx.Args().Get(0) == "latest": + headerHash := rawdb.ReadHeadHeaderHash(db) + n, ok := rawdb.ReadHeaderNumber(db, headerHash) + if !ok { + return fmt.Errorf("could not load head block hash") + } + blockNumber = n + case ctx.Args().Get(0) == "snapshot": + trieRootHash = rawdb.ReadSnapshotRoot(db) + blockNumber = math.MaxUint64 + default: + var err error + blockNumber, err = strconv.ParseUint(ctx.Args().Get(0), 10, 64) + if err != nil { + return fmt.Errorf("failed to parse blocknum, Args[0]: %v, err: %v", ctx.Args().Get(0), err) + } + } + + // Configure number of threads. + if ctx.NArg() <= 1 { + jobnum = uint64(runtime.NumCPU()) + } else { + var err error + jobnum, err = strconv.ParseUint(ctx.Args().Get(1), 10, 64) + if err != nil { + return fmt.Errorf("failed to parse jobnum, Args[1]: %v, err: %v", ctx.Args().Get(1), err) + } + } + + // Load head block number based on canonical hash, if applicable. + if blockNumber != math.MaxUint64 { + headerBlockHash = rawdb.ReadCanonicalHash(db, blockNumber) + if headerBlockHash == (common.Hash{}) { + return fmt.Errorf("canonical hash for block %d not found", blockNumber) + } + blockHeader := rawdb.ReadHeader(db, headerBlockHash, blockNumber) + trieRootHash = blockHeader.Root + } + + if (trieRootHash == common.Hash{}) { + log.Error("Empty root hash") + } + + log.Debug("Inspecting trie", "root", trieRootHash, "block", blockNumber) + + triedb := utils.MakeTrieDatabase(ctx, stack, db, false, true, false) + defer triedb.Close() + + theTrie, err := trie.New(trie.TrieID(trieRootHash), triedb) + if err != nil { + fmt.Printf("fail to new trie tree, err: %v, rootHash: %v\n", err, trieRootHash.String()) + return err + } + inspector, err := trie.NewInspector(theTrie, triedb, trieRootHash, blockNumber, jobnum) + if err != nil { + return err + } + inspector.Run() + inspector.DisplayResult() + return nil +} + func showDBStats(db ethdb.KeyValueStater) { stats, err := db.Stat() if err != nil { diff --git a/trie/inspect.go b/trie/inspect.go new file mode 100644 index 00000000000..415be702b44 --- /dev/null +++ b/trie/inspect.go @@ -0,0 +1,279 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "bytes" + "errors" + "fmt" + "runtime" + "slices" + "sort" + "strings" + "sync" + "sync/atomic" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/triedb/database" + "github.com/olekukonko/tablewriter" + "golang.org/x/sync/semaphore" +) + +type Inspector struct { + trie *Trie // traverse trie + db database.NodeDatabase + stateRootHash common.Hash + blocknum uint64 + root node // root of triedb + totalNum atomic.Uint64 + wg sync.WaitGroup + statLock sync.RWMutex + result map[string]*trieTreeStat + sem *semaphore.Weighted + eoaAccountNums atomic.Uint64 +} + +type trieTreeStat struct { + isAccountTrie bool + theNodeStatByLevel [15]nodeStat + totalNodeStat nodeStat +} + +type nodeStat struct { + ShortNodeCnt atomic.Uint64 + FullNodeCnt atomic.Uint64 + ValueNodeCnt atomic.Uint64 +} + +func (ns *nodeStat) IsEmpty() bool { + if ns.FullNodeCnt.Load() == 0 && ns.ShortNodeCnt.Load() == 0 && ns.ValueNodeCnt.Load() == 0 { + return true + } + return false +} + +func (trieStat *trieTreeStat) AtomicAdd(theNode node, height uint32) { + switch (theNode).(type) { + case *shortNode: + trieStat.totalNodeStat.ShortNodeCnt.Add(1) + trieStat.theNodeStatByLevel[height].ShortNodeCnt.Add(1) + case *fullNode: + trieStat.totalNodeStat.FullNodeCnt.Add(1) + trieStat.theNodeStatByLevel[height].FullNodeCnt.Add(1) + case valueNode: + trieStat.totalNodeStat.ValueNodeCnt.Add(1) + trieStat.theNodeStatByLevel[height].ValueNodeCnt.Add(1) + default: + panic(errors.New("invalid node type for statistics")) + } +} + +func (trieStat *trieTreeStat) Display(ownerAddress string, treeType string) string { + sw := new(strings.Builder) + table := tablewriter.NewWriter(sw) + table.SetHeader([]string{"-", "Level", "ShortNodeCnt", "FullNodeCnt", "ValueNodeCnt"}) + if ownerAddress == "" { + table.SetCaption(true, fmt.Sprintf("%v", treeType)) + } else { + table.SetCaption(true, fmt.Sprintf("%v-%v", treeType, ownerAddress)) + } + table.SetAlignment(1) + for i := 0; i < len(trieStat.theNodeStatByLevel); i++ { + ns := &trieStat.theNodeStatByLevel[i] + if ns.IsEmpty() { + break + } + table.AppendBulk([][]string{ + {"-", fmt.Sprintf("%d", i), fmt.Sprintf("%d", ns.ShortNodeCnt.Load()), fmt.Sprintf("%d", ns.FullNodeCnt.Load()), fmt.Sprintf("%d", ns.ValueNodeCnt.Load())}, + }) + } + table.AppendBulk([][]string{ + {"Total", "-", fmt.Sprintf("%d", trieStat.totalNodeStat.ShortNodeCnt.Load()), fmt.Sprintf("%d", trieStat.totalNodeStat.FullNodeCnt.Load()), fmt.Sprintf("%d", trieStat.totalNodeStat.ValueNodeCnt.Load())}, + }) + table.Render() + return sw.String() +} + +// NewInspector return an inspector obj +func NewInspector(tr *Trie, db database.NodeDatabase, stateRootHash common.Hash, blocknum uint64, jobnum uint64) (*Inspector, error) { + if tr == nil { + return nil, errors.New("trie is nil") + } + + if tr.root == nil { + return nil, errors.New("trie root is nil") + } + + ins := &Inspector{ + trie: tr, + db: db, + stateRootHash: stateRootHash, + blocknum: blocknum, + root: tr.root, + result: make(map[string]*trieTreeStat), + sem: semaphore.NewWeighted(int64(jobnum)), + } + + return ins, nil +} + +// Run statistics, external call +func (inspect *Inspector) Run() { + accountTrieStat := &trieTreeStat{ + isAccountTrie: true, + } + + if _, ok := inspect.result[""]; !ok { + inspect.result[""] = accountTrieStat + } + log.Info("Find Account Trie Tree", "rootHash: ", inspect.trie.Hash().String(), "BlockNum: ", inspect.blocknum) + + inspect.concurrentTraversal(inspect.trie, accountTrieStat, inspect.root, 0, []byte{}) + inspect.wg.Wait() +} + +func (inspect *Inspector) concurrentTraversal(theTrie *Trie, theTrieTreeStat *trieTreeStat, theNode node, height uint32, path []byte) { + // print process progress + totalNum := inspect.totalNum.Add(1) + if totalNum%100000 == 0 { + fmt.Printf("Complete progress: %v, go routines Num: %v\n", totalNum, runtime.NumGoroutine()) + } + + // nil node + if theNode == nil { + return + } + + switch current := (theNode).(type) { + case *shortNode: + inspect.concurrentTraversal(theTrie, theTrieTreeStat, current.Val, height, append(path, current.Key...)) + case *fullNode: + for idx, child := range current.Children { + if child == nil { + continue + } + childPath := append(path, byte(idx)) + if inspect.sem.TryAcquire(1) { + inspect.wg.Add(1) + go func() { + inspect.concurrentTraversal(theTrie, theTrieTreeStat, child, height+1, slices.Clone(childPath)) + inspect.wg.Done() + }() + } else { + inspect.concurrentTraversal(theTrie, theTrieTreeStat, child, height+1, childPath) + } + } + case hashNode: + n, err := theTrie.resolveWithoutTrack(current, path) + if err != nil { + fmt.Printf("Resolve HashNode error: %v, TrieRoot: %v, Height: %v, Path: %v\n", err, theTrie.Hash().String(), height+1, path) + return + } + inspect.concurrentTraversal(theTrie, theTrieTreeStat, n, height, path) + return + case valueNode: + if !hasTerm(path) { + break + } + var account types.StateAccount + if err := rlp.Decode(bytes.NewReader(current), &account); err != nil { + break + } + // TODO: update for 7702 + if common.BytesToHash(account.CodeHash) == types.EmptyCodeHash { + inspect.eoaAccountNums.Add(1) + } + if account.Root == (common.Hash{}) || account.Root == types.EmptyRootHash { + break + } + ownerAddress := common.BytesToHash(hexToCompact(path)) + contractTrie, err := New(StorageTrieID(inspect.stateRootHash, ownerAddress, account.Root), inspect.db) + if err != nil { + fmt.Printf("New contract trie node: %v, error: %v, Height: %v, Path: %v\n", theNode, err, height, path) + break + } + contractTrie.opTracer.reset() + trieStat := &trieTreeStat{ + isAccountTrie: false, + } + + inspect.statLock.Lock() + if _, ok := inspect.result[ownerAddress.String()]; !ok { + inspect.result[ownerAddress.String()] = trieStat + } + inspect.statLock.Unlock() + + inspect.wg.Add(1) + go func() { + inspect.concurrentTraversal(contractTrie, trieStat, contractTrie.root, 0, []byte{}) + inspect.wg.Done() + }() + default: + panic(errors.New("invalid node type for traverse")) + } + theTrieTreeStat.AtomicAdd(theNode, height) +} + +func (inspect *Inspector) DisplayResult() { + // display root hash + if _, ok := inspect.result[""]; !ok { + log.Info("Display result error", "missing account trie") + return + } + fmt.Print(inspect.result[""].Display("", "AccountTrie")) + + type sortedTrie struct { + totalNum uint64 + ownerAddress string + } + // display contract trie + var sortedTriesByNums []sortedTrie + var totalContactsNodeStat nodeStat + var contractTrieCnt uint64 = 0 + + for ownerAddress, stat := range inspect.result { + if ownerAddress == "" { + continue + } + contractTrieCnt++ + totalContactsNodeStat.ShortNodeCnt.Add(stat.totalNodeStat.ShortNodeCnt.Load()) + totalContactsNodeStat.FullNodeCnt.Add(stat.totalNodeStat.FullNodeCnt.Load()) + totalContactsNodeStat.ValueNodeCnt.Add(stat.totalNodeStat.ValueNodeCnt.Load()) + totalNodeCnt := stat.totalNodeStat.ShortNodeCnt.Load() + stat.totalNodeStat.ValueNodeCnt.Load() + stat.totalNodeStat.FullNodeCnt.Load() + sortedTriesByNums = append(sortedTriesByNums, sortedTrie{totalNum: totalNodeCnt, ownerAddress: ownerAddress}) + } + sort.Slice(sortedTriesByNums, func(i, j int) bool { + return sortedTriesByNums[i].totalNum > sortedTriesByNums[j].totalNum + }) + fmt.Println("EOA accounts num: ", inspect.eoaAccountNums.Load()) + // only display top 5 + for i, t := range sortedTriesByNums { + if i > 5 { + break + } + if stat, ok := inspect.result[t.ownerAddress]; !ok { + log.Error("Storage trie stat not found", "ownerAddress", t.ownerAddress) + } else { + fmt.Print(stat.Display(t.ownerAddress, "ContractTrie")) + } + } + fmt.Printf("Contract Trie, total trie num: %v, ShortNodeCnt: %v, FullNodeCnt: %v, ValueNodeCnt: %v\n", + contractTrieCnt, totalContactsNodeStat.ShortNodeCnt.Load(), totalContactsNodeStat.FullNodeCnt.Load(), totalContactsNodeStat.ValueNodeCnt.Load()) +} diff --git a/trie/trie.go b/trie/trie.go index 36cc732ee85..299cfa72189 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -694,6 +694,19 @@ func (t *Trie) resolveAndTrack(n hashNode, prefix []byte) (node, error) { return decodeNodeUnsafe(n, blob) } +// resolveWithoutTrack loads node from the underlying store with the given node hash +// and path prefix. +func (t *Trie) resolveWithoutTrack(n node, prefix []byte) (node, error) { + if n, ok := n.(hashNode); ok { + blob, err := t.reader.Node(prefix, common.BytesToHash(n)) + if err != nil { + return nil, err + } + return mustDecodeNode(n, blob), nil + } + return n, nil +} + // deletedNodes returns a list of node paths, referring the nodes being deleted // from the trie. It's possible a few deleted nodes were embedded in their parent // before, the deletions can be no effect by deleting nothing, filter them out. From 2f238abaa1965af6ae25fa1f7f809ed81a63c92e Mon Sep 17 00:00:00 2001 From: lightclient Date: Tue, 23 Sep 2025 20:14:29 -0600 Subject: [PATCH 2/6] core/rawdb: move tablewriter re-export to internal package --- core/rawdb/database.go | 3 ++- .../tablewriter/table.go | 5 +++-- .../tablewriter/table_tinygo.go | 2 +- .../tablewriter/table_tinygo_test.go | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) rename core/rawdb/database_tablewriter_unix.go => internal/tablewriter/table.go (92%) rename core/rawdb/database_tablewriter_tinygo.go => internal/tablewriter/table_tinygo.go (99%) rename core/rawdb/database_tablewriter_tinygo_test.go => internal/tablewriter/table_tinygo_test.go (99%) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 626d390c0d3..8a0da313498 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -35,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb/memorydb" + "github.com/ethereum/go-ethereum/internal/tablewriter" "github.com/ethereum/go-ethereum/log" "golang.org/x/sync/errgroup" ) @@ -643,7 +644,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { total.Add(uint64(ancient.size())) } - table := newTableWriter(os.Stdout) + table := tablewriter.NewWriter(os.Stdout) table.SetHeader([]string{"Database", "Category", "Size", "Items"}) table.SetFooter([]string{"", "Total", common.StorageSize(total.Load()).String(), fmt.Sprintf("%d", count.Load())}) table.AppendBulk(stats) diff --git a/core/rawdb/database_tablewriter_unix.go b/internal/tablewriter/table.go similarity index 92% rename from core/rawdb/database_tablewriter_unix.go rename to internal/tablewriter/table.go index 8bec5396e87..ed44add2c49 100644 --- a/core/rawdb/database_tablewriter_unix.go +++ b/internal/tablewriter/table.go @@ -17,7 +17,7 @@ //go:build !tinygo // +build !tinygo -package rawdb +package tablewriter import ( "io" @@ -28,6 +28,7 @@ import ( // Re-export the real tablewriter types and functions type Table = tablewriter.Table -func newTableWriter(w io.Writer) *Table { +// Re-export NewWriter. +func NewWriter(w io.Writer) *Table { return tablewriter.NewWriter(w) } diff --git a/core/rawdb/database_tablewriter_tinygo.go b/internal/tablewriter/table_tinygo.go similarity index 99% rename from core/rawdb/database_tablewriter_tinygo.go rename to internal/tablewriter/table_tinygo.go index 2f8e456fd51..620ba772152 100644 --- a/core/rawdb/database_tablewriter_tinygo.go +++ b/internal/tablewriter/table_tinygo.go @@ -19,7 +19,7 @@ //go:build tinygo // +build tinygo -package rawdb +package tablewriter import ( "errors" diff --git a/core/rawdb/database_tablewriter_tinygo_test.go b/internal/tablewriter/table_tinygo_test.go similarity index 99% rename from core/rawdb/database_tablewriter_tinygo_test.go rename to internal/tablewriter/table_tinygo_test.go index 3bcf93832b4..ee61875ee9f 100644 --- a/core/rawdb/database_tablewriter_tinygo_test.go +++ b/internal/tablewriter/table_tinygo_test.go @@ -17,7 +17,7 @@ //go:build tinygo // +build tinygo -package rawdb +package tablewriter import ( "bytes" From e2308aa821c56fc1d86239d7b0a135d9c8b5573e Mon Sep 17 00:00:00 2001 From: lightclient Date: Mon, 20 Oct 2025 19:01:04 -0600 Subject: [PATCH 3/6] trie: refactor inspect --- trie/inspect.go | 327 ++++++++++++++++++++----------------------- trie/inspect_test.go | 91 ++++++++++++ 2 files changed, 246 insertions(+), 172 deletions(-) create mode 100644 trie/inspect_test.go diff --git a/trie/inspect.go b/trie/inspect.go index 415be702b44..a44878e6440 100644 --- a/trie/inspect.go +++ b/trie/inspect.go @@ -18,9 +18,7 @@ package trie import ( "bytes" - "errors" "fmt" - "runtime" "slices" "sort" "strings" @@ -29,251 +27,236 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/internal/tablewriter" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/triedb/database" - "github.com/olekukonko/tablewriter" "golang.org/x/sync/semaphore" ) -type Inspector struct { - trie *Trie // traverse trie - db database.NodeDatabase - stateRootHash common.Hash - blocknum uint64 - root node // root of triedb - totalNum atomic.Uint64 - wg sync.WaitGroup - statLock sync.RWMutex - result map[string]*trieTreeStat - sem *semaphore.Weighted - eoaAccountNums atomic.Uint64 +type inspector struct { + triedb database.NodeDatabase + trie *Trie + root common.Hash + + storage bool + stats map[common.Hash]*triestat + m sync.Mutex + + sem *semaphore.Weighted + wg sync.WaitGroup } -type trieTreeStat struct { - isAccountTrie bool - theNodeStatByLevel [15]nodeStat - totalNodeStat nodeStat +type triestat struct { + depth [15]stat } -type nodeStat struct { - ShortNodeCnt atomic.Uint64 - FullNodeCnt atomic.Uint64 - ValueNodeCnt atomic.Uint64 +func (s *triestat) maxDepth() int { + depth := 0 + for i := range s.depth { + if s.depth[i].short.Load() != 0 || s.depth[i].full.Load() != 0 || s.depth[i].value.Load() != 0 { + depth = i + } + } + return depth } -func (ns *nodeStat) IsEmpty() bool { - if ns.FullNodeCnt.Load() == 0 && ns.ShortNodeCnt.Load() == 0 && ns.ValueNodeCnt.Load() == 0 { - return true +type trieStatByDepth map[common.Hash]*triestat + +func (s trieStatByDepth) sort() ([]common.Hash, []*triestat) { + var ( + keys = make([]common.Hash, 0, len(s)) + stats = make([]*triestat, 0, len(s)) + ) + for k := range s { + keys = append(keys, k) } - return false + sort.Slice(keys, func(i, j int) bool { return s[keys[i]].maxDepth() > s[keys[j]].maxDepth() }) + for _, k := range keys { + stats = append(stats, s[k]) + } + return keys, stats } -func (trieStat *trieTreeStat) AtomicAdd(theNode node, height uint32) { - switch (theNode).(type) { +func (s *triestat) add(n node, d uint32) { + switch (n).(type) { case *shortNode: - trieStat.totalNodeStat.ShortNodeCnt.Add(1) - trieStat.theNodeStatByLevel[height].ShortNodeCnt.Add(1) + s.depth[d].short.Add(1) case *fullNode: - trieStat.totalNodeStat.FullNodeCnt.Add(1) - trieStat.theNodeStatByLevel[height].FullNodeCnt.Add(1) + s.depth[d].full.Add(1) case valueNode: - trieStat.totalNodeStat.ValueNodeCnt.Add(1) - trieStat.theNodeStatByLevel[height].ValueNodeCnt.Add(1) + s.depth[d].value.Add(1) default: - panic(errors.New("invalid node type for statistics")) + panic(fmt.Sprintf("%T: invalid node: %v", n, n)) } } -func (trieStat *trieTreeStat) Display(ownerAddress string, treeType string) string { - sw := new(strings.Builder) - table := tablewriter.NewWriter(sw) - table.SetHeader([]string{"-", "Level", "ShortNodeCnt", "FullNodeCnt", "ValueNodeCnt"}) - if ownerAddress == "" { - table.SetCaption(true, fmt.Sprintf("%v", treeType)) - } else { - table.SetCaption(true, fmt.Sprintf("%v-%v", treeType, ownerAddress)) - } - table.SetAlignment(1) - for i := 0; i < len(trieStat.theNodeStatByLevel); i++ { - ns := &trieStat.theNodeStatByLevel[i] - if ns.IsEmpty() { - break - } - table.AppendBulk([][]string{ - {"-", fmt.Sprintf("%d", i), fmt.Sprintf("%d", ns.ShortNodeCnt.Load()), fmt.Sprintf("%d", ns.FullNodeCnt.Load()), fmt.Sprintf("%d", ns.ValueNodeCnt.Load())}, - }) - } - table.AppendBulk([][]string{ - {"Total", "-", fmt.Sprintf("%d", trieStat.totalNodeStat.ShortNodeCnt.Load()), fmt.Sprintf("%d", trieStat.totalNodeStat.FullNodeCnt.Load()), fmt.Sprintf("%d", trieStat.totalNodeStat.ValueNodeCnt.Load())}, - }) - table.Render() - return sw.String() +type stat struct { + short atomic.Uint64 + full atomic.Uint64 + value atomic.Uint64 } -// NewInspector return an inspector obj -func NewInspector(tr *Trie, db database.NodeDatabase, stateRootHash common.Hash, blocknum uint64, jobnum uint64) (*Inspector, error) { - if tr == nil { - return nil, errors.New("trie is nil") +func (s *stat) empty() bool { + if s.full.Load() == 0 && s.short.Load() == 0 && s.value.Load() == 0 { + return true } + return false +} - if tr.root == nil { - return nil, errors.New("trie root is nil") - } +func (s *stat) load() (uint64, uint64, uint64) { + return s.short.Load(), s.full.Load(), s.value.Load() +} - ins := &Inspector{ - trie: tr, - db: db, - stateRootHash: stateRootHash, - blocknum: blocknum, - root: tr.root, - result: make(map[string]*trieTreeStat), - sem: semaphore.NewWeighted(int64(jobnum)), - } +func (s *stat) add(other *stat) *stat { + s.short.Add(other.short.Load()) + s.full.Add(other.full.Load()) + s.value.Add(other.value.Load()) + return s +} - return ins, nil +func InspectTrie(triedb database.NodeDatabase, root common.Hash, storage bool) error { + return inspectTrie(triedb, root, storage) } -// Run statistics, external call -func (inspect *Inspector) Run() { - accountTrieStat := &trieTreeStat{ - isAccountTrie: true, +func inspectTrie(triedb database.NodeDatabase, root common.Hash, storage bool) error { + trie, err := New(TrieID(root), triedb) + if err != nil { + return fmt.Errorf("fail to open trie %s: %w", root, err) } - - if _, ok := inspect.result[""]; !ok { - inspect.result[""] = accountTrieStat + in := inspector{ + triedb: triedb, + trie: trie, + root: root, + storage: storage, + stats: make(map[common.Hash]*triestat), + sem: semaphore.NewWeighted(int64(128)), } - log.Info("Find Account Trie Tree", "rootHash: ", inspect.trie.Hash().String(), "BlockNum: ", inspect.blocknum) + in.stats[root] = &triestat{} - inspect.concurrentTraversal(inspect.trie, accountTrieStat, inspect.root, 0, []byte{}) - inspect.wg.Wait() + in.inspect(trie.root, 0, []byte{}, in.stats[root]) + in.wg.Wait() + in.DisplayResult() + return nil } -func (inspect *Inspector) concurrentTraversal(theTrie *Trie, theTrieTreeStat *trieTreeStat, theNode node, height uint32, path []byte) { - // print process progress - totalNum := inspect.totalNum.Add(1) - if totalNum%100000 == 0 { - fmt.Printf("Complete progress: %v, go routines Num: %v\n", totalNum, runtime.NumGoroutine()) - } - - // nil node - if theNode == nil { +func (in *inspector) inspect(n node, height uint32, path []byte, stat *triestat) { + if n == nil { return } - switch current := (theNode).(type) { + switch n := (n).(type) { case *shortNode: - inspect.concurrentTraversal(theTrie, theTrieTreeStat, current.Val, height, append(path, current.Key...)) + in.inspect(n.Val, height, append(path, n.Key...), stat) case *fullNode: - for idx, child := range current.Children { + for idx, child := range n.Children { if child == nil { continue } childPath := append(path, byte(idx)) - if inspect.sem.TryAcquire(1) { - inspect.wg.Add(1) + if in.sem.TryAcquire(1) { + in.wg.Add(1) go func() { - inspect.concurrentTraversal(theTrie, theTrieTreeStat, child, height+1, slices.Clone(childPath)) - inspect.wg.Done() + in.inspect(child, height+1, slices.Clone(childPath), stat) + in.wg.Done() }() } else { - inspect.concurrentTraversal(theTrie, theTrieTreeStat, child, height+1, childPath) + in.inspect(child, height+1, childPath, stat) } } case hashNode: - n, err := theTrie.resolveWithoutTrack(current, path) + resolved, err := in.trie.resolveWithoutTrack(n, path) if err != nil { - fmt.Printf("Resolve HashNode error: %v, TrieRoot: %v, Height: %v, Path: %v\n", err, theTrie.Hash().String(), height+1, path) + fmt.Printf("Resolve HashNode error: %v, TrieRoot: %v, Height: %v, Path: %v\n", err, in.trie.Hash().String(), height+1, path) return } - inspect.concurrentTraversal(theTrie, theTrieTreeStat, n, height, path) + in.inspect(resolved, height, path, stat) return case valueNode: if !hasTerm(path) { break } var account types.StateAccount - if err := rlp.Decode(bytes.NewReader(current), &account); err != nil { + if err := rlp.Decode(bytes.NewReader(n), &account); err != nil { + // Not an account value. break } // TODO: update for 7702 - if common.BytesToHash(account.CodeHash) == types.EmptyCodeHash { - inspect.eoaAccountNums.Add(1) - } + // if common.BytesToHash(account.CodeHash) == types.EmptyCodeHash { + // inspect.eoaAccountNums.Add(1) + // } if account.Root == (common.Hash{}) || account.Root == types.EmptyRootHash { break } - ownerAddress := common.BytesToHash(hexToCompact(path)) - contractTrie, err := New(StorageTrieID(inspect.stateRootHash, ownerAddress, account.Root), inspect.db) - if err != nil { - fmt.Printf("New contract trie node: %v, error: %v, Height: %v, Path: %v\n", theNode, err, height, path) - break - } - contractTrie.opTracer.reset() - trieStat := &trieTreeStat{ - isAccountTrie: false, - } - inspect.statLock.Lock() - if _, ok := inspect.result[ownerAddress.String()]; !ok { - inspect.result[ownerAddress.String()] = trieStat - } - inspect.statLock.Unlock() + // Start inspecting storage trie. + if in.storage { + owner := common.BytesToHash(hexToCompact(path)) + storage, err := New(StorageTrieID(in.root, owner, account.Root), in.triedb) + if err != nil { + fmt.Printf("New contract trie node: %v, error: %v, Height: %v, Path: %v\n", n, err, height, path) + break + } + // contractTrie.opTracer.reset() + stat := &triestat{} - inspect.wg.Add(1) - go func() { - inspect.concurrentTraversal(contractTrie, trieStat, contractTrie.root, 0, []byte{}) - inspect.wg.Done() - }() + in.m.Lock() + in.stats[owner] = stat + in.m.Unlock() + + in.wg.Add(1) + go func() { + in.inspect(storage.root, 0, []byte{}, stat) + in.wg.Done() + }() + } default: - panic(errors.New("invalid node type for traverse")) + panic(fmt.Sprintf("%T: invalid node: %v", n, n)) } - theTrieTreeStat.AtomicAdd(theNode, height) + + // Record stats for current height + stat.add(n, height) } -func (inspect *Inspector) DisplayResult() { - // display root hash - if _, ok := inspect.result[""]; !ok { - log.Info("Display result error", "missing account trie") - return +func (s *triestat) display(title string) { + // Shorten title if too long. + if len(title) > 32 { + title = title[0:8] + "..." + title[len(title)-8:len(title)] } - fmt.Print(inspect.result[""].Display("", "AccountTrie")) - type sortedTrie struct { - totalNum uint64 - ownerAddress string - } - // display contract trie - var sortedTriesByNums []sortedTrie - var totalContactsNodeStat nodeStat - var contractTrieCnt uint64 = 0 + b := new(strings.Builder) + table := tablewriter.NewWriter(b) + table.SetHeader([]string{title, "Level", "Short Nodes", "Full Node", "Value Node"}) + table.SetAlignment(1) - for ownerAddress, stat := range inspect.result { - if ownerAddress == "" { - continue - } - contractTrieCnt++ - totalContactsNodeStat.ShortNodeCnt.Add(stat.totalNodeStat.ShortNodeCnt.Load()) - totalContactsNodeStat.FullNodeCnt.Add(stat.totalNodeStat.FullNodeCnt.Load()) - totalContactsNodeStat.ValueNodeCnt.Add(stat.totalNodeStat.ValueNodeCnt.Load()) - totalNodeCnt := stat.totalNodeStat.ShortNodeCnt.Load() + stat.totalNodeStat.ValueNodeCnt.Load() + stat.totalNodeStat.FullNodeCnt.Load() - sortedTriesByNums = append(sortedTriesByNums, sortedTrie{totalNum: totalNodeCnt, ownerAddress: ownerAddress}) - } - sort.Slice(sortedTriesByNums, func(i, j int) bool { - return sortedTriesByNums[i].totalNum > sortedTriesByNums[j].totalNum - }) - fmt.Println("EOA accounts num: ", inspect.eoaAccountNums.Load()) - // only display top 5 - for i, t := range sortedTriesByNums { - if i > 5 { + stat := &stat{} + for i := range s.depth { + if s.depth[i].empty() { break } - if stat, ok := inspect.result[t.ownerAddress]; !ok { - log.Error("Storage trie stat not found", "ownerAddress", t.ownerAddress) - } else { - fmt.Print(stat.Display(t.ownerAddress, "ContractTrie")) + short, full, value := s.depth[i].load() + table.Append([]string{"-", fmt.Sprint(i), fmt.Sprint(short), fmt.Sprint(full), fmt.Sprint(value)}) + stat.add(&s.depth[i]) + } + short, full, value := stat.load() + table.SetFooter([]string{"Total", "", fmt.Sprint(short), fmt.Sprint(full), fmt.Sprint(value)}) + table.Render() + fmt.Print(b.String()) + fmt.Println("Max depth", s.maxDepth()) + fmt.Println() +} + +func (in *inspector) DisplayResult() { + fmt.Println("Results for trie", in.root) + in.stats[in.root].display("Accounts trie") + + fmt.Println("===") + fmt.Println() + if in.storage { + fmt.Println("Results for top storage tries") + keys, stats := trieStatByDepth(in.stats).sort() + for i := range keys[0:min(10, len(keys))] { + fmt.Printf("%d: %s\n", i+1, keys[i]) + stats[i].display("storage trie") } } - fmt.Printf("Contract Trie, total trie num: %v, ShortNodeCnt: %v, FullNodeCnt: %v, ValueNodeCnt: %v\n", - contractTrieCnt, totalContactsNodeStat.ShortNodeCnt.Load(), totalContactsNodeStat.FullNodeCnt.Load(), totalContactsNodeStat.ValueNodeCnt.Load()) } diff --git a/trie/inspect_test.go b/trie/inspect_test.go new file mode 100644 index 00000000000..0ed03fb8d90 --- /dev/null +++ b/trie/inspect_test.go @@ -0,0 +1,91 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "math/rand" + "testing" + + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/holiman/uint256" +) + +func TestInspect(t *testing.T) { + db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) + trie := NewEmpty(db) + addresses, accounts := makeAccountsWithStorage(db, 500, true) + for i := 0; i < len(addresses); i++ { + trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) + } + // Insert the accounts into the trie and hash it + root, nodes := trie.Commit(true) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Commit(root) + + if err := InspectTrie(db, root, true); err != nil { + t.Fatalf("inspect failed: %v", err) + } +} + +func makeAccountsWithStorage(db *testDb, size int, storage bool) (addresses [][20]byte, accounts [][]byte) { + // Make the random benchmark deterministic + random := rand.New(rand.NewSource(0)) + + addresses = make([][20]byte, size) + for i := 0; i < len(addresses); i++ { + data := make([]byte, 20) + random.Read(data) + copy(addresses[i][:], data) + } + accounts = make([][]byte, len(addresses)) + for i := 0; i < len(accounts); i++ { + var ( + nonce = uint64(random.Int63()) + root = types.EmptyRootHash + code = crypto.Keccak256(nil) + ) + if storage { + trie := NewEmpty(db) + for range random.Uint32()%256 + 1 { // non-zero + k, v := make([]byte, 32), make([]byte, 32) + random.Read(k) + random.Read(v) + trie.MustUpdate(k, v) + } + var nodes *trienode.NodeSet + root, nodes = trie.Commit(true) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Commit(root) + } + numBytes := random.Uint32() % 33 // [0, 32] bytes + balanceBytes := make([]byte, numBytes) + random.Read(balanceBytes) + balance := new(uint256.Int).SetBytes(balanceBytes) + data, _ := rlp.EncodeToBytes(&types.StateAccount{ + Nonce: nonce, + Balance: balance, + Root: root, + CodeHash: code, + }) + accounts[i] = data + } + return addresses, accounts +} From ec5831d20863d2b2d720f1827b0be399db755bea Mon Sep 17 00:00:00 2001 From: MariusVanDerWijden Date: Tue, 21 Oct 2025 10:31:45 +0200 Subject: [PATCH 4/6] cmd/geth: fix lint --- cmd/geth/dbcmd.go | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index f041658b723..f836ff07126 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -23,7 +23,6 @@ import ( "os" "os/signal" "path/filepath" - "runtime" "slices" "strconv" "strings" @@ -409,7 +408,7 @@ func inspectTrie(ctx *cli.Context) error { var ( blockNumber uint64 trieRootHash common.Hash - jobnum uint64 + accountOnly bool ) stack, _ := makeConfigNode(ctx) @@ -438,12 +437,10 @@ func inspectTrie(ctx *cli.Context) error { } } - // Configure number of threads. - if ctx.NArg() <= 1 { - jobnum = uint64(runtime.NumCPU()) - } else { + // Default to inspect storage + if ctx.NArg() > 1 { var err error - jobnum, err = strconv.ParseUint(ctx.Args().Get(1), 10, 64) + accountOnly, err = strconv.ParseBool(ctx.Args().Get(1)) if err != nil { return fmt.Errorf("failed to parse jobnum, Args[1]: %v, err: %v", ctx.Args().Get(1), err) } @@ -468,17 +465,9 @@ func inspectTrie(ctx *cli.Context) error { triedb := utils.MakeTrieDatabase(ctx, stack, db, false, true, false) defer triedb.Close() - theTrie, err := trie.New(trie.TrieID(trieRootHash), triedb) - if err != nil { - fmt.Printf("fail to new trie tree, err: %v, rootHash: %v\n", err, trieRootHash.String()) - return err - } - inspector, err := trie.NewInspector(theTrie, triedb, trieRootHash, blockNumber, jobnum) - if err != nil { + if err := trie.InspectTrie(triedb, trieRootHash, accountOnly); err != nil { return err } - inspector.Run() - inspector.DisplayResult() return nil } From 7a9c1dce57c9a53fa4090589de2827fcd9c0412f Mon Sep 17 00:00:00 2001 From: MariusVanDerWijden Date: Tue, 21 Oct 2025 11:42:33 +0200 Subject: [PATCH 5/6] cmd/geth: fix lint --- cmd/geth/dbcmd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index f836ff07126..9891826a9d4 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -465,7 +465,7 @@ func inspectTrie(ctx *cli.Context) error { triedb := utils.MakeTrieDatabase(ctx, stack, db, false, true, false) defer triedb.Close() - if err := trie.InspectTrie(triedb, trieRootHash, accountOnly); err != nil { + if err := trie.InspectTrie(triedb, trieRootHash, !accountOnly); err != nil { return err } return nil From a22690d6da2ddfca4c09f851cba5dcf6f034e9ca Mon Sep 17 00:00:00 2001 From: lightclient Date: Fri, 24 Oct 2025 13:22:15 -0600 Subject: [PATCH 6/6] trie,cmd: improve results display --- cmd/geth/dbcmd.go | 75 +++++------- cmd/utils/flags.go | 5 + trie/inspect.go | 268 ++++++++++++++++++++++++------------------- trie/inspect_test.go | 12 +- 4 files changed, 195 insertions(+), 165 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 9891826a9d4..24cd3b5d5ac 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -98,14 +98,11 @@ Remove blockchain and state databases`, dbInspectTrieCmd = &cli.Command{ Action: inspectTrie, Name: "inspect-trie", - ArgsUsage: " ", - Flags: []cli.Flag{ - utils.DataDirFlag, - }, - Usage: "Print detailed trie information about the structure of account trie and storage tries.", + ArgsUsage: "", + Flags: slices.Concat([]cli.Flag{utils.ExcludeStorageFlag, utils.TopFlag}, utils.NetworkFlags, utils.DatabaseFlags), + Usage: "Print detailed trie information about the structure of account trie and storage tries.", Description: `This commands iterates the entrie trie-backed state. If the 'blocknum' is not specified, -the latest block number will be used by default. 'jobnum' indicates the number of coroutines concurrently traversing -the account and storage trie.`, +the latest block number will be used by default.`, } dbCheckStateContentCmd = &cli.Command{ Action: checkStateContent, @@ -401,71 +398,61 @@ func checkStateContent(ctx *cli.Context) error { } func inspectTrie(ctx *cli.Context) error { - if ctx.NArg() > 2 { + if ctx.NArg() > 1 { return fmt.Errorf("excessive number of arguments: %v", ctx.Command.ArgsUsage) } - - var ( - blockNumber uint64 - trieRootHash common.Hash - accountOnly bool - ) - stack, _ := makeConfigNode(ctx) - defer stack.Close() - db := utils.MakeChainDatabase(ctx, stack, false) + defer stack.Close() defer db.Close() - var headerBlockHash common.Hash + var ( + trieRoot common.Hash + hash common.Hash + number uint64 + ) switch { case ctx.NArg() == 0 || ctx.Args().Get(0) == "latest": - headerHash := rawdb.ReadHeadHeaderHash(db) - n, ok := rawdb.ReadHeaderNumber(db, headerHash) + hash := rawdb.ReadHeadHeaderHash(db) + n, ok := rawdb.ReadHeaderNumber(db, hash) if !ok { return fmt.Errorf("could not load head block hash") } - blockNumber = n + number = n case ctx.Args().Get(0) == "snapshot": - trieRootHash = rawdb.ReadSnapshotRoot(db) - blockNumber = math.MaxUint64 + trieRoot = rawdb.ReadSnapshotRoot(db) + number = math.MaxUint64 default: var err error - blockNumber, err = strconv.ParseUint(ctx.Args().Get(0), 10, 64) + number, err = strconv.ParseUint(ctx.Args().Get(0), 10, 64) if err != nil { return fmt.Errorf("failed to parse blocknum, Args[0]: %v, err: %v", ctx.Args().Get(0), err) } } - // Default to inspect storage - if ctx.NArg() > 1 { - var err error - accountOnly, err = strconv.ParseBool(ctx.Args().Get(1)) - if err != nil { - return fmt.Errorf("failed to parse jobnum, Args[1]: %v, err: %v", ctx.Args().Get(1), err) - } - } - // Load head block number based on canonical hash, if applicable. - if blockNumber != math.MaxUint64 { - headerBlockHash = rawdb.ReadCanonicalHash(db, blockNumber) - if headerBlockHash == (common.Hash{}) { - return fmt.Errorf("canonical hash for block %d not found", blockNumber) + if number != math.MaxUint64 { + hash = rawdb.ReadCanonicalHash(db, number) + if hash == (common.Hash{}) { + return fmt.Errorf("canonical hash for block %d not found", number) } - blockHeader := rawdb.ReadHeader(db, headerBlockHash, blockNumber) - trieRootHash = blockHeader.Root + blockHeader := rawdb.ReadHeader(db, hash, number) + trieRoot = blockHeader.Root } - - if (trieRootHash == common.Hash{}) { + if (trieRoot == common.Hash{}) { log.Error("Empty root hash") } - log.Debug("Inspecting trie", "root", trieRootHash, "block", blockNumber) - triedb := utils.MakeTrieDatabase(ctx, stack, db, false, true, false) defer triedb.Close() - if err := trie.InspectTrie(triedb, trieRootHash, !accountOnly); err != nil { + log.Info("Inspecting trie", "root", trieRoot, "block", number) + config := &trie.InspectConfig{ + NoStorage: ctx.Bool(utils.ExcludeStorageFlag.Name), + TopN: ctx.Int(utils.TopFlag.Name), + } + err := trie.Inspect(triedb, trieRoot, config) + if err != nil { return err } return nil diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 5e96185dbd0..36c67e0b3e5 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -220,6 +220,11 @@ var ( Usage: "Max number of elements (0 = no limit)", Value: 0, } + TopFlag = &cli.IntFlag{ + Name: "top", + Usage: "Print the top N results", + Value: 5, + } SnapshotFlag = &cli.BoolFlag{ Name: "snapshot", diff --git a/trie/inspect.go b/trie/inspect.go index a44878e6440..d68f8802990 100644 --- a/trie/inspect.go +++ b/trie/inspect.go @@ -19,7 +19,6 @@ package trie import ( "bytes" "fmt" - "slices" "sort" "strings" "sync" @@ -28,125 +27,73 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/internal/tablewriter" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/triedb/database" "golang.org/x/sync/semaphore" ) +// inspector is used by the inner inspect function to coordinate across threads. type inspector struct { triedb database.NodeDatabase - trie *Trie root common.Hash - storage bool - stats map[common.Hash]*triestat - m sync.Mutex + config *InspectConfig + stats map[common.Hash]*triestat + m sync.Mutex // protects stats sem *semaphore.Weighted wg sync.WaitGroup } -type triestat struct { - depth [15]stat -} - -func (s *triestat) maxDepth() int { - depth := 0 - for i := range s.depth { - if s.depth[i].short.Load() != 0 || s.depth[i].full.Load() != 0 || s.depth[i].value.Load() != 0 { - depth = i - } - } - return depth -} - -type trieStatByDepth map[common.Hash]*triestat - -func (s trieStatByDepth) sort() ([]common.Hash, []*triestat) { - var ( - keys = make([]common.Hash, 0, len(s)) - stats = make([]*triestat, 0, len(s)) - ) - for k := range s { - keys = append(keys, k) - } - sort.Slice(keys, func(i, j int) bool { return s[keys[i]].maxDepth() > s[keys[j]].maxDepth() }) - for _, k := range keys { - stats = append(stats, s[k]) - } - return keys, stats +// InspectConfig is a set of options to control inspection and format the +// output. TopN will print the deepest min(len(results), N) storage tries. +type InspectConfig struct { + NoStorage bool + TopN int } -func (s *triestat) add(n node, d uint32) { - switch (n).(type) { - case *shortNode: - s.depth[d].short.Add(1) - case *fullNode: - s.depth[d].full.Add(1) - case valueNode: - s.depth[d].value.Add(1) - default: - panic(fmt.Sprintf("%T: invalid node: %v", n, n)) - } -} - -type stat struct { - short atomic.Uint64 - full atomic.Uint64 - value atomic.Uint64 -} - -func (s *stat) empty() bool { - if s.full.Load() == 0 && s.short.Load() == 0 && s.value.Load() == 0 { - return true - } - return false -} - -func (s *stat) load() (uint64, uint64, uint64) { - return s.short.Load(), s.full.Load(), s.value.Load() -} - -func (s *stat) add(other *stat) *stat { - s.short.Add(other.short.Load()) - s.full.Add(other.full.Load()) - s.value.Add(other.value.Load()) - return s -} - -func InspectTrie(triedb database.NodeDatabase, root common.Hash, storage bool) error { - return inspectTrie(triedb, root, storage) -} - -func inspectTrie(triedb database.NodeDatabase, root common.Hash, storage bool) error { +// Inspect walks the trie with the given root and records the number and type of +// nodes at each depth. It works by recursively calling the inner inspect +// function on each child node. +func Inspect(triedb database.NodeDatabase, root common.Hash, config *InspectConfig) error { trie, err := New(TrieID(root), triedb) if err != nil { return fmt.Errorf("fail to open trie %s: %w", root, err) } + if config == nil { + config = &InspectConfig{} + } in := inspector{ - triedb: triedb, - trie: trie, - root: root, - storage: storage, - stats: make(map[common.Hash]*triestat), - sem: semaphore.NewWeighted(int64(128)), + triedb: triedb, + root: root, + config: config, + stats: make(map[common.Hash]*triestat), + sem: semaphore.NewWeighted(int64(128)), } in.stats[root] = &triestat{} - in.inspect(trie.root, 0, []byte{}, in.stats[root]) + in.inspect(trie, trie.root, 0, []byte{}, in.stats[root]) in.wg.Wait() in.DisplayResult() return nil } -func (in *inspector) inspect(n node, height uint32, path []byte, stat *triestat) { +// inspect is called recursively down the trie. At each level it records the +// node type encountered. +func (in *inspector) inspect(trie *Trie, n node, height uint32, path []byte, stat *triestat) { if n == nil { return } + // Four types of nodes can be encountered: + // - short: extend path with key, inspect single value. + // - full: inspect all 17 children, spin up new threads when possible. + // - hash: need to resolve node from disk, retry inspect on result. + // - value: if account, begin inspecting storage trie. switch n := (n).(type) { case *shortNode: - in.inspect(n.Val, height, append(path, n.Key...), stat) + in.inspect(trie, n.Val, height+1, append(path, n.Key...), stat) case *fullNode: for idx, child := range n.Children { if child == nil { @@ -156,20 +103,22 @@ func (in *inspector) inspect(n node, height uint32, path []byte, stat *triestat) if in.sem.TryAcquire(1) { in.wg.Add(1) go func() { - in.inspect(child, height+1, slices.Clone(childPath), stat) + in.inspect(trie, child, height+1, childPath, stat) in.wg.Done() }() } else { - in.inspect(child, height+1, childPath, stat) + in.inspect(trie, child, height+1, childPath, stat) } } case hashNode: - resolved, err := in.trie.resolveWithoutTrack(n, path) + resolved, err := trie.resolveWithoutTrack(n, path) if err != nil { - fmt.Printf("Resolve HashNode error: %v, TrieRoot: %v, Height: %v, Path: %v\n", err, in.trie.Hash().String(), height+1, path) + log.Error("Failed to resolve HashNode", "err", err, "trie", trie.Hash(), "height", height+1, "path", path) return } - in.inspect(resolved, height, path, stat) + in.inspect(trie, resolved, height, path, stat) + + // Return early here so this level isn't recorded twice. return case valueNode: if !hasTerm(path) { @@ -180,23 +129,19 @@ func (in *inspector) inspect(n node, height uint32, path []byte, stat *triestat) // Not an account value. break } - // TODO: update for 7702 - // if common.BytesToHash(account.CodeHash) == types.EmptyCodeHash { - // inspect.eoaAccountNums.Add(1) - // } if account.Root == (common.Hash{}) || account.Root == types.EmptyRootHash { + // Account is empty, nothing further to inspect. break } // Start inspecting storage trie. - if in.storage { + if !in.config.NoStorage { owner := common.BytesToHash(hexToCompact(path)) storage, err := New(StorageTrieID(in.root, owner, account.Root), in.triedb) if err != nil { - fmt.Printf("New contract trie node: %v, error: %v, Height: %v, Path: %v\n", n, err, height, path) + log.Error("Failed to open account storage trie", "node", n, "error", err, "height", height, "path", common.Bytes2Hex(path)) break } - // contractTrie.opTracer.reset() stat := &triestat{} in.m.Lock() @@ -205,7 +150,7 @@ func (in *inspector) inspect(n node, height uint32, path []byte, stat *triestat) in.wg.Add(1) go func() { - in.inspect(storage.root, 0, []byte{}, stat) + in.inspect(storage, storage.root, 0, []byte{}, stat) in.wg.Done() }() } @@ -217,6 +162,109 @@ func (in *inspector) inspect(n node, height uint32, path []byte, stat *triestat) stat.add(n, height) } +// Display results prints out the inspect results. +func (in *inspector) DisplayResult() { + fmt.Println("Results for trie", in.root) + in.stats[in.root].display("Accounts trie") + fmt.Println("===") + fmt.Println() + + if !in.config.NoStorage { + // Sort stats by max node depth. + keys, stats := sortedTriestat(in.stats).sort() + + fmt.Println("Results for top storage tries") + for i := range keys[0:min(in.config.TopN, len(keys))] { + fmt.Printf("%d: %s\n", i+1, keys[i]) + stats[i].display("storage trie") + } + } +} + +// triestat tracks the type and count of trie nodes at each level in the trie. +// +// Note: theoretically it is possible to have up to 64 trie level. Since it is +// unlikely to encounter such a large trie, the stats are capped at 16 levels to +// avoid substantial unneeded allocation. +type triestat struct { + level [16]stat +} + +// maxDepth iterates each level and finds the deepest level with at least one +// trie node. +func (s *triestat) maxDepth() int { + depth := 0 + for i := range s.level { + if s.level[i].short.Load() != 0 || s.level[i].full.Load() != 0 || s.level[i].value.Load() != 0 { + depth = i + } + } + return depth +} + +// sortedTriestat implements sort(). +type sortedTriestat map[common.Hash]*triestat + +// sort returns the keys and triestats in decending order of the maximum trie +// node depth. +func (s sortedTriestat) sort() ([]common.Hash, []*triestat) { + var ( + keys = make([]common.Hash, 0, len(s)) + stats = make([]*triestat, 0, len(s)) + ) + for k := range s { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { return s[keys[i]].maxDepth() > s[keys[j]].maxDepth() }) + for _, k := range keys { + stats = append(stats, s[k]) + } + return keys, stats +} + +// add increases the node count by one for the specified node type and depth. +func (s *triestat) add(n node, d uint32) { + switch (n).(type) { + case *shortNode: + s.level[d].short.Add(1) + case *fullNode: + s.level[d].full.Add(1) + case valueNode: + s.level[d].value.Add(1) + default: + panic(fmt.Sprintf("%T: invalid node: %v", n, n)) + } +} + +// stat is a specific level's count of each node type. +type stat struct { + short atomic.Uint64 + full atomic.Uint64 + value atomic.Uint64 +} + +// empty is a helper that returns whether there are any trie nodes at the level. +func (s *stat) empty() bool { + if s.full.Load() == 0 && s.short.Load() == 0 && s.value.Load() == 0 { + return true + } + return false +} + +// load is a helper that loads each node type's value. +func (s *stat) load() (uint64, uint64, uint64) { + return s.short.Load(), s.full.Load(), s.value.Load() +} + +// add is a helper that adds two level's stats together. +func (s *stat) add(other *stat) *stat { + s.short.Add(other.short.Load()) + s.full.Add(other.full.Load()) + s.value.Add(other.value.Load()) + return s +} + +// display will print a table displaying the trie's node statistics. func (s *triestat) display(title string) { // Shorten title if too long. if len(title) > 32 { @@ -229,13 +277,13 @@ func (s *triestat) display(title string) { table.SetAlignment(1) stat := &stat{} - for i := range s.depth { - if s.depth[i].empty() { + for i := range s.level { + if s.level[i].empty() { break } - short, full, value := s.depth[i].load() + short, full, value := s.level[i].load() table.Append([]string{"-", fmt.Sprint(i), fmt.Sprint(short), fmt.Sprint(full), fmt.Sprint(value)}) - stat.add(&s.depth[i]) + stat.add(&s.level[i]) } short, full, value := stat.load() table.SetFooter([]string{"Total", "", fmt.Sprint(short), fmt.Sprint(full), fmt.Sprint(value)}) @@ -244,19 +292,3 @@ func (s *triestat) display(title string) { fmt.Println("Max depth", s.maxDepth()) fmt.Println() } - -func (in *inspector) DisplayResult() { - fmt.Println("Results for trie", in.root) - in.stats[in.root].display("Accounts trie") - - fmt.Println("===") - fmt.Println() - if in.storage { - fmt.Println("Results for top storage tries") - keys, stats := trieStatByDepth(in.stats).sort() - for i := range keys[0:min(10, len(keys))] { - fmt.Printf("%d: %s\n", i+1, keys[i]) - stats[i].display("storage trie") - } - } -} diff --git a/trie/inspect_test.go b/trie/inspect_test.go index 0ed03fb8d90..eb9e16722a2 100644 --- a/trie/inspect_test.go +++ b/trie/inspect_test.go @@ -28,10 +28,16 @@ import ( "github.com/holiman/uint256" ) +// TestInspect inspects a randomly generated account trie. It's useful for +// quickly verifying changes to the results display. func TestInspect(t *testing.T) { db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) - trie := NewEmpty(db) - addresses, accounts := makeAccountsWithStorage(db, 500, true) + trie, err := NewStateTrie(TrieID(types.EmptyRootHash), db) + if err != nil { + t.Fatalf("failed to create state trie: %v", err) + } + // Create a realistic looking account trie with storage. + addresses, accounts := makeAccountsWithStorage(db, 11, true) for i := 0; i < len(addresses); i++ { trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -40,7 +46,7 @@ func TestInspect(t *testing.T) { db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) db.Commit(root) - if err := InspectTrie(db, root, true); err != nil { + if err := Inspect(db, root, &InspectConfig{TopN: 1}); err != nil { t.Fatalf("inspect failed: %v", err) } }