From deb2cf51a9178b182c96e47d128cca935d675bbd Mon Sep 17 00:00:00 2001 From: bianyuanop Date: Fri, 19 Jan 2024 12:14:12 -0500 Subject: [PATCH 1/8] root of transactions by stateless merkledb --- chain/block.go | 46 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/chain/block.go b/chain/block.go index 4aab647fe4..d334c3df1e 100644 --- a/chain/block.go +++ b/chain/block.go @@ -9,12 +9,15 @@ import ( "fmt" "time" + "github.com/ava-labs/avalanchego/database" + "github.com/ava-labs/avalanchego/database/memdb" "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/snow/choices" "github.com/ava-labs/avalanchego/snow/consensus/snowman" "github.com/ava-labs/avalanchego/snow/engine/snowman/block" "github.com/ava-labs/avalanchego/snow/validators" "github.com/ava-labs/avalanchego/utils/set" + "github.com/ava-labs/avalanchego/utils/units" "github.com/ava-labs/avalanchego/vms/platformvm/warp" "github.com/ava-labs/avalanchego/x/merkledb" "go.opentelemetry.io/otel/attribute" @@ -40,7 +43,8 @@ type StatefulBlock struct { Tmstmp int64 `json:"timestamp"` Hght uint64 `json:"height"` - Txs []*Transaction `json:"txs"` + Txs []*Transaction `json:"txs"` + TxsRoot []byte `json:"txsRoot"` // StateRoot is the root of the post-execution state // of [Prnt]. @@ -290,6 +294,46 @@ func (b *StatelessBlock) initializeBuilt( b.containsWarp = true } } + + // transaction hash generation + db, err := merkledb.New(ctx, memdb.New(), merkledb.Config{ + BranchFactor: merkledb.BranchFactor16, + HistoryLength: 100, + EvictionBatchSize: units.MiB, + IntermediateNodeCacheSize: units.MiB, + ValueNodeCacheSize: units.MiB, + Tracer: b.vm.Tracer(), + }) + if err != nil { + return err + } + // collect keys, values from transactions/results + var ops []database.BatchOp + for _, tx := range b.Txs { + key := utils.ToID(tx.Bytes()) + ops = append(ops, database.BatchOp{ + Key: key[:], + Value: tx.Bytes(), + }) + } + for _, result := range b.results { + key := utils.ToID(result.Output) + ops = append(ops, database.BatchOp{ + Key: key[:], + Value: result.Output, + }) + } + view, err = db.NewView(ctx, merkledb.ViewChanges{BatchOps: ops}) + if err != nil { + return err + } + view.CommitToDB(ctx) + txsRoot, err := db.GetMerkleRoot(ctx) + if err != nil { + return err + } + b.TxsRoot = txsRoot[:] + return nil } From 0804930f27112107e5f74b4168b9db127a2bc475 Mon Sep 17 00:00:00 2001 From: bianyuanop Date: Fri, 19 Jan 2024 16:36:26 -0500 Subject: [PATCH 2/8] make root generation a function --- chain/block.go | 39 ++++++--------------------------------- utils/utils.go | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 33 deletions(-) diff --git a/chain/block.go b/chain/block.go index d334c3df1e..6eca2967cd 100644 --- a/chain/block.go +++ b/chain/block.go @@ -9,15 +9,12 @@ import ( "fmt" "time" - "github.com/ava-labs/avalanchego/database" - "github.com/ava-labs/avalanchego/database/memdb" "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/snow/choices" "github.com/ava-labs/avalanchego/snow/consensus/snowman" "github.com/ava-labs/avalanchego/snow/engine/snowman/block" "github.com/ava-labs/avalanchego/snow/validators" "github.com/ava-labs/avalanchego/utils/set" - "github.com/ava-labs/avalanchego/utils/units" "github.com/ava-labs/avalanchego/vms/platformvm/warp" "github.com/ava-labs/avalanchego/x/merkledb" "go.opentelemetry.io/otel/attribute" @@ -296,43 +293,19 @@ func (b *StatelessBlock) initializeBuilt( } // transaction hash generation - db, err := merkledb.New(ctx, memdb.New(), merkledb.Config{ - BranchFactor: merkledb.BranchFactor16, - HistoryLength: 100, - EvictionBatchSize: units.MiB, - IntermediateNodeCacheSize: units.MiB, - ValueNodeCacheSize: units.MiB, - Tracer: b.vm.Tracer(), - }) - if err != nil { - return err - } - // collect keys, values from transactions/results - var ops []database.BatchOp + var merkleItems [][]byte for _, tx := range b.Txs { - key := utils.ToID(tx.Bytes()) - ops = append(ops, database.BatchOp{ - Key: key[:], - Value: tx.Bytes(), - }) + merkleItems = append(merkleItems, tx.Bytes()) } for _, result := range b.results { - key := utils.ToID(result.Output) - ops = append(ops, database.BatchOp{ - Key: key[:], - Value: result.Output, - }) + merkleItems = append(merkleItems, result.Output) } - view, err = db.NewView(ctx, merkledb.ViewChanges{BatchOps: ops}) - if err != nil { - return err - } - view.CommitToDB(ctx) - txsRoot, err := db.GetMerkleRoot(ctx) + + root, _, err := utils.GenerateMerkleRoot(ctx, b.vm.Tracer(), merkleItems) if err != nil { return err } - b.TxsRoot = txsRoot[:] + b.TxsRoot = root return nil } diff --git a/utils/utils.go b/utils/utils.go index 2635e1aa4b..86384e1034 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -4,6 +4,7 @@ package utils import ( + "context" "fmt" "math" "net" @@ -13,9 +14,14 @@ import ( "strconv" "time" + "github.com/ava-labs/avalanchego/database" + "github.com/ava-labs/avalanchego/database/memdb" "github.com/ava-labs/avalanchego/ids" + "github.com/ava-labs/avalanchego/trace" "github.com/ava-labs/avalanchego/utils/hashing" "github.com/ava-labs/avalanchego/utils/perms" + "github.com/ava-labs/avalanchego/utils/units" + "github.com/ava-labs/avalanchego/x/merkledb" "github.com/ava-labs/hypersdk/consts" formatter "github.com/onsi/ginkgo/v2/formatter" ) @@ -116,3 +122,44 @@ func LoadBytes(filename string, expectedSize int) ([]byte, error) { } return bytes, nil } + +// Generate merkle root for a set of items +// this function does not take ownership of given bytes array +func GenerateMerkleRoot(ctx context.Context, tracer trace.Tracer, merkleItems [][]byte) ([]byte, merkledb.MerkleDB, error) { + var batchOps []database.BatchOp + + for _, item := range merkleItems { + key := ToID(item) + batchOps = append(batchOps, database.BatchOp{ + Key: key[:], + Value: item, + }) + } + + db, err := merkledb.New(ctx, memdb.New(), merkledb.Config{ + BranchFactor: merkledb.BranchFactor16, + HistoryLength: 100, + EvictionBatchSize: units.MiB, + IntermediateNodeCacheSize: units.MiB, + ValueNodeCacheSize: units.MiB, + Tracer: tracer, + }) + if err != nil { + return nil, nil, err + } + + view, err := db.NewView(ctx, merkledb.ViewChanges{BatchOps: batchOps}) + if err != nil { + return nil, nil, err + } + if err := view.CommitToDB(ctx); err != nil { + return nil, nil, err + } + + root, err := db.GetMerkleRoot(ctx) + if err != nil { + return nil, nil, err + } + + return root[:], db, nil +} From eb41d9a7a9ea9902fbaf0531642a166c4b505cfd Mon Sep 17 00:00:00 2001 From: bianyuanop Date: Fri, 19 Jan 2024 17:05:46 -0500 Subject: [PATCH 3/8] preallocate memory for merkle array and consumebytes flag --- chain/block.go | 5 +++-- hypersdk.code-workspace | 8 ++++++++ utils/utils.go | 6 +++--- 3 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 hypersdk.code-workspace diff --git a/chain/block.go b/chain/block.go index 6eca2967cd..03eedd7e09 100644 --- a/chain/block.go +++ b/chain/block.go @@ -293,7 +293,7 @@ func (b *StatelessBlock) initializeBuilt( } // transaction hash generation - var merkleItems [][]byte + merkleItems := make([][]byte, 0, len(b.Txs)+len(b.results)) for _, tx := range b.Txs { merkleItems = append(merkleItems, tx.Bytes()) } @@ -301,7 +301,8 @@ func (b *StatelessBlock) initializeBuilt( merkleItems = append(merkleItems, result.Output) } - root, _, err := utils.GenerateMerkleRoot(ctx, b.vm.Tracer(), merkleItems) + // consume bytes to avoid extra copying + root, _, err := utils.GenerateMerkleRoot(ctx, b.vm.Tracer(), merkleItems, true) if err != nil { return err } diff --git a/hypersdk.code-workspace b/hypersdk.code-workspace new file mode 100644 index 0000000000..876a1499c0 --- /dev/null +++ b/hypersdk.code-workspace @@ -0,0 +1,8 @@ +{ + "folders": [ + { + "path": "." + } + ], + "settings": {} +} \ No newline at end of file diff --git a/utils/utils.go b/utils/utils.go index 86384e1034..fac2f0bacc 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -125,8 +125,8 @@ func LoadBytes(filename string, expectedSize int) ([]byte, error) { // Generate merkle root for a set of items // this function does not take ownership of given bytes array -func GenerateMerkleRoot(ctx context.Context, tracer trace.Tracer, merkleItems [][]byte) ([]byte, merkledb.MerkleDB, error) { - var batchOps []database.BatchOp +func GenerateMerkleRoot(ctx context.Context, tracer trace.Tracer, merkleItems [][]byte, consumeBytes bool) ([]byte, merkledb.MerkleDB, error) { + batchOps := make([]database.BatchOp, 0, len(merkleItems)) for _, item := range merkleItems { key := ToID(item) @@ -148,7 +148,7 @@ func GenerateMerkleRoot(ctx context.Context, tracer trace.Tracer, merkleItems [] return nil, nil, err } - view, err := db.NewView(ctx, merkledb.ViewChanges{BatchOps: batchOps}) + view, err := db.NewView(ctx, merkledb.ViewChanges{BatchOps: batchOps, ConsumeBytes: consumeBytes}) if err != nil { return nil, nil, err } From 77c79524cbf6b9706060e91434782637c019d08f Mon Sep 17 00:00:00 2001 From: bianyuanop Date: Fri, 19 Jan 2024 17:28:24 -0500 Subject: [PATCH 4/8] remove wrong merkleroot docstring --- utils/utils.go | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/utils.go b/utils/utils.go index fac2f0bacc..b7f5f3feca 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -124,7 +124,6 @@ func LoadBytes(filename string, expectedSize int) ([]byte, error) { } // Generate merkle root for a set of items -// this function does not take ownership of given bytes array func GenerateMerkleRoot(ctx context.Context, tracer trace.Tracer, merkleItems [][]byte, consumeBytes bool) ([]byte, merkledb.MerkleDB, error) { batchOps := make([]database.BatchOp, 0, len(merkleItems)) From 6dc40e9a44197a7dbf99da1a11e71584bd87c4a5 Mon Sep 17 00:00:00 2001 From: bianyuanop Date: Fri, 19 Jan 2024 21:53:33 -0500 Subject: [PATCH 5/8] add <*.code-workspace> to .gitignore and remove it from git commit --- .gitignore | 2 ++ hypersdk.code-workspace | 8 -------- 2 files changed, 2 insertions(+), 8 deletions(-) delete mode 100644 hypersdk.code-workspace diff --git a/.gitignore b/.gitignore index c556d1e590..410da12571 100644 --- a/.gitignore +++ b/.gitignore @@ -68,3 +68,5 @@ osxcross/ target/ Cargo.lock **/*.rs.bk + +*.code-workspace \ No newline at end of file diff --git a/hypersdk.code-workspace b/hypersdk.code-workspace deleted file mode 100644 index 876a1499c0..0000000000 --- a/hypersdk.code-workspace +++ /dev/null @@ -1,8 +0,0 @@ -{ - "folders": [ - { - "path": "." - } - ], - "settings": {} -} \ No newline at end of file From 09b0b6baf2d37620ca91b2931731292bcee22e94 Mon Sep 17 00:00:00 2001 From: bianyuanop Date: Fri, 19 Jan 2024 22:11:51 -0500 Subject: [PATCH 6/8] move root generation func to merkle package, tx root by items of [txID + result] --- chain/block.go | 16 ++++++++------- merkle/merkle.go | 52 ++++++++++++++++++++++++++++++++++++++++++++++++ utils/utils.go | 46 ------------------------------------------ 3 files changed, 61 insertions(+), 53 deletions(-) create mode 100644 merkle/merkle.go diff --git a/chain/block.go b/chain/block.go index 03eedd7e09..ff731ef28b 100644 --- a/chain/block.go +++ b/chain/block.go @@ -23,6 +23,7 @@ import ( "github.com/ava-labs/hypersdk/codec" "github.com/ava-labs/hypersdk/consts" + "github.com/ava-labs/hypersdk/merkle" "github.com/ava-labs/hypersdk/state" "github.com/ava-labs/hypersdk/utils" "github.com/ava-labs/hypersdk/window" @@ -293,16 +294,17 @@ func (b *StatelessBlock) initializeBuilt( } // transaction hash generation - merkleItems := make([][]byte, 0, len(b.Txs)+len(b.results)) - for _, tx := range b.Txs { - merkleItems = append(merkleItems, tx.Bytes()) - } - for _, result := range b.results { - merkleItems = append(merkleItems, result.Output) + // [len(b.Txs)] should be equal to [b.results] + merkleItems := make([][]byte, 0, len(b.Txs)) + for i := 0; i < len(b.Txs); i++ { + txID := b.Txs[i].ID() + resultOutput := b.results[i].Output + // [txID + resultOutput] + merkleItems = append(merkleItems, append(txID[:], resultOutput...)) } // consume bytes to avoid extra copying - root, _, err := utils.GenerateMerkleRoot(ctx, b.vm.Tracer(), merkleItems, true) + root, _, err := merkle.GenerateMerkleRoot(ctx, b.vm.Tracer(), merkleItems, true) if err != nil { return err } diff --git a/merkle/merkle.go b/merkle/merkle.go new file mode 100644 index 0000000000..53333b7692 --- /dev/null +++ b/merkle/merkle.go @@ -0,0 +1,52 @@ +package merkle + +import ( + "context" + + "github.com/ava-labs/avalanchego/database" + "github.com/ava-labs/avalanchego/database/memdb" + "github.com/ava-labs/avalanchego/trace" + "github.com/ava-labs/avalanchego/utils/units" + "github.com/ava-labs/avalanchego/x/merkledb" + "github.com/ava-labs/hypersdk/utils" +) + +// Generate merkle root for a set of items +func GenerateMerkleRoot(ctx context.Context, tracer trace.Tracer, merkleItems [][]byte, consumeBytes bool) ([]byte, merkledb.MerkleDB, error) { + batchOps := make([]database.BatchOp, 0, len(merkleItems)) + + for _, item := range merkleItems { + key := utils.ToID(item) + batchOps = append(batchOps, database.BatchOp{ + Key: key[:], + Value: item, + }) + } + + db, err := merkledb.New(ctx, memdb.New(), merkledb.Config{ + BranchFactor: merkledb.BranchFactor16, + HistoryLength: 100, + EvictionBatchSize: units.MiB, + IntermediateNodeCacheSize: units.MiB, + ValueNodeCacheSize: units.MiB, + Tracer: tracer, + }) + if err != nil { + return nil, nil, err + } + + view, err := db.NewView(ctx, merkledb.ViewChanges{BatchOps: batchOps, ConsumeBytes: consumeBytes}) + if err != nil { + return nil, nil, err + } + if err := view.CommitToDB(ctx); err != nil { + return nil, nil, err + } + + root, err := db.GetMerkleRoot(ctx) + if err != nil { + return nil, nil, err + } + + return root[:], db, nil +} diff --git a/utils/utils.go b/utils/utils.go index b7f5f3feca..2635e1aa4b 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -4,7 +4,6 @@ package utils import ( - "context" "fmt" "math" "net" @@ -14,14 +13,9 @@ import ( "strconv" "time" - "github.com/ava-labs/avalanchego/database" - "github.com/ava-labs/avalanchego/database/memdb" "github.com/ava-labs/avalanchego/ids" - "github.com/ava-labs/avalanchego/trace" "github.com/ava-labs/avalanchego/utils/hashing" "github.com/ava-labs/avalanchego/utils/perms" - "github.com/ava-labs/avalanchego/utils/units" - "github.com/ava-labs/avalanchego/x/merkledb" "github.com/ava-labs/hypersdk/consts" formatter "github.com/onsi/ginkgo/v2/formatter" ) @@ -122,43 +116,3 @@ func LoadBytes(filename string, expectedSize int) ([]byte, error) { } return bytes, nil } - -// Generate merkle root for a set of items -func GenerateMerkleRoot(ctx context.Context, tracer trace.Tracer, merkleItems [][]byte, consumeBytes bool) ([]byte, merkledb.MerkleDB, error) { - batchOps := make([]database.BatchOp, 0, len(merkleItems)) - - for _, item := range merkleItems { - key := ToID(item) - batchOps = append(batchOps, database.BatchOp{ - Key: key[:], - Value: item, - }) - } - - db, err := merkledb.New(ctx, memdb.New(), merkledb.Config{ - BranchFactor: merkledb.BranchFactor16, - HistoryLength: 100, - EvictionBatchSize: units.MiB, - IntermediateNodeCacheSize: units.MiB, - ValueNodeCacheSize: units.MiB, - Tracer: tracer, - }) - if err != nil { - return nil, nil, err - } - - view, err := db.NewView(ctx, merkledb.ViewChanges{BatchOps: batchOps, ConsumeBytes: consumeBytes}) - if err != nil { - return nil, nil, err - } - if err := view.CommitToDB(ctx); err != nil { - return nil, nil, err - } - - root, err := db.GetMerkleRoot(ctx) - if err != nil { - return nil, nil, err - } - - return root[:], db, nil -} From 7586f403750b9232520506758ee6e850a3b78f17 Mon Sep 17 00:00:00 2001 From: bianyuanop Date: Sat, 20 Jan 2024 09:03:31 -0500 Subject: [PATCH 7/8] comments about appending outputs to slice of txID --- chain/block.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/chain/block.go b/chain/block.go index ff731ef28b..af58104280 100644 --- a/chain/block.go +++ b/chain/block.go @@ -300,6 +300,9 @@ func (b *StatelessBlock) initializeBuilt( txID := b.Txs[i].ID() resultOutput := b.results[i].Output // [txID + resultOutput] + // txID is a fixed length array, hence [append] will always allocate new memory and copy + // so slice with new address will be returned and no reflect on txID, then later + // we consume those bytes merkleItems = append(merkleItems, append(txID[:], resultOutput...)) } From f009424f24e9a01b4d1fb83492ad8fdfd533d071 Mon Sep 17 00:00:00 2001 From: bianyuanop Date: Sun, 28 Jan 2024 09:30:49 -0500 Subject: [PATCH 8/8] rebase & blk marshal/unmarshal & merkleroot to ids.ID --- chain/block.go | 5 ++++- merkle/merkle.go | 14 +++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/chain/block.go b/chain/block.go index af58104280..3a42c1a1c5 100644 --- a/chain/block.go +++ b/chain/block.go @@ -42,7 +42,7 @@ type StatefulBlock struct { Hght uint64 `json:"height"` Txs []*Transaction `json:"txs"` - TxsRoot []byte `json:"txsRoot"` + TxsRoot ids.ID `json:"txsRoot"` // StateRoot is the root of the post-execution state // of [Prnt]. @@ -1017,6 +1017,8 @@ func (b *StatefulBlock) Marshal() ([]byte, error) { p.PackID(b.StateRoot) p.PackUint64(uint64(b.WarpResults)) + p.PackID(b.TxsRoot) + bytes := p.Bytes() if err := p.Err(); err != nil { return nil, err @@ -1052,6 +1054,7 @@ func UnmarshalBlock(raw []byte, parser Parser) (*StatefulBlock, error) { p.UnpackID(false, &b.StateRoot) b.WarpResults = set.Bits64(p.UnpackUint64(false)) + p.UnpackID(false, &b.TxsRoot) // Ensure no leftover bytes if !p.Empty() { diff --git a/merkle/merkle.go b/merkle/merkle.go index 53333b7692..e76e7efbad 100644 --- a/merkle/merkle.go +++ b/merkle/merkle.go @@ -5,6 +5,7 @@ import ( "github.com/ava-labs/avalanchego/database" "github.com/ava-labs/avalanchego/database/memdb" + "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/trace" "github.com/ava-labs/avalanchego/utils/units" "github.com/ava-labs/avalanchego/x/merkledb" @@ -12,7 +13,7 @@ import ( ) // Generate merkle root for a set of items -func GenerateMerkleRoot(ctx context.Context, tracer trace.Tracer, merkleItems [][]byte, consumeBytes bool) ([]byte, merkledb.MerkleDB, error) { +func GenerateMerkleRoot(ctx context.Context, tracer trace.Tracer, merkleItems [][]byte, consumeBytes bool) (ids.ID, merkledb.MerkleDB, error) { batchOps := make([]database.BatchOp, 0, len(merkleItems)) for _, item := range merkleItems { @@ -26,27 +27,26 @@ func GenerateMerkleRoot(ctx context.Context, tracer trace.Tracer, merkleItems [] db, err := merkledb.New(ctx, memdb.New(), merkledb.Config{ BranchFactor: merkledb.BranchFactor16, HistoryLength: 100, - EvictionBatchSize: units.MiB, IntermediateNodeCacheSize: units.MiB, ValueNodeCacheSize: units.MiB, Tracer: tracer, }) if err != nil { - return nil, nil, err + return ids.Empty, nil, err } view, err := db.NewView(ctx, merkledb.ViewChanges{BatchOps: batchOps, ConsumeBytes: consumeBytes}) if err != nil { - return nil, nil, err + return ids.Empty, nil, err } if err := view.CommitToDB(ctx); err != nil { - return nil, nil, err + return ids.Empty, nil, err } root, err := db.GetMerkleRoot(ctx) if err != nil { - return nil, nil, err + return ids.Empty, nil, err } - return root[:], db, nil + return root, db, nil }