From 5cd79a16f99a59d68a56b2e0c3c31b64bcfb7808 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 25 Sep 2023 08:13:20 -0700 Subject: [PATCH 01/72] Initial change for adding changeset proto and command line tool --- Makefile | 5 +- cmd/changeset/main.go | 141 +++++++++ diff.go | 156 ++++++++++ go.mod | 18 +- go.sum | 117 +------- immutable_tree.go | 6 + iterator.go | 74 +++++ nodedb.go | 35 +++ proto/changeset.pb.go | 599 +++++++++++++++++++++++++++++++++++++ proto/iavl/changeset.proto | 14 + 10 files changed, 1035 insertions(+), 130 deletions(-) create mode 100644 cmd/changeset/main.go create mode 100644 diff.go create mode 100644 proto/changeset.pb.go create mode 100644 proto/iavl/changeset.proto diff --git a/Makefile b/Makefile index ecba766..fabf121 100644 --- a/Makefile +++ b/Makefile @@ -15,11 +15,14 @@ all: lint test install install: ifeq ($(COLORS_ON),) go install ./cmd/iaviewer + go install ./cmd/changeset else go install $(CMDFLAGS) ./cmd/iaviewer + go install $(CMDFLAGS) ./cmd/changeset endif .PHONY: install + test-short: @echo "--> Running go test" @go test ./... $(LDFLAGS) -v --race --short @@ -125,5 +128,5 @@ proto-check-breaking: proto-gen: @echo "Generating Protobuf files" - $(DOCKER) run --rm -v $(CURDIR):/workspace --workdir /workspace tendermintdev/sdk-proto-gen:master sh scripts/protocgen.sh + $(DOCKER) run --platform linux/amd64 --rm -v $(CURDIR):/workspace --workdir /workspace tendermintdev/sdk-proto-gen:master sh scripts/protocgen.sh .PHONY: proto-gen-d diff --git a/cmd/changeset/main.go b/cmd/changeset/main.go new file mode 100644 index 0000000..45a5df7 --- /dev/null +++ b/cmd/changeset/main.go @@ -0,0 +1,141 @@ +package main + +import ( + "fmt" + "github.com/cosmos/iavl" + "github.com/pkg/errors" + "github.com/spf13/cobra" + dbm "github.com/tendermint/tm-db" + "math" + "os" + "strings" + "time" +) + +const ( + DefaultCacheSize int = 10000 + DefaultStore string = "" +) + +func main() { + rootCmd := &cobra.Command{ + Use: "changeset", + Short: "Dump change sets files which can be ingested into DB", + } + rootCmd.PersistentFlags().String("home-dir", "/root/.sei", "Home directory") + rootCmd.PersistentFlags().String("start-version", "0", "start version") + rootCmd.PersistentFlags().String("end-version", "0", "start version") + rootCmd.PersistentFlags().StringP("store", "s", DefaultStore, "store key name") + rootCmd.AddCommand(DumpChangeSetCmd()) + if err := rootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func DumpChangeSetCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "dump", + Short: "Extract changesets from iavl versions, and save to plain file format", + RunE: executeDumpChangesetCmd, + } + + // we handle one stores at a time, because stores don't share much in db, handle concurrently reduces cache efficiency. + return cmd +} + +func executeDumpChangesetCmd(cmd *cobra.Command, _ []string) error { + homeDir, _ := cmd.Flags().GetString("home-dir") + storeKey, _ := cmd.Flags().GetString("store") + startVersion, _ := cmd.Flags().GetInt64("start-version") + endVersion, _ := cmd.Flags().GetInt64("end-version") + prefix := []byte(storeKey) + if storeKey != "" { + fmt.Println("Begin dumping store", storeKey, time.Now().Format(time.RFC3339)) + db, err := OpenDB(homeDir) + if err != nil { + return nil + } + if len(storeKey) != 0 { + db = dbm.NewPrefixDB(db, prefix) + } + tree, err := iavl.NewMutableTree(db, DefaultCacheSize, true) + if err != nil { + return err + } + // Make sure we have a correct end version + if endVersion <= 0 { + latestVersion, _ := tree.LazyLoadVersion(0) + endVersion = latestVersion + 1 + } + + storeStartVersion, err := getNextVersion(dbm.NewPrefixDB(db, prefix), 0) + if storeStartVersion <= 0 { + // store not exists + return errors.New("skip empty store") + } + if startVersion > storeStartVersion { + storeStartVersion = startVersion + } + iavlTree := iavl.NewImmutableTree(dbm.NewPrefixDB(db, prefix), DefaultCacheSize, true) + if err := iavlTree.TraverseStateChanges(storeStartVersion, endVersion, func(version int64, changeSet *iavl.ChangeSet) error { + return WriteChangeSet(version, *changeSet) + }); err != nil { + panic(err) + } + } + return nil +} + +func OpenDB(dir string) (dbm.DB, error) { + switch { + case strings.HasSuffix(dir, ".db"): + dir = dir[:len(dir)-3] + case strings.HasSuffix(dir, ".db/"): + dir = dir[:len(dir)-4] + default: + return nil, fmt.Errorf("database directory must end with .db") + } + // TODO: doesn't work on windows! + cut := strings.LastIndex(dir, "/") + if cut == -1 { + return nil, fmt.Errorf("cannot cut paths on %s", dir) + } + name := dir[cut+1:] + db, err := dbm.NewGoLevelDB(name, dir[:cut]) + if err != nil { + return nil, err + } + return db, nil +} + +func getNextVersion(db dbm.DB, version int64) (int64, error) { + rootKeyFormat := iavl.NewKeyFormat('r', 8) + itr, err := db.Iterator( + rootKeyFormat.Key(version+1), + rootKeyFormat.Key(math.MaxInt64), + ) + if err != nil { + return 0, err + } + defer itr.Close() + + var nversion int64 + for ; itr.Valid(); itr.Next() { + rootKeyFormat.Scan(itr.Key(), &nversion) + return nversion, nil + } + + if err := itr.Error(); err != nil { + return 0, err + } + + return 0, nil +} + +func WriteChangeSet(version int64, cs iavl.ChangeSet) error { + for _, pair := range cs.Pairs { + fmt.Printf("Version: %d, delete: %t, key: %X, value: %X\n", version, pair.Delete, pair.Key, pair.Key) + } + return nil +} diff --git a/diff.go b/diff.go new file mode 100644 index 0000000..4697550 --- /dev/null +++ b/diff.go @@ -0,0 +1,156 @@ +package iavl + +import ( + "bytes" + + "github.com/cosmos/iavl/proto" +) + +type ( + KVPair = proto.KVPair + ChangeSet = proto.ChangeSet +) + +// KVPairReceiver is callback parameter of method `extractStateChanges` to receive stream of `KVPair`s. +type KVPairReceiver func(pair *KVPair) error + +// extractStateChanges extracts the state changes by between two versions of the tree. +// it first traverse the `root` tree until the first `sharedNode` and record the new leave nodes, +// then traverse the `prevRoot` tree until the current `sharedNode` to find out orphaned leave nodes, +// compare orphaned leave nodes and new leave nodes to produce stream of `KVPair`s and passed to callback. +// +// The algorithm don't run in constant memory strictly, but it tried the best the only +// keep minimal intermediate states in memory. +func (ndb *nodeDB) extractStateChanges(prevVersion int64, prevRoot []byte, root []byte, receiver KVPairReceiver) error { + curIter, err := NewNodeIterator(root, ndb) + if err != nil { + return err + } + + prevIter, err := NewNodeIterator(prevRoot, ndb) + if err != nil { + return err + } + + var ( + // current shared node between two versions + sharedNode *Node + // record the newly added leaf nodes during the traversal to the `sharedNode`, + // will be compared with found orphaned nodes to produce change set stream. + newLeaves []*Node + ) + + // consumeNewLeaves concumes remaining `newLeaves` nodes and produce insertion `KVPair`. + consumeNewLeaves := func() error { + for _, node := range newLeaves { + if err := receiver(&KVPair{ + Key: node.key, + Value: node.value, + }); err != nil { + return err + } + } + + newLeaves = newLeaves[:0] + return nil + } + + // advanceSharedNode forward `curIter` until the next `sharedNode`, + // `sharedNode` will be `nil` if the new version is exhausted. + // it also records the new leaf nodes during the traversal. + advanceSharedNode := func() error { + if err := consumeNewLeaves(); err != nil { + return err + } + + sharedNode = nil + for curIter.Valid() { + node := curIter.GetNode() + shared := node.version <= prevVersion + curIter.Next(shared) + if shared { + sharedNode = node + break + } else if node.isLeaf() { + newLeaves = append(newLeaves, node) + } + } + + return nil + } + if err := advanceSharedNode(); err != nil { + return err + } + + // addOrphanedLeave receives a new orphaned leave node found in previous version, + // compare with the current newLeaves, to produce `iavl.KVPair` stream. + addOrphanedLeave := func(orphaned *Node) error { + for len(newLeaves) > 0 { + new := newLeaves[0] + switch bytes.Compare(orphaned.key, new.key) { + case 1: + // consume a new node as insertion and continue + newLeaves = newLeaves[1:] + if err := receiver(&KVPair{ + Key: new.key, + Value: new.value, + }); err != nil { + return err + } + continue + + case -1: + // removal, don't consume new nodes + return receiver(&KVPair{ + Delete: true, + Key: orphaned.key, + }) + + case 0: + // update, consume the new node and stop + newLeaves = newLeaves[1:] + return receiver(&KVPair{ + Key: new.key, + Value: new.value, + }) + } + } + + // removal + return receiver(&KVPair{ + Delete: true, + Key: orphaned.key, + }) + } + + // Traverse `prevIter` to find orphaned nodes in the previous version, + // and compare them with newLeaves to generate `KVPair` stream. + for prevIter.Valid() { + node := prevIter.GetNode() + shared := sharedNode != nil && (node == sharedNode || bytes.Equal(node.hash, sharedNode.hash)) + // skip sub-tree of shared nodes + prevIter.Next(shared) + if shared { + if err := advanceSharedNode(); err != nil { + return err + } + } else if node.isLeaf() { + if err := addOrphanedLeave(node); err != nil { + return err + } + } + } + + if err := consumeNewLeaves(); err != nil { + return err + } + + if err := curIter.Error(); err != nil { + return err + } + if err := prevIter.Error(); err != nil { + return err + } + + return nil +} diff --git a/go.mod b/go.mod index 9068f6e..fc46d1e 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,10 @@ go 1.18 require ( github.com/confio/ics23/go v0.7.0 - github.com/cosmos/cosmos-db v1.0.0-rc.1 github.com/gogo/protobuf v1.3.2 github.com/golang/mock v1.6.0 github.com/pkg/errors v0.9.1 + github.com/spf13/cobra v1.5.0 github.com/stretchr/testify v1.8.1 github.com/tendermint/tendermint v0.34.20 github.com/tendermint/tm-db v0.6.6 @@ -16,14 +16,7 @@ require ( require ( github.com/DataDog/zstd v1.4.5 // indirect - github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect github.com/cespare/xxhash v1.1.0 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/cockroachdb/errors v1.8.1 // indirect - github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect - github.com/cockroachdb/pebble v0.0.0-20220817183557-09c6e030a677 // indirect - github.com/cockroachdb/redact v1.0.8 // indirect - github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgraph-io/badger/v2 v2.2007.2 // indirect github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de // indirect @@ -32,18 +25,13 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.1.2 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jmhodges/levigo v1.0.0 // indirect - github.com/klauspost/compress v1.15.9 // indirect - github.com/kr/pretty v0.3.0 // indirect - github.com/kr/text v0.2.0 // indirect - github.com/linxGnu/grocksdb v1.7.14 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/rogpeppe/go-internal v1.8.1 // indirect - github.com/spf13/cast v1.5.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca // indirect github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect go.etcd.io/bbolt v1.3.6 // indirect - golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect golang.org/x/net v0.0.0-20220812174116-3211cb980234 // indirect golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2 // indirect google.golang.org/protobuf v1.28.0 // indirect diff --git a/go.sum b/go.sum index ce146be..eeafcd4 100644 --- a/go.sum +++ b/go.sum @@ -61,7 +61,6 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= contrib.go.opencensus.io/exporter/stackdriver v0.13.4/go.mod h1:aXENhDJ1Y4lIg4EUaVTwzvYETVNZk10Pu26tevFKLUc= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/Antonboom/errname v0.1.6/go.mod h1:7lz79JAnuoMNDAWE9MeeIr1/c/VpSUWatBv2FH9NYpI= github.com/Antonboom/nilnil v0.1.1/go.mod h1:L1jBqoWM7AOeTD+tSquifKSesRHs4ZdaxvZR+xdJEaI= github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw= @@ -74,8 +73,6 @@ github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbi github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d/go.mod h1:URdX5+vg25ts3aCh8H5IFZybJYKWhJHYMTnf+ULtoC4= -github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= -github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= @@ -84,10 +81,7 @@ github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs= github.com/GaijinEntertainment/go-exhaustruct/v2 v2.1.0/go.mod h1:LGOGuvEgCfCQsy3JF2tRmpGDpzA53iZfyGEWSPwQ6/4= github.com/HdrHistogram/hdrhistogram-go v1.1.0/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= -github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= -github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= -github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= @@ -99,7 +93,6 @@ github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8 github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OpenPeeDeeP/depguard v1.1.0/go.mod h1:JtAMzWkmFEzDPyAd+W0NHl1lvpQKTvT9jnRVsohBKpc= -github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= @@ -107,7 +100,6 @@ github.com/Workiva/go-datastructures v1.0.53/go.mod h1:1yZL+zfsztete+ePzZz/Zb1/t github.com/adlio/schema v1.3.3/go.mod h1:1EsRssiv9/Ce2CMzq5DoL7RiMshhuigQxrR4DMV9fHg= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= -github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -136,7 +128,6 @@ github.com/aws/aws-sdk-go v1.40.45/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm github.com/aws/aws-sdk-go-v2 v1.9.1/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.8.1/go.mod h1:CM+19rL1+4dFWnOQKwDc7H1KwXTz+h61oUSHyhV0b3o= github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= -github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -172,7 +163,6 @@ github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charithe/durationcheck v0.0.9/go.mod h1:SSbRIBVfMjCi/kEB6K65XEA83D6prSM8ap1UCpNKtgg= github.com/chavacava/garif v0.0.0-20220316182200-5cad0b5181d4/go.mod h1:W8EnPSQ8Nv4fUjc/v1/8tHFqhuOJXnRub0dTfuAQktU= @@ -196,19 +186,6 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= -github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= -github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y= -github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/cockroachdb/pebble v0.0.0-20220817183557-09c6e030a677 h1:qbb/AE938DFhOajUYh9+OXELpSF9KZw2ZivtmW6eX1Q= -github.com/cockroachdb/pebble v0.0.0-20220817183557-09c6e030a677/go.mod h1:890yq1fUb9b6dGNwssgeUO5vQV9qfXnCPxAJhBQfXw0= -github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw= -github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= -github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= -github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= -github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/confio/ics23/go v0.7.0 h1:00d2kukk7sPoHWL4zZBZwzxnpA2pec1NPdwbSokJ5w8= github.com/confio/ics23/go v0.7.0/go.mod h1:E45NqnlpxGnpfTWL/xauN7MRwEE28T4Dd4uraToOaKg= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= @@ -223,8 +200,6 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cosmos/cosmos-db v1.0.0-rc.1 h1:SjnT8B6WKMW9WEIX32qMhnEEKcI7ZP0+G1Sa9HD3nmY= -github.com/cosmos/cosmos-db v1.0.0-rc.1/go.mod h1:Dnmk3flSf5lkwCqvvjNpoxjpXzhxnCAFzKHlbaForso= github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -244,7 +219,6 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/denis-tingaikin/go-header v0.4.3/go.mod h1:0wOCWuN71D5qIgE2nz9KrKmuYBAC2Mra5RassOIQ2/c= github.com/denisenkom/go-mssqldb v0.12.0/go.mod h1:iiK0YP1ZeepvmBQk/QpLEhhTNJgfzrpArPY/aFvc9yU= -github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger/v2 v2.2007.2 h1:EjjK0KqwaFMlPin1ajhP943VPENHJdEz1KLIegjaI3k= github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA= @@ -266,7 +240,6 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -281,7 +254,6 @@ github.com/envoyproxy/protoc-gen-validate v0.0.14/go.mod h1:iSmxcyjqTsJpI2R4NaDN github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= github.com/esimonov/ifshort v1.0.4/go.mod h1:Pe8zjlRrJ80+q2CxHLfEOfTwxCZ4O+MuhcHcfgNWTk0= -github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/ettle/strcase v0.1.1/go.mod h1:hzDLsPC7/lwKyBOywSHEP89nt2pDgdy+No1NBA9o9VY= github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ= github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= @@ -289,23 +261,19 @@ github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojt github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQDg5gKsWoLBOB0n+ZW8s599zru8FJ2/Y= github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= -github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/firefart/nonamedreturns v1.0.1/go.mod h1:D3dpIBojGGNh5UfElmwPu73SwDCm+VKhHYqwlNOk2uQ= -github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/franela/goblin v0.0.0-20210519012713-85d372ac71e2/go.mod h1:VzmDKDJVZI3aJmnRI9VjAn9nJ8qPPsN1fqzr9dqInIo= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= -github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -314,15 +282,8 @@ github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwV github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fullstorydev/grpcurl v1.6.0/go.mod h1:ZQ+ayqbKMJNhzLmbpCiurTVlaK2M/3nqZCxaQ2Ze/sM= github.com/fzipp/gocyclo v0.5.1/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= -github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= -github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= -github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= -github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-critic/go-critic v0.6.3/go.mod h1:c6b3ZP1MQ7o6lPR7Rv3lEf7pYQUmAcx8ABHgdZCQt/k= -github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= -github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -336,7 +297,6 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-redis/redis v6.15.8+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= @@ -356,22 +316,16 @@ github.com/go-toolsmith/typep v1.0.2/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2 github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= -github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/sqlexp v0.0.0-20170517235910-f1bb20e5a188/go.mod h1:vXjM/+wXQnTPR4KqTKDgJukSZ6amVRtWMPEjE6sQoK8= @@ -426,7 +380,6 @@ github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZ github.com/golangci/misspell v0.3.5/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA= github.com/golangci/revgrep v0.0.0-20210930125155-c22e5001d4f2/go.mod h1:LK+zW4MpyytAWQRz0M4xnzEk50lSvqDQKfx304apFkY= github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= -github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= @@ -448,7 +401,6 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -497,7 +449,6 @@ github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= @@ -547,10 +498,8 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.4.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -568,7 +517,6 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= github.com/hudl/fargo v1.4.0/go.mod h1:9Ai6uvFy5fQNq6VPKtg+Ceq1+eTY4nKUlR2JElEOcDo= -github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -576,13 +524,9 @@ github.com/imdario/mergo v0.3.4/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= -github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20200827194710-b269163b24ab/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= -github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= -github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= -github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= -github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/jdxcode/netrc v0.0.0-20210204082910-926c7f70242a/go.mod h1:Zi/ZFkEqFHTm7qkjyNJjaWH4LQA9LQhGJyF0lTYGpxw= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -616,35 +560,22 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= -github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= -github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/julz/importas v0.1.0/go.mod h1:oSFU2R4XK/P7kNBrnL/FEQlDGN1/6WoxXEjSSXO0DV0= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= -github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk= -github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U= -github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw= -github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/errcheck v1.6.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -654,7 +585,6 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -664,8 +594,6 @@ github.com/kulti/thelper v0.6.2/go.mod h1:DsqKShOvP40epevkFrvIwkCMNYxMeTNjdWL4dq github.com/kunwardeep/paralleltest v1.0.3/go.mod h1:vLydzomDFpk7yu5UX02RmP0H8QfRPOV/oFhWN85Mjb4= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/kyoh86/exportloopref v0.1.8/go.mod h1:1tUcJeiioIs7VWe5gcOObrux3lb66+sBqGZrRkMwPgg= -github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= -github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/ldez/gomoddirectives v0.2.3/go.mod h1:cpgBogWITnCfRq2qGoDkKMEVSaarhdBr6g8G04uz6d0= github.com/ldez/tagliatelle v0.3.1/go.mod h1:8s6WJQwEYHbKZDsp/LjArytKOG8qaMrKQQ3mFukHs88= github.com/leonklingele/grouper v1.1.0/go.mod h1:uk3I3uDfi9B6PeUjsCKi6ndcf63Uy7snXgR4yDYQVDY= @@ -677,8 +605,6 @@ github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= -github.com/linxGnu/grocksdb v1.7.14 h1:8lMZzyWeNP5lI0BIppX05DzmQzXj/Tgu82bgWYtowLY= -github.com/linxGnu/grocksdb v1.7.14/go.mod h1:pY55D0o+r8yUYLq70QmhdudxYvoDb9F+9puf4m3/W+U= github.com/lufeee/execinquery v1.2.1/go.mod h1:EC7DrEKView09ocscGHC+apXMIaorh4xqSxS/dy8SbM= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= @@ -689,7 +615,6 @@ github.com/maratori/testpackage v1.0.1/go.mod h1:ddKdw+XG0Phzhx8BFDTKgpWP4i7MpAp github.com/matoous/godox v0.0.0-20210227103229-6504466cf951/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -697,9 +622,7 @@ github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -710,14 +633,10 @@ github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mbilski/exhaustivestruct v1.2.0/go.mod h1:OeTBVxQWoEmB2J2JCHmXWPJ0aksxSUOUy+nvtVEfzXc= -github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= -github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= github.com/mgechev/dots v0.0.0-20210922191527-e955255bf517/go.mod h1:KQ7+USdGKfpPjXk4Ga+5XxQM4Lm4e3gAogrreFAYpOg= github.com/mgechev/revive v1.2.1/go.mod h1:+Ro3wqY4vakcYNtkBWdZC7dBg1xSB6sp054wWwmeFm0= -github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= @@ -752,7 +671,6 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/moricho/tparallel v0.2.1/go.mod h1:fXEIZxG2vdfl0ZF8b42f5a78EhjjD5mX8qUplsoSU4k= -github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mozilla/scribe v0.0.0-20180711195314-fb71baf557c1/go.mod h1:FIczTrinKo8VaLxe6PWTPEXRXDIHz2QAwiaBaP5/4a8= github.com/mozilla/tls-observatory v0.0.0-20210609171429-7bc42856d2e5/go.mod h1:FUqVoUPHSEdDR0MnFM3Dh8AU0pZHLXUD127SAJGER/s= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= @@ -764,9 +682,7 @@ github.com/nakabonne/nestif v0.3.1/go.mod h1:9EtoZochLn5iUprVDmDjqGKPofoUEBL8U4N github.com/nats-io/jwt v1.2.2/go.mod h1:/xX356yQA6LuXI9xWW7mZNpxgF2mBmGecH+Fj34sP5Q= github.com/nats-io/jwt/v2 v2.0.3/go.mod h1:VRP+deawSXyhNjXmxPCHskrR6Mq50BqpEI5SEcNiGlY= github.com/nats-io/nats-server/v2 v2.5.0/go.mod h1:Kj86UtrXAL6LwYRA6H4RqzkHhK0Vcv2ZnKD5WbQ1t3g= -github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= github.com/nats-io/nats.go v1.12.1/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= -github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= github.com/nats-io/nkeys v0.2.0/go.mod h1:XdZpAbhgyyODYqjTawOnIOI7VlbKSarI9Gfy1tqEu/s= github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= @@ -788,7 +704,6 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= @@ -837,8 +752,6 @@ github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCr github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d/go.mod h1:3OzsM7FXDQlpCiw2j81fOmAwQLnZnLGXVKUzeKQXIAw= github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= -github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= -github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -903,7 +816,6 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= @@ -915,14 +827,12 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/ryancurrah/gomodguard v1.2.3/go.mod h1:rYbA/4Tg5c54mV1sv4sQTP5WOPBcoLtnBZ7/TEhXAbg= github.com/ryanrolds/sqlclosecheck v0.3.0/go.mod h1:1gREqxyTGR3lVtpngyFo3hZAgk0KCtEdgEkHwDbigdA= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= github.com/sagikazarmark/crypt v0.5.0/go.mod h1:l+nzl7KWh51rpzp2h7t4MZWyiEWdhNpOAnclKvg+mdA= github.com/sagikazarmark/crypt v0.6.0/go.mod h1:U8+INwJo3nBv1m6A/8OBXAq7Jnpspk5AxSgDyEQcea8= github.com/sanposhiho/wastedassign/v2 v2.0.6/go.mod h1:KyZ0MWTwxxBmfwn33zh3k1dmsbF2ud9pAAGfoLfjhtI= github.com/sasha-s/go-deadlock v0.2.1-0.20190427202633-1595213edefa h1:0U2s5loxrTy6/VgfVoLuVLFJcURKLH49ie0zSch7gh4= github.com/sasha-s/go-deadlock v0.2.1-0.20190427202633-1595213edefa/go.mod h1:F73l+cr82YSh10GxyRI6qZiCgK64VaZjwesgfQ1/iLM= -github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= @@ -956,17 +866,18 @@ github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= +github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= @@ -1023,20 +934,15 @@ github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoi github.com/tommy-muehle/go-mnd/v2 v2.5.0/go.mod h1:WsUAkMJMYww6l/ufffCD3m+P7LEvr8TnZn9lwVDlgzw= github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ultraware/funlen v0.0.3/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= github.com/ultraware/whitespace v0.0.5/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/uudashr/gocognit v1.0.5/go.mod h1:wgYz0mitoKOTysqxTDMOUXg+Jb5SvtihkfmugIZYpEA= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= github.com/valyala/fasthttp v1.30.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD3K/7o2Cus= -github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/quicktemplate v1.7.0/go.mod h1:sqKJnoaOF88V07vkO+9FL8fb9uZg/VPSJnLYn+LmLk8= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/vektra/mockery/v2 v2.14.0/go.mod h1:bnD1T8tExSgPD1ripLkDbr60JA9VtQeu12P3wgLZd7M= github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8/go.mod h1:dniwbG03GafCjFohMDmz6Zc6oCuiqgH6tGNyXTkHzXE= @@ -1050,7 +956,6 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yagipy/maintidx v1.0.0/go.mod h1:0qNf/I/CCZXSMhsRsrEPDZ+DkekpKLXAJfsTACwgXLk= -github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/yeya24/promlinter v0.2.0/go.mod h1:u54lkmBOZrpEbQQ6gox2zWKKLKu2SGe+2KOiextY+IA= github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= @@ -1156,9 +1061,6 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= -golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= -golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= -golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= @@ -1200,7 +1102,6 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -1210,7 +1111,6 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1292,7 +1192,6 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 h1:w8s32wxx3sY+OjLlv9qltkLU5yvJzxjjgiHWLjdIcw4= golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1315,7 +1214,6 @@ golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1385,7 +1283,6 @@ golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210909193231-528a39cd75f3/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1440,7 +1337,6 @@ golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -1452,7 +1348,6 @@ golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190321232350-e250d351ecad/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -1550,7 +1445,6 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= -gonum.org/v1/gonum v0.8.2 h1:CCXrcPKiGGotvnN6jfUsKk4rRqm7q09/YbKb5xCEvtM= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= @@ -1604,7 +1498,6 @@ google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20170818010345-ee236bd376b0/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181107211654-5fc9ac540362/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1692,7 +1585,6 @@ google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= -google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1757,12 +1649,9 @@ gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= -gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= -gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= diff --git a/immutable_tree.go b/immutable_tree.go index 3f6fe52..3bd958c 100644 --- a/immutable_tree.go +++ b/immutable_tree.go @@ -332,3 +332,9 @@ func (t *ImmutableTree) nodeSize() int { }) return size } + +// TraverseStateChanges iterate the range of versions, compare each version to it's predecessor to extract the state changes of it. +// endVersion is exclusive. +func (t *ImmutableTree) TraverseStateChanges(startVersion, endVersion int64, fn func(version int64, changeSet *ChangeSet) error) error { + return t.ndb.traverseStateChanges(startVersion, endVersion, fn) +} diff --git a/iterator.go b/iterator.go index 520d8f2..c3cb9cd 100644 --- a/iterator.go +++ b/iterator.go @@ -287,3 +287,77 @@ func (iter *Iterator) Error() error { func (iter *Iterator) IsFast() bool { return false } + +// NodeIterator is an iterator for nodeDB to traverse a tree in depth-first, preorder manner. +type NodeIterator struct { + nodesToVisit []*Node + ndb *nodeDB + err error +} + +// NewNodeIterator returns a new NodeIterator to traverse the tree of the root node. +func NewNodeIterator(rootKey []byte, ndb *nodeDB) (*NodeIterator, error) { + if len(rootKey) == 0 { + return &NodeIterator{ + nodesToVisit: []*Node{}, + ndb: ndb, + }, nil + } + + node, err := ndb.GetNode(rootKey) + if err != nil { + return nil, err + } + + return &NodeIterator{ + nodesToVisit: []*Node{node}, + ndb: ndb, + }, nil +} + +// GetNode returns the current visiting node. +func (iter *NodeIterator) GetNode() *Node { + return iter.nodesToVisit[len(iter.nodesToVisit)-1] +} + +// Valid checks if the validator is valid. +func (iter *NodeIterator) Valid() bool { + return iter.err == nil && len(iter.nodesToVisit) > 0 +} + +// Error returns an error if any errors. +func (iter *NodeIterator) Error() error { + return iter.err +} + +// Next moves forward the traversal. +// if isSkipped is true, the subtree under the current node is skipped. +func (iter *NodeIterator) Next(isSkipped bool) { + if !iter.Valid() { + return + } + node := iter.GetNode() + iter.nodesToVisit = iter.nodesToVisit[:len(iter.nodesToVisit)-1] + + if isSkipped { + return + } + + if node.isLeaf() { + return + } + + rightNode, err := iter.ndb.GetNode(node.rightHash) + if err != nil { + iter.err = err + return + } + iter.nodesToVisit = append(iter.nodesToVisit, rightNode) + + leftNode, err := iter.ndb.GetNode(node.leftHash) + if err != nil { + iter.err = err + return + } + iter.nodesToVisit = append(iter.nodesToVisit, leftNode) +} diff --git a/nodedb.go b/nodedb.go index 18bd44a..f4eae46 100644 --- a/nodedb.go +++ b/nodedb.go @@ -1061,6 +1061,41 @@ func (ndb *nodeDB) traverseNodes(fn func(hash []byte, node *Node) error) error { return nil } +// traverseStateChanges iterate the range of versions, compare each version to it's predecessor to extract the state changes of it. +// endVersion is exclusive, set to `math.MaxInt64` to cover the latest version. +func (ndb *nodeDB) traverseStateChanges(startVersion, endVersion int64, fn func(version int64, changeSet *ChangeSet) error) error { + predecessor, err := ndb.getPreviousVersion(startVersion) + if err != nil { + return err + } + prevRoot, err := ndb.getRoot(predecessor) + if err != nil { + return err + } + return ndb.traverseRange(rootKeyFormat.Key(startVersion), rootKeyFormat.Key(endVersion), func(k, hash []byte) error { + var version int64 + rootKeyFormat.Scan(k, &version) + + var changeSet ChangeSet + receiveKVPair := func(pair *KVPair) error { + changeSet.Pairs = append(changeSet.Pairs, pair) + return nil + } + + if err := ndb.extractStateChanges(predecessor, prevRoot, hash, receiveKVPair); err != nil { + return err + } + + if err := fn(version, &changeSet); err != nil { + return err + } + + predecessor = version + prevRoot = hash + return nil + }) +} + func (ndb *nodeDB) String() (string, error) { buf := bufPool.Get().(*bytes.Buffer) defer bufPool.Put(buf) diff --git a/proto/changeset.pb.go b/proto/changeset.pb.go new file mode 100644 index 0000000..359b34e --- /dev/null +++ b/proto/changeset.pb.go @@ -0,0 +1,599 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: iavl/changeset.proto + +package proto + +import ( + fmt "fmt" + proto "github.com/gogo/protobuf/proto" + io "io" + math "math" + math_bits "math/bits" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +type KVPair struct { + Delete bool `protobuf:"varint,1,opt,name=delete,proto3" json:"delete,omitempty"` + Key []byte `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` + Value []byte `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"` +} + +func (m *KVPair) Reset() { *m = KVPair{} } +func (m *KVPair) String() string { return proto.CompactTextString(m) } +func (*KVPair) ProtoMessage() {} +func (*KVPair) Descriptor() ([]byte, []int) { + return fileDescriptor_21609c3776972f61, []int{0} +} +func (m *KVPair) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *KVPair) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_KVPair.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *KVPair) XXX_Merge(src proto.Message) { + xxx_messageInfo_KVPair.Merge(m, src) +} +func (m *KVPair) XXX_Size() int { + return m.Size() +} +func (m *KVPair) XXX_DiscardUnknown() { + xxx_messageInfo_KVPair.DiscardUnknown(m) +} + +var xxx_messageInfo_KVPair proto.InternalMessageInfo + +func (m *KVPair) GetDelete() bool { + if m != nil { + return m.Delete + } + return false +} + +func (m *KVPair) GetKey() []byte { + if m != nil { + return m.Key + } + return nil +} + +func (m *KVPair) GetValue() []byte { + if m != nil { + return m.Value + } + return nil +} + +type ChangeSet struct { + Pairs []*KVPair `protobuf:"bytes,1,rep,name=pairs,proto3" json:"pairs,omitempty"` +} + +func (m *ChangeSet) Reset() { *m = ChangeSet{} } +func (m *ChangeSet) String() string { return proto.CompactTextString(m) } +func (*ChangeSet) ProtoMessage() {} +func (*ChangeSet) Descriptor() ([]byte, []int) { + return fileDescriptor_21609c3776972f61, []int{1} +} +func (m *ChangeSet) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ChangeSet) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ChangeSet.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ChangeSet) XXX_Merge(src proto.Message) { + xxx_messageInfo_ChangeSet.Merge(m, src) +} +func (m *ChangeSet) XXX_Size() int { + return m.Size() +} +func (m *ChangeSet) XXX_DiscardUnknown() { + xxx_messageInfo_ChangeSet.DiscardUnknown(m) +} + +var xxx_messageInfo_ChangeSet proto.InternalMessageInfo + +func (m *ChangeSet) GetPairs() []*KVPair { + if m != nil { + return m.Pairs + } + return nil +} + +func init() { + proto.RegisterType((*KVPair)(nil), "iavl.KVPair") + proto.RegisterType((*ChangeSet)(nil), "iavl.ChangeSet") +} + +func init() { proto.RegisterFile("iavl/changeset.proto", fileDescriptor_21609c3776972f61) } + +var fileDescriptor_21609c3776972f61 = []byte{ + // 182 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0xc9, 0x4c, 0x2c, 0xcb, + 0xd1, 0x4f, 0xce, 0x48, 0xcc, 0x4b, 0x4f, 0x2d, 0x4e, 0x2d, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, + 0x17, 0x62, 0x01, 0x89, 0x2a, 0x79, 0x70, 0xb1, 0x79, 0x87, 0x05, 0x24, 0x66, 0x16, 0x09, 0x89, + 0x71, 0xb1, 0xa5, 0xa4, 0xe6, 0xa4, 0x96, 0xa4, 0x4a, 0x30, 0x2a, 0x30, 0x6a, 0x70, 0x04, 0x41, + 0x79, 0x42, 0x02, 0x5c, 0xcc, 0xd9, 0xa9, 0x95, 0x12, 0x4c, 0x0a, 0x8c, 0x1a, 0x3c, 0x41, 0x20, + 0xa6, 0x90, 0x08, 0x17, 0x6b, 0x59, 0x62, 0x4e, 0x69, 0xaa, 0x04, 0x33, 0x58, 0x0c, 0xc2, 0x51, + 0xd2, 0xe7, 0xe2, 0x74, 0x06, 0x5b, 0x11, 0x9c, 0x5a, 0x22, 0xa4, 0xc4, 0xc5, 0x5a, 0x90, 0x98, + 0x59, 0x54, 0x2c, 0xc1, 0xa8, 0xc0, 0xac, 0xc1, 0x6d, 0xc4, 0xa3, 0x07, 0xb2, 0x4c, 0x0f, 0x62, + 0x53, 0x10, 0x44, 0xca, 0x49, 0xfe, 0xc4, 0x23, 0x39, 0xc6, 0x0b, 0x8f, 0xe4, 0x18, 0x1f, 0x3c, + 0x92, 0x63, 0x9c, 0xf0, 0x58, 0x8e, 0xe1, 0xc2, 0x63, 0x39, 0x86, 0x1b, 0x8f, 0xe5, 0x18, 0xa2, + 0x58, 0xc1, 0x2e, 0x4c, 0x62, 0x03, 0x53, 0xc6, 0x80, 0x00, 0x00, 0x00, 0xff, 0xff, 0x26, 0xfc, + 0xaa, 0xbd, 0xc0, 0x00, 0x00, 0x00, +} + +func (m *KVPair) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *KVPair) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *KVPair) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Value) > 0 { + i -= len(m.Value) + copy(dAtA[i:], m.Value) + i = encodeVarintChangeset(dAtA, i, uint64(len(m.Value))) + i-- + dAtA[i] = 0x1a + } + if len(m.Key) > 0 { + i -= len(m.Key) + copy(dAtA[i:], m.Key) + i = encodeVarintChangeset(dAtA, i, uint64(len(m.Key))) + i-- + dAtA[i] = 0x12 + } + if m.Delete { + i-- + if m.Delete { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *ChangeSet) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ChangeSet) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ChangeSet) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Pairs) > 0 { + for iNdEx := len(m.Pairs) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Pairs[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintChangeset(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func encodeVarintChangeset(dAtA []byte, offset int, v uint64) int { + offset -= sovChangeset(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *KVPair) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Delete { + n += 2 + } + l = len(m.Key) + if l > 0 { + n += 1 + l + sovChangeset(uint64(l)) + } + l = len(m.Value) + if l > 0 { + n += 1 + l + sovChangeset(uint64(l)) + } + return n +} + +func (m *ChangeSet) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Pairs) > 0 { + for _, e := range m.Pairs { + l = e.Size() + n += 1 + l + sovChangeset(uint64(l)) + } + } + return n +} + +func sovChangeset(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozChangeset(x uint64) (n int) { + return sovChangeset(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *KVPair) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowChangeset + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: KVPair: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: KVPair: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Delete", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowChangeset + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Delete = bool(v != 0) + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowChangeset + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthChangeset + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthChangeset + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Key = append(m.Key[:0], dAtA[iNdEx:postIndex]...) + if m.Key == nil { + m.Key = []byte{} + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Value", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowChangeset + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthChangeset + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthChangeset + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Value = append(m.Value[:0], dAtA[iNdEx:postIndex]...) + if m.Value == nil { + m.Value = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipChangeset(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthChangeset + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthChangeset + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ChangeSet) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowChangeset + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ChangeSet: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ChangeSet: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Pairs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowChangeset + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthChangeset + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthChangeset + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Pairs = append(m.Pairs, &KVPair{}) + if err := m.Pairs[len(m.Pairs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipChangeset(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthChangeset + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthChangeset + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipChangeset(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + depth := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowChangeset + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowChangeset + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + case 1: + iNdEx += 8 + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowChangeset + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthChangeset + } + iNdEx += length + case 3: + depth++ + case 4: + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupChangeset + } + depth-- + case 5: + iNdEx += 4 + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + if iNdEx < 0 { + return 0, ErrInvalidLengthChangeset + } + if depth == 0 { + return iNdEx, nil + } + } + return 0, io.ErrUnexpectedEOF +} + +var ( + ErrInvalidLengthChangeset = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowChangeset = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupChangeset = fmt.Errorf("proto: unexpected end of group") +) diff --git a/proto/iavl/changeset.proto b/proto/iavl/changeset.proto new file mode 100644 index 0000000..84f6287 --- /dev/null +++ b/proto/iavl/changeset.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; +package iavl; + +option go_package = "proto"; + +message KVPair { + bool delete = 1; + bytes key = 2; + bytes value = 3; +} + +message ChangeSet { + repeated KVPair pairs = 1; +} \ No newline at end of file From 275fdcfe05c3d02c41cde1ec5863b666f6a4593c Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 25 Sep 2023 08:51:45 -0700 Subject: [PATCH 02/72] Fix bug --- cmd/changeset/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/changeset/main.go b/cmd/changeset/main.go index 45a5df7..ad879cc 100644 --- a/cmd/changeset/main.go +++ b/cmd/changeset/main.go @@ -54,7 +54,7 @@ func executeDumpChangesetCmd(cmd *cobra.Command, _ []string) error { fmt.Println("Begin dumping store", storeKey, time.Now().Format(time.RFC3339)) db, err := OpenDB(homeDir) if err != nil { - return nil + return err } if len(storeKey) != 0 { db = dbm.NewPrefixDB(db, prefix) From 97eb985c698da48d0c3bd3e6724e6706bf6fb450 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 25 Sep 2023 08:59:26 -0700 Subject: [PATCH 03/72] Fix bug --- cmd/changeset/main.go | 39 +++------------------------------------ 1 file changed, 3 insertions(+), 36 deletions(-) diff --git a/cmd/changeset/main.go b/cmd/changeset/main.go index ad879cc..a44e9e0 100644 --- a/cmd/changeset/main.go +++ b/cmd/changeset/main.go @@ -3,10 +3,8 @@ package main import ( "fmt" "github.com/cosmos/iavl" - "github.com/pkg/errors" "github.com/spf13/cobra" dbm "github.com/tendermint/tm-db" - "math" "os" "strings" "time" @@ -22,7 +20,7 @@ func main() { Use: "changeset", Short: "Dump change sets files which can be ingested into DB", } - rootCmd.PersistentFlags().String("home-dir", "/root/.sei", "Home directory") + rootCmd.PersistentFlags().String("home-dir", "/root/.sei/data/application.db", "Home directory") rootCmd.PersistentFlags().String("start-version", "0", "start version") rootCmd.PersistentFlags().String("end-version", "0", "start version") rootCmd.PersistentFlags().StringP("store", "s", DefaultStore, "store key name") @@ -69,16 +67,9 @@ func executeDumpChangesetCmd(cmd *cobra.Command, _ []string) error { endVersion = latestVersion + 1 } - storeStartVersion, err := getNextVersion(dbm.NewPrefixDB(db, prefix), 0) - if storeStartVersion <= 0 { - // store not exists - return errors.New("skip empty store") - } - if startVersion > storeStartVersion { - storeStartVersion = startVersion - } iavlTree := iavl.NewImmutableTree(dbm.NewPrefixDB(db, prefix), DefaultCacheSize, true) - if err := iavlTree.TraverseStateChanges(storeStartVersion, endVersion, func(version int64, changeSet *iavl.ChangeSet) error { + fmt.Printf("Going to traverse changeset from version %d to %d\n", startVersion, endVersion) + if err := iavlTree.TraverseStateChanges(startVersion, endVersion, func(version int64, changeSet *iavl.ChangeSet) error { return WriteChangeSet(version, *changeSet) }); err != nil { panic(err) @@ -109,30 +100,6 @@ func OpenDB(dir string) (dbm.DB, error) { return db, nil } -func getNextVersion(db dbm.DB, version int64) (int64, error) { - rootKeyFormat := iavl.NewKeyFormat('r', 8) - itr, err := db.Iterator( - rootKeyFormat.Key(version+1), - rootKeyFormat.Key(math.MaxInt64), - ) - if err != nil { - return 0, err - } - defer itr.Close() - - var nversion int64 - for ; itr.Valid(); itr.Next() { - rootKeyFormat.Scan(itr.Key(), &nversion) - return nversion, nil - } - - if err := itr.Error(); err != nil { - return 0, err - } - - return 0, nil -} - func WriteChangeSet(version int64, cs iavl.ChangeSet) error { for _, pair := range cs.Pairs { fmt.Printf("Version: %d, delete: %t, key: %X, value: %X\n", version, pair.Delete, pair.Key, pair.Key) From f2a4c9b80c777f8ac339f2417b64d9ea7950e0f7 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 25 Sep 2023 09:00:59 -0700 Subject: [PATCH 04/72] Fix bug --- cmd/changeset/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/changeset/main.go b/cmd/changeset/main.go index a44e9e0..5bf31fe 100644 --- a/cmd/changeset/main.go +++ b/cmd/changeset/main.go @@ -21,8 +21,8 @@ func main() { Short: "Dump change sets files which can be ingested into DB", } rootCmd.PersistentFlags().String("home-dir", "/root/.sei/data/application.db", "Home directory") - rootCmd.PersistentFlags().String("start-version", "0", "start version") - rootCmd.PersistentFlags().String("end-version", "0", "start version") + rootCmd.PersistentFlags().Int64("start-version", 0, "start version") + rootCmd.PersistentFlags().Int64("end-version", 0, "start version") rootCmd.PersistentFlags().StringP("store", "s", DefaultStore, "store key name") rootCmd.AddCommand(DumpChangeSetCmd()) if err := rootCmd.Execute(); err != nil { From c0f5567b635a4ab19d45a3cb1a4d819222e50dc7 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 25 Sep 2023 09:10:57 -0700 Subject: [PATCH 05/72] Add logs --- cmd/changeset/main.go | 2 +- diff.go | 2 +- nodedb.go | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/changeset/main.go b/cmd/changeset/main.go index 5bf31fe..7cfac83 100644 --- a/cmd/changeset/main.go +++ b/cmd/changeset/main.go @@ -102,7 +102,7 @@ func OpenDB(dir string) (dbm.DB, error) { func WriteChangeSet(version int64, cs iavl.ChangeSet) error { for _, pair := range cs.Pairs { - fmt.Printf("Version: %d, delete: %t, key: %X, value: %X\n", version, pair.Delete, pair.Key, pair.Key) + fmt.Printf("Version: %d, delete: %t, key: %X, value: %X\n", version, pair.Delete, pair.Key, pair.Value) } return nil } diff --git a/diff.go b/diff.go index 4697550..327de6e 100644 --- a/diff.go +++ b/diff.go @@ -40,7 +40,7 @@ func (ndb *nodeDB) extractStateChanges(prevVersion int64, prevRoot []byte, root newLeaves []*Node ) - // consumeNewLeaves concumes remaining `newLeaves` nodes and produce insertion `KVPair`. + // consumeNewLeaves consumes remaining `newLeaves` nodes and produce insertion `KVPair`. consumeNewLeaves := func() error { for _, node := range newLeaves { if err := receiver(&KVPair{ diff --git a/nodedb.go b/nodedb.go index f4eae46..793023e 100644 --- a/nodedb.go +++ b/nodedb.go @@ -1065,10 +1065,12 @@ func (ndb *nodeDB) traverseNodes(fn func(hash []byte, node *Node) error) error { // endVersion is exclusive, set to `math.MaxInt64` to cover the latest version. func (ndb *nodeDB) traverseStateChanges(startVersion, endVersion int64, fn func(version int64, changeSet *ChangeSet) error) error { predecessor, err := ndb.getPreviousVersion(startVersion) + fmt.Printf("predecessor:%v\n", predecessor) if err != nil { return err } prevRoot, err := ndb.getRoot(predecessor) + fmt.Printf("prevRoot:%v\n", prevRoot) if err != nil { return err } @@ -1082,6 +1084,7 @@ func (ndb *nodeDB) traverseStateChanges(startVersion, endVersion int64, fn func( return nil } + fmt.Printf("extractStateChanges:%v\n", version) if err := ndb.extractStateChanges(predecessor, prevRoot, hash, receiveKVPair); err != nil { return err } From 962ae0d69d826f9ae6c60ab9bc7af3433978fc57 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 25 Sep 2023 09:17:52 -0700 Subject: [PATCH 06/72] Add logs --- cmd/changeset/main.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmd/changeset/main.go b/cmd/changeset/main.go index 7cfac83..ff4383b 100644 --- a/cmd/changeset/main.go +++ b/cmd/changeset/main.go @@ -64,10 +64,17 @@ func executeDumpChangesetCmd(cmd *cobra.Command, _ []string) error { // Make sure we have a correct end version if endVersion <= 0 { latestVersion, _ := tree.LazyLoadVersion(0) + fmt.Printf("Got tree version: %d\n", latestVersion) endVersion = latestVersion + 1 } iavlTree := iavl.NewImmutableTree(dbm.NewPrefixDB(db, prefix), DefaultCacheSize, true) + treeHash, err := iavlTree.Hash() + if err != nil { + fmt.Fprintf(os.Stderr, "Error hashing tree: %s\n", err) + os.Exit(1) + } + fmt.Printf("Tree hash is %X, tree size is %d\n", treeHash, tree.ImmutableTree().Size()) fmt.Printf("Going to traverse changeset from version %d to %d\n", startVersion, endVersion) if err := iavlTree.TraverseStateChanges(startVersion, endVersion, func(version int64, changeSet *iavl.ChangeSet) error { return WriteChangeSet(version, *changeSet) From ad553d11427cabf8e0843afe753762ec39e4f222 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 25 Sep 2023 09:19:21 -0700 Subject: [PATCH 07/72] Add logs --- cmd/changeset/main.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cmd/changeset/main.go b/cmd/changeset/main.go index ff4383b..8396817 100644 --- a/cmd/changeset/main.go +++ b/cmd/changeset/main.go @@ -54,10 +54,7 @@ func executeDumpChangesetCmd(cmd *cobra.Command, _ []string) error { if err != nil { return err } - if len(storeKey) != 0 { - db = dbm.NewPrefixDB(db, prefix) - } - tree, err := iavl.NewMutableTree(db, DefaultCacheSize, true) + tree, err := iavl.NewMutableTree(dbm.NewPrefixDB(db, prefix), DefaultCacheSize, true) if err != nil { return err } From e305f9f6dfd8bb0cb6f6b40b3f3def9985ff13d1 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 25 Sep 2023 09:28:12 -0700 Subject: [PATCH 08/72] Add logs --- cmd/changeset/main.go | 1 + nodedb.go | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/cmd/changeset/main.go b/cmd/changeset/main.go index 8396817..9d53333 100644 --- a/cmd/changeset/main.go +++ b/cmd/changeset/main.go @@ -105,6 +105,7 @@ func OpenDB(dir string) (dbm.DB, error) { } func WriteChangeSet(version int64, cs iavl.ChangeSet) error { + fmt.Printf("Process changeset for version:%d, %d pairs\n", version, len(cs.Pairs)) for _, pair := range cs.Pairs { fmt.Printf("Version: %d, delete: %t, key: %X, value: %X\n", version, pair.Delete, pair.Key, pair.Value) } diff --git a/nodedb.go b/nodedb.go index 793023e..f4eae46 100644 --- a/nodedb.go +++ b/nodedb.go @@ -1065,12 +1065,10 @@ func (ndb *nodeDB) traverseNodes(fn func(hash []byte, node *Node) error) error { // endVersion is exclusive, set to `math.MaxInt64` to cover the latest version. func (ndb *nodeDB) traverseStateChanges(startVersion, endVersion int64, fn func(version int64, changeSet *ChangeSet) error) error { predecessor, err := ndb.getPreviousVersion(startVersion) - fmt.Printf("predecessor:%v\n", predecessor) if err != nil { return err } prevRoot, err := ndb.getRoot(predecessor) - fmt.Printf("prevRoot:%v\n", prevRoot) if err != nil { return err } @@ -1084,7 +1082,6 @@ func (ndb *nodeDB) traverseStateChanges(startVersion, endVersion int64, fn func( return nil } - fmt.Printf("extractStateChanges:%v\n", version) if err := ndb.extractStateChanges(predecessor, prevRoot, hash, receiveKVPair); err != nil { return err } From 40f09ef1aa9f813797f02f51a40ce6ae6141f817 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 25 Sep 2023 10:20:39 -0700 Subject: [PATCH 09/72] Add logs --- cmd/changeset/main.go | 117 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 108 insertions(+), 9 deletions(-) diff --git a/cmd/changeset/main.go b/cmd/changeset/main.go index 9d53333..e3ab0a4 100644 --- a/cmd/changeset/main.go +++ b/cmd/changeset/main.go @@ -1,13 +1,19 @@ package main import ( + "encoding/binary" "fmt" "github.com/cosmos/iavl" "github.com/spf13/cobra" dbm "github.com/tendermint/tm-db" + "io" + "math/bits" "os" + "path/filepath" "strings" "time" + + "github.com/klauspost/compress/zstd" ) const ( @@ -20,10 +26,12 @@ func main() { Use: "changeset", Short: "Dump change sets files which can be ingested into DB", } - rootCmd.PersistentFlags().String("home-dir", "/root/.sei/data/application.db", "Home directory") + rootCmd.PersistentFlags().StringP("home-dir", "h", "/root/.sei/data/application.db", "Home directory") rootCmd.PersistentFlags().Int64("start-version", 0, "start version") rootCmd.PersistentFlags().Int64("end-version", 0, "start version") rootCmd.PersistentFlags().StringP("store", "s", DefaultStore, "store key name") + rootCmd.PersistentFlags().StringP("output-dir", "o", "output", "output directory") + rootCmd.AddCommand(DumpChangeSetCmd()) if err := rootCmd.Execute(); err != nil { fmt.Println(err) @@ -47,9 +55,14 @@ func executeDumpChangesetCmd(cmd *cobra.Command, _ []string) error { storeKey, _ := cmd.Flags().GetString("store") startVersion, _ := cmd.Flags().GetInt64("start-version") endVersion, _ := cmd.Flags().GetInt64("end-version") - prefix := []byte(storeKey) + outDir, _ := cmd.Flags().GetString("output-dir") + prefix := []byte(fmt.Sprintf("s/k:%s/", storeKey)) + if storeKey != "" { - fmt.Println("Begin dumping store", storeKey, time.Now().Format(time.RFC3339)) + fmt.Println("Begin dumping store with prefix", prefix, time.Now().Format(time.RFC3339)) + if err := os.MkdirAll(outDir, os.ModePerm); err != nil { + return err + } db, err := OpenDB(homeDir) if err != nil { return err @@ -68,13 +81,21 @@ func executeDumpChangesetCmd(cmd *cobra.Command, _ []string) error { iavlTree := iavl.NewImmutableTree(dbm.NewPrefixDB(db, prefix), DefaultCacheSize, true) treeHash, err := iavlTree.Hash() if err != nil { - fmt.Fprintf(os.Stderr, "Error hashing tree: %s\n", err) - os.Exit(1) + return err } fmt.Printf("Tree hash is %X, tree size is %d\n", treeHash, tree.ImmutableTree().Size()) fmt.Printf("Going to traverse changeset from version %d to %d\n", startVersion, endVersion) + taskFile := filepath.Join(outDir, fmt.Sprintf("changeset-%s-%d.zst", storeKey, startVersion)) + outputFile, err := createFile(taskFile) + if err != nil { + return err + } + zstdWriter, err := zstd.NewWriter(outputFile) + if err != nil { + return err + } if err := iavlTree.TraverseStateChanges(startVersion, endVersion, func(version int64, changeSet *iavl.ChangeSet) error { - return WriteChangeSet(version, *changeSet) + return WriteChangeSet(zstdWriter, version, *changeSet) }); err != nil { panic(err) } @@ -91,7 +112,6 @@ func OpenDB(dir string) (dbm.DB, error) { default: return nil, fmt.Errorf("database directory must end with .db") } - // TODO: doesn't work on windows! cut := strings.LastIndex(dir, "/") if cut == -1 { return nil, fmt.Errorf("cannot cut paths on %s", dir) @@ -104,10 +124,89 @@ func OpenDB(dir string) (dbm.DB, error) { return db, nil } -func WriteChangeSet(version int64, cs iavl.ChangeSet) error { - fmt.Printf("Process changeset for version:%d, %d pairs\n", version, len(cs.Pairs)) +func createFile(name string) (*os.File, error) { + return os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) +} + +func WriteChangeSet(writer io.Writer, version int64, cs iavl.ChangeSet) error { + if len(cs.Pairs) <= 0 { + return nil + } + + var size int + items := make([][]byte, 0, len(cs.Pairs)) for _, pair := range cs.Pairs { fmt.Printf("Version: %d, delete: %t, key: %X, value: %X\n", version, pair.Delete, pair.Key, pair.Value) + buf, err := encodeKVPair(pair) + if err != nil { + return err + } + size += len(buf) + items = append(items, buf) + } + + // Write header + var versionHeader [16]byte + binary.LittleEndian.PutUint64(versionHeader[:], uint64(version)) + binary.LittleEndian.PutUint64(versionHeader[8:], uint64(size)) + + if _, err := writer.Write(versionHeader[:]); err != nil { + return err + } + for _, item := range items { + if _, err := writer.Write(item); err != nil { + return err + } } return nil } + +// encodeKVPair encode a key-value pair in change set. +// see godoc of `encodedSizeOfKVPair` for layout description, +// returns error if key/value length overflows. +func encodeKVPair(pair *iavl.KVPair) ([]byte, error) { + buf := make([]byte, encodedSizeOfKVPair(pair)) + + offset := 1 + keyLen := len(pair.Key) + offset += binary.PutUvarint(buf[offset:], uint64(keyLen)) + + copy(buf[offset:], pair.Key) + if pair.Delete { + buf[0] = 1 + return buf, nil + } + + offset += keyLen + offset += binary.PutUvarint(buf[offset:], uint64(len(pair.Value))) + copy(buf[offset:], pair.Value) + return buf, nil +} + +// encodedSizeOfKVPair returns the encoded length of a key-value pair +// +// layout: deletion(1) + keyLen(varint) + key + [ valueLen(varint) + value ] +func encodedSizeOfKVPair(pair *iavl.KVPair) int { + keyLen := len(pair.Key) + size := 1 + uvarintSize(uint64(keyLen)) + keyLen + if pair.Delete { + return size + } + + valueLen := len(pair.Value) + return size + uvarintSize(uint64(valueLen)) + valueLen +} + +// uvarintSize function calculates the size (in bytes) needed to encode an unsigned integer in a variable-length format +// based on the number of bits required to represent the integer's value. +// This is a common operation when serializing data structures into binary formats where compactness and +// variable-length encoding are desired. +func uvarintSize(num uint64) int { + bits := bits.Len64(num) + q, r := bits/7, bits%7 + size := q + if r > 0 || size == 0 { + size++ + } + return size +} From d13275a10fb66ad8215bd1061609833c089b94dd Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 25 Sep 2023 10:21:11 -0700 Subject: [PATCH 10/72] Add logs --- go.mod | 1 + go.sum | 1 + 2 files changed, 2 insertions(+) diff --git a/go.mod b/go.mod index fc46d1e..7f15bd6 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/confio/ics23/go v0.7.0 github.com/gogo/protobuf v1.3.2 github.com/golang/mock v1.6.0 + github.com/klauspost/compress v1.15.1 github.com/pkg/errors v0.9.1 github.com/spf13/cobra v1.5.0 github.com/stretchr/testify v1.8.1 diff --git a/go.sum b/go.sum index eeafcd4..e2f255b 100644 --- a/go.sum +++ b/go.sum @@ -575,6 +575,7 @@ github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6 github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= From c42bbb718ab40fe64e76cb46553793ef55ffeea7 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Mon, 25 Sep 2023 10:23:35 -0700 Subject: [PATCH 11/72] Add logs --- cmd/changeset/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/changeset/main.go b/cmd/changeset/main.go index e3ab0a4..f5885f2 100644 --- a/cmd/changeset/main.go +++ b/cmd/changeset/main.go @@ -26,7 +26,7 @@ func main() { Use: "changeset", Short: "Dump change sets files which can be ingested into DB", } - rootCmd.PersistentFlags().StringP("home-dir", "h", "/root/.sei/data/application.db", "Home directory") + rootCmd.PersistentFlags().StringP("db-dir", "d", "/root/.sei/data/application.db", "Home directory") rootCmd.PersistentFlags().Int64("start-version", 0, "start version") rootCmd.PersistentFlags().Int64("end-version", 0, "start version") rootCmd.PersistentFlags().StringP("store", "s", DefaultStore, "store key name") @@ -51,7 +51,7 @@ func DumpChangeSetCmd() *cobra.Command { } func executeDumpChangesetCmd(cmd *cobra.Command, _ []string) error { - homeDir, _ := cmd.Flags().GetString("home-dir") + homeDir, _ := cmd.Flags().GetString("db-dir") storeKey, _ := cmd.Flags().GetString("store") startVersion, _ := cmd.Flags().GetInt64("start-version") endVersion, _ := cmd.Flags().GetInt64("end-version") From 71c938aa4635267b53f6ea9f646f87df1d91ae94 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Wed, 4 Oct 2023 04:33:00 -0700 Subject: [PATCH 12/72] Add export command --- cmd/changeset/changeset/export.go | 218 ++++++++++++++++++++++++++++++ cmd/changeset/changeset/import.go | 1 + cmd/changeset/main.go | 178 +++++------------------- docs/node/node.md | 4 +- fast_node.go | 2 +- go.mod | 1 + go.sum | 2 + internal/encoding/encoding.go | 4 +- node.go | 6 +- 9 files changed, 265 insertions(+), 151 deletions(-) create mode 100644 cmd/changeset/changeset/export.go create mode 100644 cmd/changeset/changeset/import.go diff --git a/cmd/changeset/changeset/export.go b/cmd/changeset/changeset/export.go new file mode 100644 index 0000000..b5214d2 --- /dev/null +++ b/cmd/changeset/changeset/export.go @@ -0,0 +1,218 @@ +package changeset + +import ( + "context" + "encoding/binary" + "fmt" + "io" + "math/bits" + "os" + "path/filepath" + "sync" + + "github.com/alitto/pond" + "github.com/cosmos/iavl" + "github.com/klauspost/compress/zstd" + dbm "github.com/tendermint/tm-db" +) + +const ( + DefaultCacheSize int = 100000 +) + +type CSExporter struct { + db *dbm.PrefixDB + start int64 + end int64 + concurrency int + segmentSize int + outputDir string +} + +func NewCSExporter(db *dbm.PrefixDB, start, end int64, concurrency int, segmentSize int, outputDir string) *CSExporter { + return &CSExporter{ + db: db, + start: start, + end: end, + concurrency: concurrency, + segmentSize: segmentSize, + outputDir: outputDir, + } +} + +func (exporter *CSExporter) Export() error { + + // use a worker pool with enough buffer to parallelize the export + pool := pond.New(exporter.concurrency, 1024) + defer pool.StopAndWait() + + // share the iavl tree between tasks to reuse the node cache + iavlTreePool := sync.Pool{ + New: func() any { + // use separate prefixdb and iavl tree in each task to maximize concurrency performance + return iavl.NewImmutableTree(exporter.db, DefaultCacheSize, true) + }, + } + + // split into segments + var segmentSize = exporter.segmentSize + var groups []*pond.TaskGroupWithContext + for i := exporter.start; i < exporter.end; i += int64(segmentSize) { + end := i + int64(segmentSize) + if end > exporter.end { + end = exporter.end + } + + group, _ := pool.GroupContext(context.Background()) + // split each segment according to number of workers + for _, workRange := range splitIntoSegments(exporter.concurrency, i, end) { + segmentRange := workRange + // each task group will export a segment file + group.Submit(func() error { + tree := iavlTreePool.Get().(*iavl.ImmutableTree) + defer iavlTreePool.Put(tree) + return dumpChangesetSegment(exporter.outputDir, tree, segmentRange) + }) + } + + groups = append(groups, group) + } + + // wait for all task groups to complete + for _, group := range groups { + if err := group.Wait(); err != nil { + return err + } + } + return nil +} + +type SegmentRange struct { + start int64 + end int64 +} + +func splitIntoSegments(numWorkers int, start int64, end int64) []SegmentRange { + var segments []SegmentRange + segmentSize := (end - start) / int64(numWorkers) + for i := start; i < end; i += segmentSize { + rangeEnd := i + segmentSize + if rangeEnd > end { + rangeEnd = end + } + segments = append(segments, SegmentRange{start, end}) + } + return segments +} + +func dumpChangesetSegment(outputDir string, tree *iavl.ImmutableTree, segment SegmentRange) (returnErr error) { + segmentFilePath := filepath.Join(outputDir, fmt.Sprintf("changeset-%d-%d.zst", segment.start, segment.end)) + segmentFile, err := createFile(segmentFilePath) + if err != nil { + return err + } + defer func() { + if err := segmentFile.Close(); returnErr == nil { + returnErr = err + } + }() + + zstdWriter, err := zstd.NewWriter(segmentFile) + if err != nil { + return err + } + if err := tree.TraverseStateChanges(segment.start, segment.end, func(version int64, changeSet *iavl.ChangeSet) error { + return WriteChangeSet(zstdWriter, version, *changeSet) + }); err != nil { + panic(err) + } + + return zstdWriter.Flush() +} + +func createFile(name string) (*os.File, error) { + return os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) +} + +func WriteChangeSet(writer io.Writer, version int64, cs iavl.ChangeSet) error { + if len(cs.Pairs) <= 0 { + return nil + } + + var size int + items := make([][]byte, 0, len(cs.Pairs)) + for _, pair := range cs.Pairs { + fmt.Printf("Version: %d, delete: %t, key: %X, value: %X\n", version, pair.Delete, pair.Key, pair.Value) + buf, err := encodeKVPair(pair) + if err != nil { + return err + } + size += len(buf) + items = append(items, buf) + } + + // Write header + var versionHeader [16]byte + binary.LittleEndian.PutUint64(versionHeader[:], uint64(version)) + binary.LittleEndian.PutUint64(versionHeader[8:], uint64(size)) + + if _, err := writer.Write(versionHeader[:]); err != nil { + return err + } + for _, item := range items { + if _, err := writer.Write(item); err != nil { + return err + } + } + return nil +} + +// encodeKVPair encode a key-value pair in change set. +// see godoc of `encodedSizeOfKVPair` for layout description, +// returns error if key/value length overflows. +func encodeKVPair(pair *iavl.KVPair) ([]byte, error) { + buf := make([]byte, encodedSizeOfKVPair(pair)) + + offset := 1 + keyLen := len(pair.Key) + offset += binary.PutUvarint(buf[offset:], uint64(keyLen)) + + copy(buf[offset:], pair.Key) + if pair.Delete { + buf[0] = 1 + return buf, nil + } + + offset += keyLen + offset += binary.PutUvarint(buf[offset:], uint64(len(pair.Value))) + copy(buf[offset:], pair.Value) + return buf, nil +} + +// encodedSizeOfKVPair returns the encoded length of a key-value pair +// +// layout: deletion(1) + keyLen(varint) + key + [ valueLen(varint) + value ] +func encodedSizeOfKVPair(pair *iavl.KVPair) int { + keyLen := len(pair.Key) + size := 1 + uvarintSize(uint64(keyLen)) + keyLen + if pair.Delete { + return size + } + + valueLen := len(pair.Value) + return size + uvarintSize(uint64(valueLen)) + valueLen +} + +// uvarintSize function calculates the size (in bytes) needed to encode an unsigned integer in a variable-length format +// based on the number of bits required to represent the integer's value. +// This is a common operation when serializing data structures into binary formats where compactness and +// variable-length encoding are desired. +func uvarintSize(num uint64) int { + bits := bits.Len64(num) + q, r := bits/7, bits%7 + size := q + if r > 0 || size == 0 { + size++ + } + return size +} diff --git a/cmd/changeset/changeset/import.go b/cmd/changeset/changeset/import.go new file mode 100644 index 0000000..d57836e --- /dev/null +++ b/cmd/changeset/changeset/import.go @@ -0,0 +1 @@ +package changeset diff --git a/cmd/changeset/main.go b/cmd/changeset/main.go index f5885f2..c27362a 100644 --- a/cmd/changeset/main.go +++ b/cmd/changeset/main.go @@ -1,104 +1,83 @@ package main import ( - "encoding/binary" + "errors" "fmt" - "github.com/cosmos/iavl" - "github.com/spf13/cobra" - dbm "github.com/tendermint/tm-db" - "io" - "math/bits" "os" "path/filepath" "strings" "time" - "github.com/klauspost/compress/zstd" -) - -const ( - DefaultCacheSize int = 10000 - DefaultStore string = "" + "github.com/cosmos/iavl/cmd/changeset/changeset" + "github.com/spf13/cobra" + dbm "github.com/tendermint/tm-db" ) func main() { rootCmd := &cobra.Command{ Use: "changeset", - Short: "Dump change sets files which can be ingested into DB", + Short: "Tools for export/import IAVL changesets", } - rootCmd.PersistentFlags().StringP("db-dir", "d", "/root/.sei/data/application.db", "Home directory") - rootCmd.PersistentFlags().Int64("start-version", 0, "start version") - rootCmd.PersistentFlags().Int64("end-version", 0, "start version") - rootCmd.PersistentFlags().StringP("store", "s", DefaultStore, "store key name") - rootCmd.PersistentFlags().StringP("output-dir", "o", "output", "output directory") + rootCmd.PersistentFlags().StringP("input-dir", "i", "/root/.sei/data/application.db", "Home directory") + rootCmd.PersistentFlags().StringP("stores", "s", "", "comma separated store key names, such as bank,staking") + rootCmd.PersistentFlags().StringP("output-dir", "o", "changeset-output", "output directory") - rootCmd.AddCommand(DumpChangeSetCmd()) + rootCmd.AddCommand(ExportChangeSetCmd()) if err := rootCmd.Execute(); err != nil { fmt.Println(err) os.Exit(1) } } -func DumpChangeSetCmd() *cobra.Command { +func ExportChangeSetCmd() *cobra.Command { cmd := &cobra.Command{ - Use: "dump", - Short: "Extract changesets from iavl versions, and save to plain file format", - RunE: executeDumpChangesetCmd, + Use: "export", + Short: "Extract changesets from IAVL versions, and save to plain file format", + RunE: exportChangeset, } + cmd.PersistentFlags().Int64("start-version", 0, "start version number") + cmd.PersistentFlags().Int64("end-version", 0, "end version number") + cmd.PersistentFlags().Int64("concurrency", 10, "number of concurrent workers") + cmd.PersistentFlags().Int64("segment-size", 1000000, "number of blocks in each segment") // we handle one stores at a time, because stores don't share much in db, handle concurrently reduces cache efficiency. return cmd } -func executeDumpChangesetCmd(cmd *cobra.Command, _ []string) error { - homeDir, _ := cmd.Flags().GetString("db-dir") - storeKey, _ := cmd.Flags().GetString("store") +func exportChangeset(cmd *cobra.Command, _ []string) error { + inputDir, _ := cmd.Flags().GetString("input-dir") + storeKeys, _ := cmd.Flags().GetString("stores") startVersion, _ := cmd.Flags().GetInt64("start-version") endVersion, _ := cmd.Flags().GetInt64("end-version") + concurrency, _ := cmd.Flags().GetInt("concurrency") outDir, _ := cmd.Flags().GetString("output-dir") - prefix := []byte(fmt.Sprintf("s/k:%s/", storeKey)) - if storeKey != "" { - fmt.Println("Begin dumping store with prefix", prefix, time.Now().Format(time.RFC3339)) + if storeKeys == "" { + return errors.New("stores is required") + } + storeKeysList := strings.Split(storeKeys, ",") + for _, storeKey := range storeKeysList { + prefix := []byte(fmt.Sprintf("s/k:%s/", storeKey)) + fmt.Printf("Begin exporting store with prefix %s, ", prefix, time.Now().Format(time.RFC3339)) if err := os.MkdirAll(outDir, os.ModePerm); err != nil { return err } - db, err := OpenDB(homeDir) - if err != nil { - return err - } - tree, err := iavl.NewMutableTree(dbm.NewPrefixDB(db, prefix), DefaultCacheSize, true) - if err != nil { + storeDir := filepath.Join(outDir, storeKey) + if err := os.MkdirAll(storeDir, os.ModePerm); err != nil { return err } - // Make sure we have a correct end version - if endVersion <= 0 { - latestVersion, _ := tree.LazyLoadVersion(0) - fmt.Printf("Got tree version: %d\n", latestVersion) - endVersion = latestVersion + 1 - } - iavlTree := iavl.NewImmutableTree(dbm.NewPrefixDB(db, prefix), DefaultCacheSize, true) - treeHash, err := iavlTree.Hash() + db, err := OpenDB(inputDir) if err != nil { return err } - fmt.Printf("Tree hash is %X, tree size is %d\n", treeHash, tree.ImmutableTree().Size()) - fmt.Printf("Going to traverse changeset from version %d to %d\n", startVersion, endVersion) - taskFile := filepath.Join(outDir, fmt.Sprintf("changeset-%s-%d.zst", storeKey, startVersion)) - outputFile, err := createFile(taskFile) + prefixDB := dbm.NewPrefixDB(db, prefix) + exporter := changeset.NewCSExporter(prefixDB, startVersion, endVersion, concurrency, storeDir) + err = exporter.Export() if err != nil { return err } - zstdWriter, err := zstd.NewWriter(outputFile) - if err != nil { - return err - } - if err := iavlTree.TraverseStateChanges(startVersion, endVersion, func(version int64, changeSet *iavl.ChangeSet) error { - return WriteChangeSet(zstdWriter, version, *changeSet) - }); err != nil { - panic(err) - } + } return nil } @@ -123,90 +102,3 @@ func OpenDB(dir string) (dbm.DB, error) { } return db, nil } - -func createFile(name string) (*os.File, error) { - return os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) -} - -func WriteChangeSet(writer io.Writer, version int64, cs iavl.ChangeSet) error { - if len(cs.Pairs) <= 0 { - return nil - } - - var size int - items := make([][]byte, 0, len(cs.Pairs)) - for _, pair := range cs.Pairs { - fmt.Printf("Version: %d, delete: %t, key: %X, value: %X\n", version, pair.Delete, pair.Key, pair.Value) - buf, err := encodeKVPair(pair) - if err != nil { - return err - } - size += len(buf) - items = append(items, buf) - } - - // Write header - var versionHeader [16]byte - binary.LittleEndian.PutUint64(versionHeader[:], uint64(version)) - binary.LittleEndian.PutUint64(versionHeader[8:], uint64(size)) - - if _, err := writer.Write(versionHeader[:]); err != nil { - return err - } - for _, item := range items { - if _, err := writer.Write(item); err != nil { - return err - } - } - return nil -} - -// encodeKVPair encode a key-value pair in change set. -// see godoc of `encodedSizeOfKVPair` for layout description, -// returns error if key/value length overflows. -func encodeKVPair(pair *iavl.KVPair) ([]byte, error) { - buf := make([]byte, encodedSizeOfKVPair(pair)) - - offset := 1 - keyLen := len(pair.Key) - offset += binary.PutUvarint(buf[offset:], uint64(keyLen)) - - copy(buf[offset:], pair.Key) - if pair.Delete { - buf[0] = 1 - return buf, nil - } - - offset += keyLen - offset += binary.PutUvarint(buf[offset:], uint64(len(pair.Value))) - copy(buf[offset:], pair.Value) - return buf, nil -} - -// encodedSizeOfKVPair returns the encoded length of a key-value pair -// -// layout: deletion(1) + keyLen(varint) + key + [ valueLen(varint) + value ] -func encodedSizeOfKVPair(pair *iavl.KVPair) int { - keyLen := len(pair.Key) - size := 1 + uvarintSize(uint64(keyLen)) + keyLen - if pair.Delete { - return size - } - - valueLen := len(pair.Value) - return size + uvarintSize(uint64(valueLen)) + valueLen -} - -// uvarintSize function calculates the size (in bytes) needed to encode an unsigned integer in a variable-length format -// based on the number of bits required to represent the integer's value. -// This is a common operation when serializing data structures into binary formats where compactness and -// variable-length encoding are desired. -func uvarintSize(num uint64) int { - bits := bits.Len64(num) - q, r := bits/7, bits%7 - size := q - if r > 0 || size == 0 { - size++ - } - return size -} diff --git a/docs/node/node.md b/docs/node/node.md index 24647fe..51adbcf 100644 --- a/docs/node/node.md +++ b/docs/node/node.md @@ -32,7 +32,7 @@ Size is the number of leaves under a given node. With a full subtree, `node.size Every node is persisted by encoding the key, version, height, size and hash. If the node is a leaf node, then the value is persisted as well. If the node is not a leaf node, then the leftHash and rightHash are persisted as well. ```golang -// Writes the node as a serialized byte slice to the supplied io.Writer. +// Writes the node as a serialized byte slice to the supplied changeset.Writer. func (node *Node) writeBytes(w io.Writer) error { cause := encodeVarint(w, node.height) if cause != nil { @@ -84,7 +84,7 @@ func (node *Node) writeBytes(w io.Writer) error { A node's hash is calculated by hashing the height, size, and version of the node. If the node is a leaf node, then the key and value are also hashed. If the node is an inner node, the leftHash and rightHash are included in hash but the key is not. ```golang -// Writes the node's hash to the given io.Writer. This function expects +// Writes the node's hash to the given changeset.Writer. This function expects // child hashes to be already set. func (node *Node) writeHashBytes(w io.Writer) error { err := encodeVarint(w, node.height) diff --git a/fast_node.go b/fast_node.go index b329245..76a1048 100644 --- a/fast_node.go +++ b/fast_node.go @@ -59,7 +59,7 @@ func (node *FastNode) encodedSize() int { return n } -// writeBytes writes the FastNode as a serialized byte slice to the supplied io.Writer. +// writeBytes writes the FastNode as a serialized byte slice to the supplied changeset.Writer. func (node *FastNode) writeBytes(w io.Writer) error { if node == nil { return errors.New("cannot write nil node") diff --git a/go.mod b/go.mod index 7f15bd6..2363ed9 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/cosmos/iavl go 1.18 require ( + github.com/alitto/pond v1.8.3 github.com/confio/ics23/go v0.7.0 github.com/gogo/protobuf v1.3.2 github.com/golang/mock v1.6.0 diff --git a/go.sum b/go.sum index e2f255b..143e6a5 100644 --- a/go.sum +++ b/go.sum @@ -107,6 +107,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alexkohler/prealloc v1.0.0/go.mod h1:VetnK3dIgFBBKmg0YnD9F9x6Icjd+9cvfHR56wJVlKE= +github.com/alitto/pond v1.8.3 h1:ydIqygCLVPqIX/USe5EaV/aSRXTRXDEI9JwuDdu+/xs= +github.com/alitto/pond v1.8.3/go.mod h1:CmvIIGd5jKLasGI3D87qDkQxjzChdKMmnXMg3fG6M6Q= github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= diff --git a/internal/encoding/encoding.go b/internal/encoding/encoding.go index 788a6ff..314d21b 100644 --- a/internal/encoding/encoding.go +++ b/internal/encoding/encoding.go @@ -116,7 +116,7 @@ func EncodeBytesSize(bz []byte) int { return EncodeUvarintSize(uint64(len(bz))) + len(bz) } -// EncodeUvarint writes a varint-encoded unsigned integer to an io.Writer. +// EncodeUvarint writes a varint-encoded unsigned integer to an changeset.Writer. func EncodeUvarint(w io.Writer, u uint64) error { // See comment in encodeVarint buf := uvarintPool.Get().(*[binary.MaxVarintLen64]byte) @@ -137,7 +137,7 @@ func EncodeUvarintSize(u uint64) int { return (bits.Len64(u) + 6) / 7 } -// EncodeVarint writes a varint-encoded integer to an io.Writer. +// EncodeVarint writes a varint-encoded integer to an changeset.Writer. func EncodeVarint(w io.Writer, i int64) error { // Use a pool here to reduce allocations. // diff --git a/node.go b/node.go index 898602e..ea413ae 100644 --- a/node.go +++ b/node.go @@ -473,7 +473,7 @@ func (node *Node) validate() error { return nil } -// Writes the node's hash to the given io.Writer. This function expects +// Writes the node's hash to the given changeset.Writer. This function expects // child hashes to be already set. func (node *Node) writeHashBytes(w io.Writer) error { err := encoding.EncodeVarint(w, int64(node.GetHeight())) @@ -522,7 +522,7 @@ func (node *Node) writeHashBytes(w io.Writer) error { return nil } -// Writes the node's hash to the given io.Writer. +// Writes the node's hash to the given changeset.Writer. // This function has the side-effect of calling hashWithCount. func (node *Node) writeHashBytesRecursively(w io.Writer) (hashCount int64, err error) { if node.GetLeftNode() != nil { @@ -560,7 +560,7 @@ func (node *Node) encodedSize() int { return n } -// Writes the node as a serialized byte slice to the supplied io.Writer. +// Writes the node as a serialized byte slice to the supplied changeset.Writer. func (node *Node) writeBytes(w io.Writer) error { if node == nil { return errors.New("cannot write nil node") From 89d57111a047ced76baa6d5ba0b4c870a19ce91f Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Wed, 4 Oct 2023 04:39:15 -0700 Subject: [PATCH 13/72] Add segment size --- cmd/changeset/main.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/changeset/main.go b/cmd/changeset/main.go index c27362a..38343f9 100644 --- a/cmd/changeset/main.go +++ b/cmd/changeset/main.go @@ -50,6 +50,7 @@ func exportChangeset(cmd *cobra.Command, _ []string) error { startVersion, _ := cmd.Flags().GetInt64("start-version") endVersion, _ := cmd.Flags().GetInt64("end-version") concurrency, _ := cmd.Flags().GetInt("concurrency") + segmentSize, _ := cmd.Flags().GetInt("segment-size") outDir, _ := cmd.Flags().GetString("output-dir") if storeKeys == "" { @@ -58,7 +59,7 @@ func exportChangeset(cmd *cobra.Command, _ []string) error { storeKeysList := strings.Split(storeKeys, ",") for _, storeKey := range storeKeysList { prefix := []byte(fmt.Sprintf("s/k:%s/", storeKey)) - fmt.Printf("Begin exporting store with prefix %s, ", prefix, time.Now().Format(time.RFC3339)) + fmt.Printf("Begin exporting store with prefix %s at %s", prefix, time.Now().Format(time.RFC3339)) if err := os.MkdirAll(outDir, os.ModePerm); err != nil { return err } @@ -72,7 +73,7 @@ func exportChangeset(cmd *cobra.Command, _ []string) error { return err } prefixDB := dbm.NewPrefixDB(db, prefix) - exporter := changeset.NewCSExporter(prefixDB, startVersion, endVersion, concurrency, storeDir) + exporter := changeset.NewCSExporter(prefixDB, startVersion, endVersion, concurrency, segmentSize, storeDir) err = exporter.Export() if err != nil { return err From c2b2336a2b2c102c64511167621d3ef27f4a61cf Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Wed, 4 Oct 2023 04:46:04 -0700 Subject: [PATCH 14/72] Add segment size --- cmd/changeset/changeset/export.go | 2 +- cmd/changeset/main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/changeset/changeset/export.go b/cmd/changeset/changeset/export.go index b5214d2..a1bc9d2 100644 --- a/cmd/changeset/changeset/export.go +++ b/cmd/changeset/changeset/export.go @@ -29,7 +29,7 @@ type CSExporter struct { outputDir string } -func NewCSExporter(db *dbm.PrefixDB, start, end int64, concurrency int, segmentSize int, outputDir string) *CSExporter { +func NewCSExporter(db *dbm.PrefixDB, start int64, end int64, concurrency int, segmentSize int, outputDir string) *CSExporter { return &CSExporter{ db: db, start: start, diff --git a/cmd/changeset/main.go b/cmd/changeset/main.go index 38343f9..e189ffd 100644 --- a/cmd/changeset/main.go +++ b/cmd/changeset/main.go @@ -37,7 +37,7 @@ func ExportChangeSetCmd() *cobra.Command { } cmd.PersistentFlags().Int64("start-version", 0, "start version number") cmd.PersistentFlags().Int64("end-version", 0, "end version number") - cmd.PersistentFlags().Int64("concurrency", 10, "number of concurrent workers") + cmd.PersistentFlags().Int("concurrency", 10, "number of concurrent workers") cmd.PersistentFlags().Int64("segment-size", 1000000, "number of blocks in each segment") // we handle one stores at a time, because stores don't share much in db, handle concurrently reduces cache efficiency. From be08b974beb151b7489fb53a518b391a510e01ae Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Wed, 4 Oct 2023 05:04:22 -0700 Subject: [PATCH 15/72] Add more logs --- cmd/changeset/changeset/export.go | 5 +++-- cmd/changeset/main.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/changeset/changeset/export.go b/cmd/changeset/changeset/export.go index a1bc9d2..7a36e25 100644 --- a/cmd/changeset/changeset/export.go +++ b/cmd/changeset/changeset/export.go @@ -41,7 +41,7 @@ func NewCSExporter(db *dbm.PrefixDB, start int64, end int64, concurrency int, se } func (exporter *CSExporter) Export() error { - + fmt.Printf("Exporting changeset with %v \n", exporter) // use a worker pool with enough buffer to parallelize the export pool := pond.New(exporter.concurrency, 1024) defer pool.StopAndWait() @@ -100,12 +100,13 @@ func splitIntoSegments(numWorkers int, start int64, end int64) []SegmentRange { if rangeEnd > end { rangeEnd = end } - segments = append(segments, SegmentRange{start, end}) + segments = append(segments, SegmentRange{i, rangeEnd}) } return segments } func dumpChangesetSegment(outputDir string, tree *iavl.ImmutableTree, segment SegmentRange) (returnErr error) { + fmt.Printf("Exporting changeset segment %d-%d\n", segment.start, segment.end) segmentFilePath := filepath.Join(outputDir, fmt.Sprintf("changeset-%d-%d.zst", segment.start, segment.end)) segmentFile, err := createFile(segmentFilePath) if err != nil { diff --git a/cmd/changeset/main.go b/cmd/changeset/main.go index e189ffd..715c624 100644 --- a/cmd/changeset/main.go +++ b/cmd/changeset/main.go @@ -59,7 +59,7 @@ func exportChangeset(cmd *cobra.Command, _ []string) error { storeKeysList := strings.Split(storeKeys, ",") for _, storeKey := range storeKeysList { prefix := []byte(fmt.Sprintf("s/k:%s/", storeKey)) - fmt.Printf("Begin exporting store with prefix %s at %s", prefix, time.Now().Format(time.RFC3339)) + fmt.Printf("Begin exporting store with prefix %s at %s\n", prefix, time.Now().Format(time.RFC3339)) if err := os.MkdirAll(outDir, os.ModePerm); err != nil { return err } From cbc6f4cd54f147fbe331da45b46fef1ee8378108 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Wed, 4 Oct 2023 05:06:00 -0700 Subject: [PATCH 16/72] Fix segment size --- cmd/changeset/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/changeset/main.go b/cmd/changeset/main.go index 715c624..836627e 100644 --- a/cmd/changeset/main.go +++ b/cmd/changeset/main.go @@ -50,7 +50,7 @@ func exportChangeset(cmd *cobra.Command, _ []string) error { startVersion, _ := cmd.Flags().GetInt64("start-version") endVersion, _ := cmd.Flags().GetInt64("end-version") concurrency, _ := cmd.Flags().GetInt("concurrency") - segmentSize, _ := cmd.Flags().GetInt("segment-size") + segmentSize, _ := cmd.Flags().GetInt64("segment-size") outDir, _ := cmd.Flags().GetString("output-dir") if storeKeys == "" { From f54347be670bdfe8bc8a18eb929998b4a4cac38e Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Wed, 4 Oct 2023 05:06:48 -0700 Subject: [PATCH 17/72] Fix segment size --- cmd/changeset/changeset/export.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/changeset/changeset/export.go b/cmd/changeset/changeset/export.go index 7a36e25..d313d7d 100644 --- a/cmd/changeset/changeset/export.go +++ b/cmd/changeset/changeset/export.go @@ -25,11 +25,11 @@ type CSExporter struct { start int64 end int64 concurrency int - segmentSize int + segmentSize int64 outputDir string } -func NewCSExporter(db *dbm.PrefixDB, start int64, end int64, concurrency int, segmentSize int, outputDir string) *CSExporter { +func NewCSExporter(db *dbm.PrefixDB, start int64, end int64, concurrency int, segmentSize int64, outputDir string) *CSExporter { return &CSExporter{ db: db, start: start, From 89d85c8958942905884c9d6787b40549db040c88 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Wed, 4 Oct 2023 05:14:21 -0700 Subject: [PATCH 18/72] Fix concurrency --- cmd/changeset/changeset/export.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cmd/changeset/changeset/export.go b/cmd/changeset/changeset/export.go index d313d7d..06d61e1 100644 --- a/cmd/changeset/changeset/export.go +++ b/cmd/changeset/changeset/export.go @@ -65,7 +65,7 @@ func (exporter *CSExporter) Export() error { group, _ := pool.GroupContext(context.Background()) // split each segment according to number of workers - for _, workRange := range splitIntoSegments(exporter.concurrency, i, end) { + for _, workRange := range splitIntoSegments(exporter.segmentSize, i, end) { segmentRange := workRange // each task group will export a segment file group.Submit(func() error { @@ -92,9 +92,8 @@ type SegmentRange struct { end int64 } -func splitIntoSegments(numWorkers int, start int64, end int64) []SegmentRange { +func splitIntoSegments(segmentSize int64, start int64, end int64) []SegmentRange { var segments []SegmentRange - segmentSize := (end - start) / int64(numWorkers) for i := start; i < end; i += segmentSize { rangeEnd := i + segmentSize if rangeEnd > end { From b25e7a8ab22152ee6b3b11cc99bb7fa002f17d2b Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Wed, 4 Oct 2023 05:20:18 -0700 Subject: [PATCH 19/72] Add logs --- cmd/changeset/changeset/export.go | 33 +++++++------------------------ 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/cmd/changeset/changeset/export.go b/cmd/changeset/changeset/export.go index 06d61e1..5aad296 100644 --- a/cmd/changeset/changeset/export.go +++ b/cmd/changeset/changeset/export.go @@ -57,24 +57,17 @@ func (exporter *CSExporter) Export() error { // split into segments var segmentSize = exporter.segmentSize var groups []*pond.TaskGroupWithContext - for i := exporter.start; i < exporter.end; i += int64(segmentSize) { - end := i + int64(segmentSize) + for i := exporter.start; i < exporter.end; i += segmentSize { + end := i + segmentSize if end > exporter.end { end = exporter.end } - group, _ := pool.GroupContext(context.Background()) - // split each segment according to number of workers - for _, workRange := range splitIntoSegments(exporter.segmentSize, i, end) { - segmentRange := workRange - // each task group will export a segment file - group.Submit(func() error { - tree := iavlTreePool.Get().(*iavl.ImmutableTree) - defer iavlTreePool.Put(tree) - return dumpChangesetSegment(exporter.outputDir, tree, segmentRange) - }) - } - + group.Submit(func() error { + tree := iavlTreePool.Get().(*iavl.ImmutableTree) + defer iavlTreePool.Put(tree) + return dumpChangesetSegment(exporter.outputDir, tree, SegmentRange{i, end}) + }) groups = append(groups, group) } @@ -92,18 +85,6 @@ type SegmentRange struct { end int64 } -func splitIntoSegments(segmentSize int64, start int64, end int64) []SegmentRange { - var segments []SegmentRange - for i := start; i < end; i += segmentSize { - rangeEnd := i + segmentSize - if rangeEnd > end { - rangeEnd = end - } - segments = append(segments, SegmentRange{i, rangeEnd}) - } - return segments -} - func dumpChangesetSegment(outputDir string, tree *iavl.ImmutableTree, segment SegmentRange) (returnErr error) { fmt.Printf("Exporting changeset segment %d-%d\n", segment.start, segment.end) segmentFilePath := filepath.Join(outputDir, fmt.Sprintf("changeset-%d-%d.zst", segment.start, segment.end)) From 0327e1ebe0379d13352b537cde954809937d0676 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Wed, 4 Oct 2023 05:39:01 -0700 Subject: [PATCH 20/72] Add logs --- cmd/changeset/changeset/export.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cmd/changeset/changeset/export.go b/cmd/changeset/changeset/export.go index 5aad296..d642d0a 100644 --- a/cmd/changeset/changeset/export.go +++ b/cmd/changeset/changeset/export.go @@ -41,7 +41,6 @@ func NewCSExporter(db *dbm.PrefixDB, start int64, end int64, concurrency int, se } func (exporter *CSExporter) Export() error { - fmt.Printf("Exporting changeset with %v \n", exporter) // use a worker pool with enough buffer to parallelize the export pool := pond.New(exporter.concurrency, 1024) defer pool.StopAndWait() @@ -66,7 +65,9 @@ func (exporter *CSExporter) Export() error { group.Submit(func() error { tree := iavlTreePool.Get().(*iavl.ImmutableTree) defer iavlTreePool.Put(tree) - return dumpChangesetSegment(exporter.outputDir, tree, SegmentRange{i, end}) + err := dumpChangesetSegment(exporter.outputDir, tree, SegmentRange{i, end}) + fmt.Printf("Finished exporting segment %d-%d\n", i, end) + return err }) groups = append(groups, group) } @@ -105,7 +106,7 @@ func dumpChangesetSegment(outputDir string, tree *iavl.ImmutableTree, segment Se if err := tree.TraverseStateChanges(segment.start, segment.end, func(version int64, changeSet *iavl.ChangeSet) error { return WriteChangeSet(zstdWriter, version, *changeSet) }); err != nil { - panic(err) + return err } return zstdWriter.Flush() @@ -123,7 +124,7 @@ func WriteChangeSet(writer io.Writer, version int64, cs iavl.ChangeSet) error { var size int items := make([][]byte, 0, len(cs.Pairs)) for _, pair := range cs.Pairs { - fmt.Printf("Version: %d, delete: %t, key: %X, value: %X\n", version, pair.Delete, pair.Key, pair.Value) + //fmt.Printf("Version: %d, delete: %t, key: %X, value: %X\n", version, pair.Delete, pair.Key, pair.Value) buf, err := encodeKVPair(pair) if err != nil { return err From 4e9cd09b33cef7e00dd79c73bb7d68a2c6cb19a1 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Wed, 4 Oct 2023 05:42:12 -0700 Subject: [PATCH 21/72] Add logs --- cmd/changeset/changeset/export.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/changeset/changeset/export.go b/cmd/changeset/changeset/export.go index d642d0a..837ed16 100644 --- a/cmd/changeset/changeset/export.go +++ b/cmd/changeset/changeset/export.go @@ -75,6 +75,7 @@ func (exporter *CSExporter) Export() error { // wait for all task groups to complete for _, group := range groups { if err := group.Wait(); err != nil { + fmt.Printf("Error: %v\n", err) return err } } From 4f54e2cd5214ab1f028e5fcfb3962d8e293a8524 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Wed, 4 Oct 2023 05:47:32 -0700 Subject: [PATCH 22/72] Add logs --- cmd/changeset/changeset/export.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/cmd/changeset/changeset/export.go b/cmd/changeset/changeset/export.go index 837ed16..3b610a4 100644 --- a/cmd/changeset/changeset/export.go +++ b/cmd/changeset/changeset/export.go @@ -62,11 +62,13 @@ func (exporter *CSExporter) Export() error { end = exporter.end } group, _ := pool.GroupContext(context.Background()) + startPos := i + endPos := end group.Submit(func() error { tree := iavlTreePool.Get().(*iavl.ImmutableTree) defer iavlTreePool.Put(tree) - err := dumpChangesetSegment(exporter.outputDir, tree, SegmentRange{i, end}) - fmt.Printf("Finished exporting segment %d-%d\n", i, end) + err := dumpChangesetSegment(exporter.outputDir, tree, startPos, endPos) + fmt.Printf("Finished exporting segment %d-%d\n", startPos, endPos) return err }) groups = append(groups, group) @@ -82,14 +84,9 @@ func (exporter *CSExporter) Export() error { return nil } -type SegmentRange struct { - start int64 - end int64 -} - -func dumpChangesetSegment(outputDir string, tree *iavl.ImmutableTree, segment SegmentRange) (returnErr error) { - fmt.Printf("Exporting changeset segment %d-%d\n", segment.start, segment.end) - segmentFilePath := filepath.Join(outputDir, fmt.Sprintf("changeset-%d-%d.zst", segment.start, segment.end)) +func dumpChangesetSegment(outputDir string, tree *iavl.ImmutableTree, start int64, end int64) (returnErr error) { + fmt.Printf("Exporting changeset segment %d-%d\n", start, end) + segmentFilePath := filepath.Join(outputDir, fmt.Sprintf("changeset-%d-%d.zst", start, end)) segmentFile, err := createFile(segmentFilePath) if err != nil { return err @@ -104,7 +101,7 @@ func dumpChangesetSegment(outputDir string, tree *iavl.ImmutableTree, segment Se if err != nil { return err } - if err := tree.TraverseStateChanges(segment.start, segment.end, func(version int64, changeSet *iavl.ChangeSet) error { + if err := tree.TraverseStateChanges(start, end, func(version int64, changeSet *iavl.ChangeSet) error { return WriteChangeSet(zstdWriter, version, *changeSet) }); err != nil { return err From a15ab32cb80a087e20cc8887fed712c9bf3ddb55 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Wed, 4 Oct 2023 05:58:38 -0700 Subject: [PATCH 23/72] Add logs --- cmd/changeset/changeset/export.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/cmd/changeset/changeset/export.go b/cmd/changeset/changeset/export.go index 3b610a4..c158ba8 100644 --- a/cmd/changeset/changeset/export.go +++ b/cmd/changeset/changeset/export.go @@ -88,22 +88,31 @@ func dumpChangesetSegment(outputDir string, tree *iavl.ImmutableTree, start int6 fmt.Printf("Exporting changeset segment %d-%d\n", start, end) segmentFilePath := filepath.Join(outputDir, fmt.Sprintf("changeset-%d-%d.zst", start, end)) segmentFile, err := createFile(segmentFilePath) + zstdWriter, err := zstd.NewWriter(segmentFile) + if err != nil { + fmt.Printf("Error: %v\n", err) return err } + defer func() { + err := zstdWriter.Close() + if err != nil { + returnErr = err + } if err := segmentFile.Close(); returnErr == nil { returnErr = err } }() - zstdWriter, err := zstd.NewWriter(segmentFile) if err != nil { + fmt.Printf("Error: %v\n", err) return err } if err := tree.TraverseStateChanges(start, end, func(version int64, changeSet *iavl.ChangeSet) error { return WriteChangeSet(zstdWriter, version, *changeSet) }); err != nil { + fmt.Printf("Error: %v\n", err) return err } @@ -115,6 +124,7 @@ func createFile(name string) (*os.File, error) { } func WriteChangeSet(writer io.Writer, version int64, cs iavl.ChangeSet) error { + fmt.Printf("Writing version: %d\n", version) if len(cs.Pairs) <= 0 { return nil } @@ -125,6 +135,7 @@ func WriteChangeSet(writer io.Writer, version int64, cs iavl.ChangeSet) error { //fmt.Printf("Version: %d, delete: %t, key: %X, value: %X\n", version, pair.Delete, pair.Key, pair.Value) buf, err := encodeKVPair(pair) if err != nil { + fmt.Printf("Error: %v\n", err) return err } size += len(buf) @@ -137,10 +148,12 @@ func WriteChangeSet(writer io.Writer, version int64, cs iavl.ChangeSet) error { binary.LittleEndian.PutUint64(versionHeader[8:], uint64(size)) if _, err := writer.Write(versionHeader[:]); err != nil { + fmt.Printf("Error: %v\n", err) return err } for _, item := range items { if _, err := writer.Write(item); err != nil { + fmt.Printf("Error: %v\n", err) return err } } From 77372218ef5d051040309cfeb284b2e7da515458 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Wed, 4 Oct 2023 06:04:22 -0700 Subject: [PATCH 24/72] Add logs --- nodedb.go | 1 + 1 file changed, 1 insertion(+) diff --git a/nodedb.go b/nodedb.go index f4eae46..decb468 100644 --- a/nodedb.go +++ b/nodedb.go @@ -1076,6 +1076,7 @@ func (ndb *nodeDB) traverseStateChanges(startVersion, endVersion int64, fn func( var version int64 rootKeyFormat.Scan(k, &version) + fmt.Printf("traversed version: %d\n", version) var changeSet ChangeSet receiveKVPair := func(pair *KVPair) error { changeSet.Pairs = append(changeSet.Pairs, pair) From 09ce6380bdeb86a42ec7ea9706bda04888d60c36 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Wed, 4 Oct 2023 06:09:19 -0700 Subject: [PATCH 25/72] Add logs --- nodedb.go | 1 + 1 file changed, 1 insertion(+) diff --git a/nodedb.go b/nodedb.go index decb468..88aec8d 100644 --- a/nodedb.go +++ b/nodedb.go @@ -812,6 +812,7 @@ func (ndb *nodeDB) traverse(fn func(key, value []byte) error) error { // Traverse all keys between a given range (excluding end) and return error if any, nil otherwise func (ndb *nodeDB) traverseRange(start []byte, end []byte, fn func(k, v []byte) error) error { itr, err := ndb.db.Iterator(start, end) + fmt.Printf("itr range: %v\n", itr) if err != nil { return err } From 55d9388716c92c56930b31ebaa40086f1380becc Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Wed, 4 Oct 2023 06:10:11 -0700 Subject: [PATCH 26/72] Add logs --- nodedb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodedb.go b/nodedb.go index 88aec8d..761753a 100644 --- a/nodedb.go +++ b/nodedb.go @@ -812,7 +812,7 @@ func (ndb *nodeDB) traverse(fn func(key, value []byte) error) error { // Traverse all keys between a given range (excluding end) and return error if any, nil otherwise func (ndb *nodeDB) traverseRange(start []byte, end []byte, fn func(k, v []byte) error) error { itr, err := ndb.db.Iterator(start, end) - fmt.Printf("itr range: %v\n", itr) + fmt.Printf("itr range from %s to %s\n", string(start), string(end)) if err != nil { return err } From 9dba74c3c50c8bce8a0ada7adbe43baae1fad848 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Wed, 4 Oct 2023 06:11:17 -0700 Subject: [PATCH 27/72] Add logs --- nodedb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodedb.go b/nodedb.go index 761753a..14004c4 100644 --- a/nodedb.go +++ b/nodedb.go @@ -812,7 +812,6 @@ func (ndb *nodeDB) traverse(fn func(key, value []byte) error) error { // Traverse all keys between a given range (excluding end) and return error if any, nil otherwise func (ndb *nodeDB) traverseRange(start []byte, end []byte, fn func(k, v []byte) error) error { itr, err := ndb.db.Iterator(start, end) - fmt.Printf("itr range from %s to %s\n", string(start), string(end)) if err != nil { return err } @@ -1073,6 +1072,7 @@ func (ndb *nodeDB) traverseStateChanges(startVersion, endVersion int64, fn func( if err != nil { return err } + fmt.Printf("Traversing range from %d to %d\n", startVersion, endVersion) return ndb.traverseRange(rootKeyFormat.Key(startVersion), rootKeyFormat.Key(endVersion), func(k, hash []byte) error { var version int64 rootKeyFormat.Scan(k, &version) From 09eb59903220ad3334735af912201d9dfec03651 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Wed, 4 Oct 2023 06:13:54 -0700 Subject: [PATCH 28/72] Add logs --- nodedb.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nodedb.go b/nodedb.go index 14004c4..3593977 100644 --- a/nodedb.go +++ b/nodedb.go @@ -1084,10 +1084,12 @@ func (ndb *nodeDB) traverseStateChanges(startVersion, endVersion int64, fn func( return nil } + fmt.Printf("extractStateChanges version: %d\n", version) if err := ndb.extractStateChanges(predecessor, prevRoot, hash, receiveKVPair); err != nil { return err } + fmt.Printf("applying fn for version: %d\n", version) if err := fn(version, &changeSet); err != nil { return err } From ad8d8bb4457fe9adf484b34721264d990b073cc1 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Wed, 4 Oct 2023 06:33:02 -0700 Subject: [PATCH 29/72] Add logs --- diff.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/diff.go b/diff.go index 327de6e..7463f0c 100644 --- a/diff.go +++ b/diff.go @@ -2,6 +2,7 @@ package iavl import ( "bytes" + "fmt" "github.com/cosmos/iavl/proto" ) @@ -21,16 +22,18 @@ type KVPairReceiver func(pair *KVPair) error // // The algorithm don't run in constant memory strictly, but it tried the best the only // keep minimal intermediate states in memory. -func (ndb *nodeDB) extractStateChanges(prevVersion int64, prevRoot []byte, root []byte, receiver KVPairReceiver) error { +func (ndb *nodeDB) EextractStateChanges(prevVersion int64, prevRoot []byte, root []byte, receiver KVPairReceiver) error { curIter, err := NewNodeIterator(root, ndb) if err != nil { return err } + fmt.Printf("curIter %d\n", prevVersion) prevIter, err := NewNodeIterator(prevRoot, ndb) if err != nil { return err } + fmt.Printf("prevIter \n") var ( // current shared node between two versions @@ -78,10 +81,13 @@ func (ndb *nodeDB) extractStateChanges(prevVersion int64, prevRoot []byte, root return nil } + if err := advanceSharedNode(); err != nil { return err } + fmt.Printf("after advanceSharedNode\n") + // addOrphanedLeave receives a new orphaned leave node found in previous version, // compare with the current newLeaves, to produce `iavl.KVPair` stream. addOrphanedLeave := func(orphaned *Node) error { @@ -140,11 +146,14 @@ func (ndb *nodeDB) extractStateChanges(prevVersion int64, prevRoot []byte, root } } } + fmt.Printf("before consumeNewLeaves \n") if err := consumeNewLeaves(); err != nil { return err } + fmt.Printf("after consumeNewLeaves \n") + if err := curIter.Error(); err != nil { return err } From eeba76a54394bc082573d553e254a214a3b4a9b7 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Wed, 4 Oct 2023 06:33:47 -0700 Subject: [PATCH 30/72] Add logs --- diff.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/diff.go b/diff.go index 7463f0c..4893417 100644 --- a/diff.go +++ b/diff.go @@ -22,7 +22,7 @@ type KVPairReceiver func(pair *KVPair) error // // The algorithm don't run in constant memory strictly, but it tried the best the only // keep minimal intermediate states in memory. -func (ndb *nodeDB) EextractStateChanges(prevVersion int64, prevRoot []byte, root []byte, receiver KVPairReceiver) error { +func (ndb *nodeDB) ExtractStateChanges(prevVersion int64, prevRoot []byte, root []byte, receiver KVPairReceiver) error { curIter, err := NewNodeIterator(root, ndb) if err != nil { return err From 4d4d88256904828ecce7a449378c96efa7e02b72 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Wed, 4 Oct 2023 06:34:27 -0700 Subject: [PATCH 31/72] Add logs --- diff.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/diff.go b/diff.go index 4893417..312fe90 100644 --- a/diff.go +++ b/diff.go @@ -22,7 +22,7 @@ type KVPairReceiver func(pair *KVPair) error // // The algorithm don't run in constant memory strictly, but it tried the best the only // keep minimal intermediate states in memory. -func (ndb *nodeDB) ExtractStateChanges(prevVersion int64, prevRoot []byte, root []byte, receiver KVPairReceiver) error { +func (ndb *nodeDB) extractStateChanges(prevVersion int64, prevRoot []byte, root []byte, receiver KVPairReceiver) error { curIter, err := NewNodeIterator(root, ndb) if err != nil { return err From aff12e141332e3993dc72167e432a47b01ef0746 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Wed, 4 Oct 2023 06:35:31 -0700 Subject: [PATCH 32/72] Add logs --- diff.go | 1 + 1 file changed, 1 insertion(+) diff --git a/diff.go b/diff.go index 312fe90..2e2c6d8 100644 --- a/diff.go +++ b/diff.go @@ -81,6 +81,7 @@ func (ndb *nodeDB) extractStateChanges(prevVersion int64, prevRoot []byte, root return nil } + fmt.Printf("before advanceSharedNode\n") if err := advanceSharedNode(); err != nil { return err From aaf2bb68e158128c13c6429deefc8b032424ddbb Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Wed, 4 Oct 2023 06:38:40 -0700 Subject: [PATCH 33/72] Add logs --- diff.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/diff.go b/diff.go index 2e2c6d8..15288d2 100644 --- a/diff.go +++ b/diff.go @@ -65,10 +65,12 @@ func (ndb *nodeDB) extractStateChanges(prevVersion int64, prevRoot []byte, root if err := consumeNewLeaves(); err != nil { return err } + fmt.Printf("after consumeNewLeaves\n") sharedNode = nil for curIter.Valid() { node := curIter.GetNode() + fmt.Printf("curIter is valid %d\n", node.version) shared := node.version <= prevVersion curIter.Next(shared) if shared { From 3b1672234c6da51652c5f70eafce7e1873418164 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Wed, 4 Oct 2023 06:45:10 -0700 Subject: [PATCH 34/72] Add logs --- diff.go | 4 +--- nodedb.go | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/diff.go b/diff.go index 15288d2..e05541d 100644 --- a/diff.go +++ b/diff.go @@ -33,7 +33,6 @@ func (ndb *nodeDB) extractStateChanges(prevVersion int64, prevRoot []byte, root if err != nil { return err } - fmt.Printf("prevIter \n") var ( // current shared node between two versions @@ -65,13 +64,12 @@ func (ndb *nodeDB) extractStateChanges(prevVersion int64, prevRoot []byte, root if err := consumeNewLeaves(); err != nil { return err } - fmt.Printf("after consumeNewLeaves\n") sharedNode = nil for curIter.Valid() { node := curIter.GetNode() - fmt.Printf("curIter is valid %d\n", node.version) shared := node.version <= prevVersion + fmt.Printf("curIter is valid with version %d and prevVersion %d\n", node.version, prevVersion) curIter.Next(shared) if shared { sharedNode = node diff --git a/nodedb.go b/nodedb.go index 3593977..ca5025a 100644 --- a/nodedb.go +++ b/nodedb.go @@ -1072,7 +1072,7 @@ func (ndb *nodeDB) traverseStateChanges(startVersion, endVersion int64, fn func( if err != nil { return err } - fmt.Printf("Traversing range from %d to %d\n", startVersion, endVersion) + fmt.Printf("Traversing range from %d to %d with predecessor %d\n", startVersion, endVersion, predecessor) return ndb.traverseRange(rootKeyFormat.Key(startVersion), rootKeyFormat.Key(endVersion), func(k, hash []byte) error { var version int64 rootKeyFormat.Scan(k, &version) From 505449f8edee75e5800e73ce408729d0a7c319db Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Wed, 4 Oct 2023 07:10:51 -0700 Subject: [PATCH 35/72] Add logs --- diff.go | 2 +- nodedb.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/diff.go b/diff.go index e05541d..d4f52f4 100644 --- a/diff.go +++ b/diff.go @@ -69,7 +69,7 @@ func (ndb *nodeDB) extractStateChanges(prevVersion int64, prevRoot []byte, root for curIter.Valid() { node := curIter.GetNode() shared := node.version <= prevVersion - fmt.Printf("curIter is valid with version %d and prevVersion %d\n", node.version, prevVersion) + fmt.Printf("curIter version %d has %d nodesToVisit \n", node.version, len(curIter.nodesToVisit)) curIter.Next(shared) if shared { sharedNode = node diff --git a/nodedb.go b/nodedb.go index ca5025a..5264432 100644 --- a/nodedb.go +++ b/nodedb.go @@ -764,7 +764,6 @@ func (ndb *nodeDB) getPreviousVersion(version int64) (int64, error) { rootKeyFormat.Scan(k, &pversion) return pversion, nil } - if err := itr.Error(); err != nil { return 0, err } From 13b74a6de67851b915f6618a81917bde9c64e2bf Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Thu, 5 Oct 2023 10:14:36 -0700 Subject: [PATCH 36/72] Remove logs --- diff.go | 5 ----- nodedb.go | 6 +----- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/diff.go b/diff.go index d4f52f4..277b854 100644 --- a/diff.go +++ b/diff.go @@ -27,7 +27,6 @@ func (ndb *nodeDB) extractStateChanges(prevVersion int64, prevRoot []byte, root if err != nil { return err } - fmt.Printf("curIter %d\n", prevVersion) prevIter, err := NewNodeIterator(prevRoot, ndb) if err != nil { @@ -69,7 +68,6 @@ func (ndb *nodeDB) extractStateChanges(prevVersion int64, prevRoot []byte, root for curIter.Valid() { node := curIter.GetNode() shared := node.version <= prevVersion - fmt.Printf("curIter version %d has %d nodesToVisit \n", node.version, len(curIter.nodesToVisit)) curIter.Next(shared) if shared { sharedNode = node @@ -81,14 +79,11 @@ func (ndb *nodeDB) extractStateChanges(prevVersion int64, prevRoot []byte, root return nil } - fmt.Printf("before advanceSharedNode\n") if err := advanceSharedNode(); err != nil { return err } - fmt.Printf("after advanceSharedNode\n") - // addOrphanedLeave receives a new orphaned leave node found in previous version, // compare with the current newLeaves, to produce `iavl.KVPair` stream. addOrphanedLeave := func(orphaned *Node) error { diff --git a/nodedb.go b/nodedb.go index 5264432..019a4ec 100644 --- a/nodedb.go +++ b/nodedb.go @@ -1060,7 +1060,7 @@ func (ndb *nodeDB) traverseNodes(fn func(hash []byte, node *Node) error) error { return nil } -// traverseStateChanges iterate the range of versions, compare each version to it's predecessor to extract the state changes of it. +// traverseStateChanges iterate the range of versions, compare each version to its predecessor to extract the state changes of it. // endVersion is exclusive, set to `math.MaxInt64` to cover the latest version. func (ndb *nodeDB) traverseStateChanges(startVersion, endVersion int64, fn func(version int64, changeSet *ChangeSet) error) error { predecessor, err := ndb.getPreviousVersion(startVersion) @@ -1071,24 +1071,20 @@ func (ndb *nodeDB) traverseStateChanges(startVersion, endVersion int64, fn func( if err != nil { return err } - fmt.Printf("Traversing range from %d to %d with predecessor %d\n", startVersion, endVersion, predecessor) return ndb.traverseRange(rootKeyFormat.Key(startVersion), rootKeyFormat.Key(endVersion), func(k, hash []byte) error { var version int64 rootKeyFormat.Scan(k, &version) - fmt.Printf("traversed version: %d\n", version) var changeSet ChangeSet receiveKVPair := func(pair *KVPair) error { changeSet.Pairs = append(changeSet.Pairs, pair) return nil } - fmt.Printf("extractStateChanges version: %d\n", version) if err := ndb.extractStateChanges(predecessor, prevRoot, hash, receiveKVPair); err != nil { return err } - fmt.Printf("applying fn for version: %d\n", version) if err := fn(version, &changeSet); err != nil { return err } From d75b4598ebbefad7bdf7ace99960c6fc646df16c Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Thu, 5 Oct 2023 10:40:28 -0700 Subject: [PATCH 37/72] Remove logs --- diff.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/diff.go b/diff.go index 277b854..47019a9 100644 --- a/diff.go +++ b/diff.go @@ -2,8 +2,6 @@ package iavl import ( "bytes" - "fmt" - "github.com/cosmos/iavl/proto" ) @@ -142,14 +140,10 @@ func (ndb *nodeDB) extractStateChanges(prevVersion int64, prevRoot []byte, root } } } - fmt.Printf("before consumeNewLeaves \n") if err := consumeNewLeaves(); err != nil { return err } - - fmt.Printf("after consumeNewLeaves \n") - if err := curIter.Error(); err != nil { return err } From 56378bd34a9b67adc6f01b48c0ef28ad3ffde71d Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Thu, 5 Oct 2023 10:53:05 -0700 Subject: [PATCH 38/72] Remove logs --- cmd/changeset/changeset/export.go | 3 +-- cmd/changeset/main.go | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/cmd/changeset/changeset/export.go b/cmd/changeset/changeset/export.go index c158ba8..31b51a7 100644 --- a/cmd/changeset/changeset/export.go +++ b/cmd/changeset/changeset/export.go @@ -17,7 +17,7 @@ import ( ) const ( - DefaultCacheSize int = 100000 + DefaultCacheSize int = 1000000 ) type CSExporter struct { @@ -124,7 +124,6 @@ func createFile(name string) (*os.File, error) { } func WriteChangeSet(writer io.Writer, version int64, cs iavl.ChangeSet) error { - fmt.Printf("Writing version: %d\n", version) if len(cs.Pairs) <= 0 { return nil } diff --git a/cmd/changeset/main.go b/cmd/changeset/main.go index 836627e..8070b6d 100644 --- a/cmd/changeset/main.go +++ b/cmd/changeset/main.go @@ -57,6 +57,7 @@ func exportChangeset(cmd *cobra.Command, _ []string) error { return errors.New("stores is required") } storeKeysList := strings.Split(storeKeys, ",") + db, err := OpenDB(inputDir) for _, storeKey := range storeKeysList { prefix := []byte(fmt.Sprintf("s/k:%s/", storeKey)) fmt.Printf("Begin exporting store with prefix %s at %s\n", prefix, time.Now().Format(time.RFC3339)) @@ -67,8 +68,6 @@ func exportChangeset(cmd *cobra.Command, _ []string) error { if err := os.MkdirAll(storeDir, os.ModePerm); err != nil { return err } - - db, err := OpenDB(inputDir) if err != nil { return err } From 3cdf9ff4036cceb9c74df287a68a9fd68547dc0e Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Fri, 6 Oct 2023 12:18:07 -0700 Subject: [PATCH 39/72] Add importer and reader for reading changeset --- .../changeset => changeset}/export.go | 15 +- changeset/import.go | 143 ++++++++++++++++++ changeset/reader.go | 31 ++++ cmd/changeset/changeset/import.go | 1 - cmd/changeset/export.go | 87 +++++++++++ cmd/changeset/main.go | 84 +--------- cmd/changeset/print.go | 36 +++++ 7 files changed, 310 insertions(+), 87 deletions(-) rename {cmd/changeset/changeset => changeset}/export.go (95%) create mode 100644 changeset/import.go create mode 100644 changeset/reader.go delete mode 100644 cmd/changeset/changeset/import.go create mode 100644 cmd/changeset/export.go create mode 100644 cmd/changeset/print.go diff --git a/cmd/changeset/changeset/export.go b/changeset/export.go similarity index 95% rename from cmd/changeset/changeset/export.go rename to changeset/export.go index 31b51a7..578c97f 100644 --- a/cmd/changeset/changeset/export.go +++ b/changeset/export.go @@ -20,7 +20,7 @@ const ( DefaultCacheSize int = 1000000 ) -type CSExporter struct { +type Exporter struct { db *dbm.PrefixDB start int64 end int64 @@ -29,8 +29,15 @@ type CSExporter struct { outputDir string } -func NewCSExporter(db *dbm.PrefixDB, start int64, end int64, concurrency int, segmentSize int64, outputDir string) *CSExporter { - return &CSExporter{ +func NewExporter( + db *dbm.PrefixDB, + start int64, + end int64, + concurrency int, + segmentSize int64, + outputDir string, +) *Exporter { + return &Exporter{ db: db, start: start, end: end, @@ -40,7 +47,7 @@ func NewCSExporter(db *dbm.PrefixDB, start int64, end int64, concurrency int, se } } -func (exporter *CSExporter) Export() error { +func (exporter *Exporter) Start() error { // use a worker pool with enough buffer to parallelize the export pool := pond.New(exporter.concurrency, 1024) defer pool.StopAndWait() diff --git a/changeset/import.go b/changeset/import.go new file mode 100644 index 0000000..0b0757e --- /dev/null +++ b/changeset/import.go @@ -0,0 +1,143 @@ +package changeset + +import ( + "bufio" + "encoding/binary" + "fmt" + "github.com/DataDog/zstd" + "github.com/cosmos/iavl" + "io" + "os" +) + +type Importer struct { + inputFile string + processFn func(version int64, changeset *iavl.ChangeSet) (bool, error) +} + +func NewImporter(inputFile string) *Importer { + return &Importer{ + inputFile: inputFile, + } +} + +func (importer *Importer) WithProcessFn(processFn func(version int64, changeset *iavl.ChangeSet) (bool, error)) *Importer { + importer.processFn = processFn + return importer +} + +// Reader combines `io.Reader` and `io.ByteReader`. +type Reader interface { + io.ReadCloser + io.ByteReader + io.Closer +} + +func (importer *Importer) Start() (int64, error) { + reader, err := openChangesetFile(importer.inputFile) + if err != nil { + return 0, err + } + defer func(reader Reader) { + _ = reader.Close() + }(reader) + return iterateChangeSet(reader, importer.processFn) + +} + +func openChangesetFile(fileName string) (Reader, error) { + fp, err := os.Open(fileName) + if err != nil { + return nil, err + } + zstdReaderCloser := zstd.NewReader(fp) + bufReader := bufio.NewReader(zstdReaderCloser) + return &WrapReader{zstdReaderCloser, bufReader, fp}, nil +} + +func iterateChangeSet(reader Reader, fn func(version int64, changeset *iavl.ChangeSet) (bool, error)) (int64, error) { + var lastOffset int64 + for true { + offset, version, changeSet, err := readNextChangeset(reader) + if err != nil { + if err == io.EOF { + // Reaching here means we are done reading everything in the file + break + } else { + return lastOffset, err + } + } + shouldStop, err := fn(version, changeSet) + lastOffset += offset + if shouldStop { + break + } + } + return lastOffset, nil +} + +// readNextChangeset reads the next changeset from the given reader and returns +// the read offset, the version, the changeset itself and the error +func readNextChangeset(reader Reader) (int64, int64, *iavl.ChangeSet, error) { + var versionHeader [16]byte + if _, err := io.ReadFull(reader, versionHeader[:]); err != nil { + return 0, 0, nil, err + } + // Read header + version := binary.LittleEndian.Uint64(versionHeader[:8]) + size := int64(binary.LittleEndian.Uint64(versionHeader[8:16])) + if size <= 0 { + return 16, int64(version), nil, nil + } + var changeset iavl.ChangeSet + var offset int64 + for offset < size { + pair, err := readKVPair(reader) + if err != nil { + return 0, 0, nil, err + } + offset += int64(encodedSizeOfKVPair(pair)) + changeset.Pairs = append(changeset.Pairs, pair) + } + if offset != size { + return 0, 0, nil, fmt.Errorf("read version %d beyond payload size limit, size: %d, offset: %d", version, size, offset) + } + return size + 16, int64(version), &changeset, nil +} + +// readKVPair decode a key-value pair from reader +// see `encodedSizeOfKVPair` for layout description +func readKVPair(reader Reader) (*iavl.KVPair, error) { + deletion, err := reader.ReadByte() + if err != nil { + return nil, err + } + + keyLen, err := binary.ReadUvarint(reader) + if err != nil { + return nil, err + } + + pair := iavl.KVPair{ + Delete: deletion == 1, + Key: make([]byte, keyLen), + } + if _, err := io.ReadFull(reader, pair.Key); err != nil { + return nil, err + } + + if pair.Delete { + return &pair, nil + } + + valueLen, err := binary.ReadUvarint(reader) + if err != nil { + return nil, err + } + pair.Value = make([]byte, valueLen) + if _, err := io.ReadFull(reader, pair.Value); err != nil { + return nil, err + } + + return &pair, nil +} diff --git a/changeset/reader.go b/changeset/reader.go new file mode 100644 index 0000000..8448bad --- /dev/null +++ b/changeset/reader.go @@ -0,0 +1,31 @@ +package changeset + +import ( + "errors" + "io" +) + +type WrapReader struct { + readerCloser io.ReadCloser + byteReader io.ByteReader + closer io.Closer +} + +func (r *WrapReader) Read(p []byte) (int, error) { + return r.readerCloser.Read(p) +} + +func (r *WrapReader) ReadByte() (byte, error) { + return r.byteReader.ReadByte() +} + +func (r *WrapReader) Close() error { + var errs []error + if closer, ok := r.readerCloser.(io.Closer); ok { + errs = append(errs, closer.Close()) + } + if r.closer != nil { + errs = append(errs, r.closer.Close()) + } + return errors.Join(errs...) +} diff --git a/cmd/changeset/changeset/import.go b/cmd/changeset/changeset/import.go deleted file mode 100644 index d57836e..0000000 --- a/cmd/changeset/changeset/import.go +++ /dev/null @@ -1 +0,0 @@ -package changeset diff --git a/cmd/changeset/export.go b/cmd/changeset/export.go new file mode 100644 index 0000000..0d517d3 --- /dev/null +++ b/cmd/changeset/export.go @@ -0,0 +1,87 @@ +package main + +import ( + "errors" + "fmt" + "github.com/cosmos/iavl/changeset" + "github.com/spf13/cobra" + dbm "github.com/tendermint/tm-db" + "os" + "path/filepath" + "strings" + "time" +) + +func ExportChangeSetCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "export", + Short: "Extract changesets from IAVL versions, and save to plain file format", + RunE: handleExportChangeset, + } + cmd.PersistentFlags().Int64("start-version", 0, "start version number") + cmd.PersistentFlags().Int64("end-version", 0, "end version number") + cmd.PersistentFlags().Int("concurrency", 10, "number of concurrent workers") + cmd.PersistentFlags().Int64("segment-size", 1000000, "number of blocks in each segment") + + // we handle one stores at a time, because stores don't share much in db, handle concurrently reduces cache efficiency. + return cmd +} + +func handleExportChangeset(cmd *cobra.Command, _ []string) error { + inputDir, _ := cmd.Flags().GetString("input-dir") + storeKeys, _ := cmd.Flags().GetString("stores") + startVersion, _ := cmd.Flags().GetInt64("start-version") + endVersion, _ := cmd.Flags().GetInt64("end-version") + concurrency, _ := cmd.Flags().GetInt("concurrency") + segmentSize, _ := cmd.Flags().GetInt64("segment-size") + outDir, _ := cmd.Flags().GetString("output-dir") + + if storeKeys == "" { + return errors.New("stores is required") + } + storeKeysList := strings.Split(storeKeys, ",") + db, err := OpenDB(inputDir) + for _, storeKey := range storeKeysList { + prefix := []byte(fmt.Sprintf("s/k:%s/", storeKey)) + fmt.Printf("Begin exporting store with prefix %s at %s\n", prefix, time.Now().Format(time.RFC3339)) + if err := os.MkdirAll(outDir, os.ModePerm); err != nil { + return err + } + storeDir := filepath.Join(outDir, storeKey) + if err := os.MkdirAll(storeDir, os.ModePerm); err != nil { + return err + } + if err != nil { + return err + } + prefixDB := dbm.NewPrefixDB(db, prefix) + exporter := changeset.NewExporter(prefixDB, startVersion, endVersion, concurrency, segmentSize, storeDir) + err = exporter.Start() + if err != nil { + return err + } + + } + return nil +} + +func OpenDB(dir string) (dbm.DB, error) { + switch { + case strings.HasSuffix(dir, ".db"): + dir = dir[:len(dir)-3] + case strings.HasSuffix(dir, ".db/"): + dir = dir[:len(dir)-4] + default: + return nil, fmt.Errorf("database directory must end with .db") + } + cut := strings.LastIndex(dir, "/") + if cut == -1 { + return nil, fmt.Errorf("cannot cut paths on %s", dir) + } + name := dir[cut+1:] + db, err := dbm.NewGoLevelDB(name, dir[:cut]) + if err != nil { + return nil, err + } + return db, nil +} diff --git a/cmd/changeset/main.go b/cmd/changeset/main.go index 8070b6d..a4140a6 100644 --- a/cmd/changeset/main.go +++ b/cmd/changeset/main.go @@ -1,16 +1,9 @@ package main import ( - "errors" "fmt" - "os" - "path/filepath" - "strings" - "time" - - "github.com/cosmos/iavl/cmd/changeset/changeset" "github.com/spf13/cobra" - dbm "github.com/tendermint/tm-db" + "os" ) func main() { @@ -23,82 +16,9 @@ func main() { rootCmd.PersistentFlags().StringP("output-dir", "o", "changeset-output", "output directory") rootCmd.AddCommand(ExportChangeSetCmd()) + rootCmd.AddCommand() if err := rootCmd.Execute(); err != nil { fmt.Println(err) os.Exit(1) } } - -func ExportChangeSetCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "export", - Short: "Extract changesets from IAVL versions, and save to plain file format", - RunE: exportChangeset, - } - cmd.PersistentFlags().Int64("start-version", 0, "start version number") - cmd.PersistentFlags().Int64("end-version", 0, "end version number") - cmd.PersistentFlags().Int("concurrency", 10, "number of concurrent workers") - cmd.PersistentFlags().Int64("segment-size", 1000000, "number of blocks in each segment") - - // we handle one stores at a time, because stores don't share much in db, handle concurrently reduces cache efficiency. - return cmd -} - -func exportChangeset(cmd *cobra.Command, _ []string) error { - inputDir, _ := cmd.Flags().GetString("input-dir") - storeKeys, _ := cmd.Flags().GetString("stores") - startVersion, _ := cmd.Flags().GetInt64("start-version") - endVersion, _ := cmd.Flags().GetInt64("end-version") - concurrency, _ := cmd.Flags().GetInt("concurrency") - segmentSize, _ := cmd.Flags().GetInt64("segment-size") - outDir, _ := cmd.Flags().GetString("output-dir") - - if storeKeys == "" { - return errors.New("stores is required") - } - storeKeysList := strings.Split(storeKeys, ",") - db, err := OpenDB(inputDir) - for _, storeKey := range storeKeysList { - prefix := []byte(fmt.Sprintf("s/k:%s/", storeKey)) - fmt.Printf("Begin exporting store with prefix %s at %s\n", prefix, time.Now().Format(time.RFC3339)) - if err := os.MkdirAll(outDir, os.ModePerm); err != nil { - return err - } - storeDir := filepath.Join(outDir, storeKey) - if err := os.MkdirAll(storeDir, os.ModePerm); err != nil { - return err - } - if err != nil { - return err - } - prefixDB := dbm.NewPrefixDB(db, prefix) - exporter := changeset.NewCSExporter(prefixDB, startVersion, endVersion, concurrency, segmentSize, storeDir) - err = exporter.Export() - if err != nil { - return err - } - - } - return nil -} - -func OpenDB(dir string) (dbm.DB, error) { - switch { - case strings.HasSuffix(dir, ".db"): - dir = dir[:len(dir)-3] - case strings.HasSuffix(dir, ".db/"): - dir = dir[:len(dir)-4] - default: - return nil, fmt.Errorf("database directory must end with .db") - } - cut := strings.LastIndex(dir, "/") - if cut == -1 { - return nil, fmt.Errorf("cannot cut paths on %s", dir) - } - name := dir[cut+1:] - db, err := dbm.NewGoLevelDB(name, dir[:cut]) - if err != nil { - return nil, err - } - return db, nil -} diff --git a/cmd/changeset/print.go b/cmd/changeset/print.go new file mode 100644 index 0000000..5ba9b41 --- /dev/null +++ b/cmd/changeset/print.go @@ -0,0 +1,36 @@ +package main + +import ( + "fmt" + "github.com/cosmos/iavl" + "github.com/cosmos/iavl/changeset" + "github.com/spf13/cobra" +) + +func PrintChangeSetCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "print", + Short: "Replay the input change set files and print all iavl changeset", + RunE: handlePrintChangeset, + } + cmd.PersistentFlags().StringP("input-file", "f", "", "Full file path of the changeset file") + return cmd +} + +func handlePrintChangeset(cmd *cobra.Command, _ []string) error { + inputFile, _ := cmd.Flags().GetString("input-file") + importer := changeset.NewImporter(inputFile). + WithProcessFn(func(version int64, cs *iavl.ChangeSet) (bool, error) { + if cs != nil { + for _, pair := range cs.Pairs { + fmt.Printf("Version: %d, delete: %t, key: %X, value: %X\n", version, pair.Delete, pair.Key, pair.Value) + } + } + return true, nil + }) + _, err := importer.Start() + if err != nil { + panic(err) + } + return nil +} From d45d7e5e2575b1213be71b22882f68caab228068 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Fri, 6 Oct 2023 12:34:13 -0700 Subject: [PATCH 40/72] Fix errors join --- changeset/reader.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/changeset/reader.go b/changeset/reader.go index 8448bad..a90cfa8 100644 --- a/changeset/reader.go +++ b/changeset/reader.go @@ -1,7 +1,6 @@ package changeset import ( - "errors" "io" ) @@ -22,10 +21,19 @@ func (r *WrapReader) ReadByte() (byte, error) { func (r *WrapReader) Close() error { var errs []error if closer, ok := r.readerCloser.(io.Closer); ok { - errs = append(errs, closer.Close()) + err := closer.Close() + if err != nil { + errs = append(errs, err) + } } if r.closer != nil { - errs = append(errs, r.closer.Close()) + err := r.closer.Close() + if err != nil { + errs = append(errs, err) + } } - return errors.Join(errs...) + if len(errs) > 0 { + return errs[0] + } + return nil } From 6564020463b47d575646fc73f5f7bbfba6ddf405 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Fri, 6 Oct 2023 12:35:05 -0700 Subject: [PATCH 41/72] Add print changeset command --- cmd/changeset/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/changeset/main.go b/cmd/changeset/main.go index a4140a6..640127d 100644 --- a/cmd/changeset/main.go +++ b/cmd/changeset/main.go @@ -16,7 +16,7 @@ func main() { rootCmd.PersistentFlags().StringP("output-dir", "o", "changeset-output", "output directory") rootCmd.AddCommand(ExportChangeSetCmd()) - rootCmd.AddCommand() + rootCmd.AddCommand(PrintChangeSetCmd()) if err := rootCmd.Execute(); err != nil { fmt.Println(err) os.Exit(1) From 38f76673c3f3282fd41d68df870c9802965247ef Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Fri, 6 Oct 2023 15:18:53 -0700 Subject: [PATCH 42/72] Add logs --- changeset/import.go | 2 ++ cmd/changeset/print.go | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/changeset/import.go b/changeset/import.go index 0b0757e..f933eda 100644 --- a/changeset/import.go +++ b/changeset/import.go @@ -85,7 +85,9 @@ func readNextChangeset(reader Reader) (int64, int64, *iavl.ChangeSet, error) { } // Read header version := binary.LittleEndian.Uint64(versionHeader[:8]) + fmt.Printf("Version: %d\n", version) size := int64(binary.LittleEndian.Uint64(versionHeader[8:16])) + fmt.Printf("Size: %d\n", size) if size <= 0 { return 16, int64(version), nil, nil } diff --git a/cmd/changeset/print.go b/cmd/changeset/print.go index 5ba9b41..4c96e87 100644 --- a/cmd/changeset/print.go +++ b/cmd/changeset/print.go @@ -14,6 +14,10 @@ func PrintChangeSetCmd() *cobra.Command { RunE: handlePrintChangeset, } cmd.PersistentFlags().StringP("input-file", "f", "", "Full file path of the changeset file") + err := cmd.MarkFlagRequired("input-file") + if err != nil { + return err + } return cmd } From 0a9c6402d3108fda5d9083a0f3df7b9b7f9001cb Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Fri, 6 Oct 2023 15:41:38 -0700 Subject: [PATCH 43/72] Add log --- cmd/changeset/print.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/changeset/print.go b/cmd/changeset/print.go index 4c96e87..cae4a52 100644 --- a/cmd/changeset/print.go +++ b/cmd/changeset/print.go @@ -16,7 +16,7 @@ func PrintChangeSetCmd() *cobra.Command { cmd.PersistentFlags().StringP("input-file", "f", "", "Full file path of the changeset file") err := cmd.MarkFlagRequired("input-file") if err != nil { - return err + panic(err) } return cmd } From febc41e0fd6a12096252a3a592be1713ff2e6d9e Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Fri, 6 Oct 2023 15:44:58 -0700 Subject: [PATCH 44/72] Add log --- cmd/changeset/print.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cmd/changeset/print.go b/cmd/changeset/print.go index cae4a52..5ba9b41 100644 --- a/cmd/changeset/print.go +++ b/cmd/changeset/print.go @@ -14,10 +14,6 @@ func PrintChangeSetCmd() *cobra.Command { RunE: handlePrintChangeset, } cmd.PersistentFlags().StringP("input-file", "f", "", "Full file path of the changeset file") - err := cmd.MarkFlagRequired("input-file") - if err != nil { - panic(err) - } return cmd } From a3ede19fba4a53b8e6d39a0d12224bc9c611c1c7 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Fri, 6 Oct 2023 16:22:18 -0700 Subject: [PATCH 45/72] Add log --- changeset/export.go | 1 - changeset/import.go | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changeset/export.go b/changeset/export.go index 578c97f..c478861 100644 --- a/changeset/export.go +++ b/changeset/export.go @@ -138,7 +138,6 @@ func WriteChangeSet(writer io.Writer, version int64, cs iavl.ChangeSet) error { var size int items := make([][]byte, 0, len(cs.Pairs)) for _, pair := range cs.Pairs { - //fmt.Printf("Version: %d, delete: %t, key: %X, value: %X\n", version, pair.Delete, pair.Key, pair.Value) buf, err := encodeKVPair(pair) if err != nil { fmt.Printf("Error: %v\n", err) diff --git a/changeset/import.go b/changeset/import.go index f933eda..d1bc705 100644 --- a/changeset/import.go +++ b/changeset/import.go @@ -114,11 +114,13 @@ func readKVPair(reader Reader) (*iavl.KVPair, error) { if err != nil { return nil, err } + fmt.Printf("deletion: %d\n", deletion) keyLen, err := binary.ReadUvarint(reader) if err != nil { return nil, err } + fmt.Printf("keyLen: %d\n", keyLen) pair := iavl.KVPair{ Delete: deletion == 1, From b459153079617f1621cd58eebd42f0d3bb796867 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Fri, 6 Oct 2023 16:40:53 -0700 Subject: [PATCH 46/72] Add log --- changeset/import.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/changeset/import.go b/changeset/import.go index d1bc705..35d5d89 100644 --- a/changeset/import.go +++ b/changeset/import.go @@ -99,6 +99,7 @@ func readNextChangeset(reader Reader) (int64, int64, *iavl.ChangeSet, error) { return 0, 0, nil, err } offset += int64(encodedSizeOfKVPair(pair)) + fmt.Printf("offset: %d\n", offset) changeset.Pairs = append(changeset.Pairs, pair) } if offset != size { @@ -138,6 +139,9 @@ func readKVPair(reader Reader) (*iavl.KVPair, error) { if err != nil { return nil, err } + + fmt.Printf("valueLen: %d\n", valueLen) + pair.Value = make([]byte, valueLen) if _, err := io.ReadFull(reader, pair.Value); err != nil { return nil, err From 677f6644ca11f7845bd088151b77db1e280c67c6 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Fri, 6 Oct 2023 17:04:05 -0700 Subject: [PATCH 47/72] Add log --- changeset/export.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/changeset/export.go b/changeset/export.go index c478861..7eebcb7 100644 --- a/changeset/export.go +++ b/changeset/export.go @@ -135,6 +135,7 @@ func WriteChangeSet(writer io.Writer, version int64, cs iavl.ChangeSet) error { return nil } + fmt.Printf("Version: %d\n", version) var size int items := make([][]byte, 0, len(cs.Pairs)) for _, pair := range cs.Pairs { @@ -152,6 +153,8 @@ func WriteChangeSet(writer io.Writer, version int64, cs iavl.ChangeSet) error { binary.LittleEndian.PutUint64(versionHeader[:], uint64(version)) binary.LittleEndian.PutUint64(versionHeader[8:], uint64(size)) + fmt.Printf("Total Size: %d\n", size) + if _, err := writer.Write(versionHeader[:]); err != nil { fmt.Printf("Error: %v\n", err) return err @@ -170,20 +173,29 @@ func WriteChangeSet(writer io.Writer, version int64, cs iavl.ChangeSet) error { // returns error if key/value length overflows. func encodeKVPair(pair *iavl.KVPair) ([]byte, error) { buf := make([]byte, encodedSizeOfKVPair(pair)) + fmt.Printf("encodedSizeOfKVPair: %d\n", encodedSizeOfKVPair(pair)) offset := 1 keyLen := len(pair.Key) + fmt.Printf("keyLen: %d\n", keyLen) + offset += binary.PutUvarint(buf[offset:], uint64(keyLen)) copy(buf[offset:], pair.Key) if pair.Delete { buf[0] = 1 return buf, nil + } else { + buf[0] = 0 } offset += keyLen offset += binary.PutUvarint(buf[offset:], uint64(len(pair.Value))) copy(buf[offset:], pair.Value) + + valueLen := len(pair.Value) + fmt.Printf("valueLen: %d\n", valueLen) + return buf, nil } From 86228a9cd459e6b75eecd4457e1f57cedcb6c1ee Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Fri, 6 Oct 2023 17:33:12 -0700 Subject: [PATCH 48/72] Add log --- changeset/export.go | 14 +++++++++----- cmd/changeset/print.go | 4 ++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/changeset/export.go b/changeset/export.go index 7eebcb7..0f49cb0 100644 --- a/changeset/export.go +++ b/changeset/export.go @@ -177,9 +177,11 @@ func encodeKVPair(pair *iavl.KVPair) ([]byte, error) { offset := 1 keyLen := len(pair.Key) - fmt.Printf("keyLen: %d\n", keyLen) + fmt.Printf("keyLen: %d, offset: %d\n", keyLen, offset) - offset += binary.PutUvarint(buf[offset:], uint64(keyLen)) + written := binary.PutUvarint(buf[offset:], uint64(keyLen)) + offset += written + fmt.Printf("offset: %d\n", written, offset) copy(buf[offset:], pair.Key) if pair.Delete { @@ -190,11 +192,13 @@ func encodeKVPair(pair *iavl.KVPair) ([]byte, error) { } offset += keyLen - offset += binary.PutUvarint(buf[offset:], uint64(len(pair.Value))) - copy(buf[offset:], pair.Value) + fmt.Printf("offset: %d\n", offset) valueLen := len(pair.Value) - fmt.Printf("valueLen: %d\n", valueLen) + offset += binary.PutUvarint(buf[offset:], uint64(valueLen)) + copy(buf[offset:], pair.Value) + + fmt.Printf("valueLen: %d, offset: %d\n", valueLen, offset) return buf, nil } diff --git a/cmd/changeset/print.go b/cmd/changeset/print.go index 5ba9b41..a0c6d34 100644 --- a/cmd/changeset/print.go +++ b/cmd/changeset/print.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "github.com/cosmos/iavl" "github.com/cosmos/iavl/changeset" @@ -19,6 +20,9 @@ func PrintChangeSetCmd() *cobra.Command { func handlePrintChangeset(cmd *cobra.Command, _ []string) error { inputFile, _ := cmd.Flags().GetString("input-file") + if inputFile == "" { + return errors.New("stores is required") + } importer := changeset.NewImporter(inputFile). WithProcessFn(func(version int64, cs *iavl.ChangeSet) (bool, error) { if cs != nil { From b85117d765c27d4820691f5db253e9e7db43542b Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Fri, 6 Oct 2023 17:37:39 -0700 Subject: [PATCH 49/72] Add log --- changeset/export.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/changeset/export.go b/changeset/export.go index 0f49cb0..c00cea6 100644 --- a/changeset/export.go +++ b/changeset/export.go @@ -181,7 +181,7 @@ func encodeKVPair(pair *iavl.KVPair) ([]byte, error) { written := binary.PutUvarint(buf[offset:], uint64(keyLen)) offset += written - fmt.Printf("offset: %d\n", written, offset) + fmt.Printf("written: %d, offset: %d\n", written, offset) copy(buf[offset:], pair.Key) if pair.Delete { @@ -195,10 +195,13 @@ func encodeKVPair(pair *iavl.KVPair) ([]byte, error) { fmt.Printf("offset: %d\n", offset) valueLen := len(pair.Value) - offset += binary.PutUvarint(buf[offset:], uint64(valueLen)) - copy(buf[offset:], pair.Value) + written = binary.PutUvarint(buf[offset:], uint64(valueLen)) + fmt.Printf("written: %d, offset: %d\n", written, offset) - fmt.Printf("valueLen: %d, offset: %d\n", valueLen, offset) + offset += written + copy(buf[offset:], pair.Value) + offset += valueLen + fmt.Printf("offset: %d\n", offset) return buf, nil } From 95c9f750ac16854480e911e5bd122c5016061987 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Fri, 6 Oct 2023 17:53:52 -0700 Subject: [PATCH 50/72] Add log --- changeset/export.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/changeset/export.go b/changeset/export.go index c00cea6..046504f 100644 --- a/changeset/export.go +++ b/changeset/export.go @@ -177,7 +177,7 @@ func encodeKVPair(pair *iavl.KVPair) ([]byte, error) { offset := 1 keyLen := len(pair.Key) - fmt.Printf("keyLen: %d, offset: %d\n", keyLen, offset) + fmt.Printf("offset: %d\n", offset) written := binary.PutUvarint(buf[offset:], uint64(keyLen)) offset += written @@ -192,16 +192,16 @@ func encodeKVPair(pair *iavl.KVPair) ([]byte, error) { } offset += keyLen - fmt.Printf("offset: %d\n", offset) + fmt.Printf("keyLen: %d, offset: %d\n", keyLen, offset) valueLen := len(pair.Value) written = binary.PutUvarint(buf[offset:], uint64(valueLen)) + offset += written fmt.Printf("written: %d, offset: %d\n", written, offset) - offset += written copy(buf[offset:], pair.Value) offset += valueLen - fmt.Printf("offset: %d\n", offset) + fmt.Printf("valueLen: %d, offset: %d\n", valueLen, offset) return buf, nil } From 5d10184bb46e321c64f318fe458bafdba48ab628 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Fri, 6 Oct 2023 18:08:36 -0700 Subject: [PATCH 51/72] Add log --- changeset/import.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changeset/import.go b/changeset/import.go index 35d5d89..beb86f2 100644 --- a/changeset/import.go +++ b/changeset/import.go @@ -131,6 +131,8 @@ func readKVPair(reader Reader) (*iavl.KVPair, error) { return nil, err } + fmt.Printf("key: %s\n", pair.Key) + if pair.Delete { return &pair, nil } From 11ab8fe00ba92b2a243a623fd0af02c7d51048d2 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Fri, 6 Oct 2023 18:09:48 -0700 Subject: [PATCH 52/72] Add log --- changeset/export.go | 1 + changeset/import.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/changeset/export.go b/changeset/export.go index 046504f..9363817 100644 --- a/changeset/export.go +++ b/changeset/export.go @@ -190,6 +190,7 @@ func encodeKVPair(pair *iavl.KVPair) ([]byte, error) { } else { buf[0] = 0 } + fmt.Printf("key: %X\n", pair.Key) offset += keyLen fmt.Printf("keyLen: %d, offset: %d\n", keyLen, offset) diff --git a/changeset/import.go b/changeset/import.go index beb86f2..6f46d56 100644 --- a/changeset/import.go +++ b/changeset/import.go @@ -131,8 +131,8 @@ func readKVPair(reader Reader) (*iavl.KVPair, error) { return nil, err } - fmt.Printf("key: %s\n", pair.Key) - + fmt.Printf("key: %X\n", pair.Key) + if pair.Delete { return &pair, nil } From e60042768a3656cba28966541fdde4f6d85625e7 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Fri, 6 Oct 2023 18:23:34 -0700 Subject: [PATCH 53/72] Add log --- changeset/import.go | 4 +--- changeset/reader.go | 5 +++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/changeset/import.go b/changeset/import.go index 6f46d56..a302610 100644 --- a/changeset/import.go +++ b/changeset/import.go @@ -1,7 +1,6 @@ package changeset import ( - "bufio" "encoding/binary" "fmt" "github.com/DataDog/zstd" @@ -51,8 +50,7 @@ func openChangesetFile(fileName string) (Reader, error) { return nil, err } zstdReaderCloser := zstd.NewReader(fp) - bufReader := bufio.NewReader(zstdReaderCloser) - return &WrapReader{zstdReaderCloser, bufReader, fp}, nil + return &WrapReader{zstdReaderCloser, fp}, nil } func iterateChangeSet(reader Reader, fn func(version int64, changeset *iavl.ChangeSet) (bool, error)) (int64, error) { diff --git a/changeset/reader.go b/changeset/reader.go index a90cfa8..7be62f8 100644 --- a/changeset/reader.go +++ b/changeset/reader.go @@ -6,7 +6,6 @@ import ( type WrapReader struct { readerCloser io.ReadCloser - byteReader io.ByteReader closer io.Closer } @@ -15,7 +14,9 @@ func (r *WrapReader) Read(p []byte) (int, error) { } func (r *WrapReader) ReadByte() (byte, error) { - return r.byteReader.ReadByte() + var singleByte [1]byte + _, err := r.readerCloser.Read(singleByte[:]) + return singleByte[0], err } func (r *WrapReader) Close() error { From 156e7ebdd40e4f4e3a6dfd08065c3d88b63b7db3 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Fri, 6 Oct 2023 18:40:49 -0700 Subject: [PATCH 54/72] Add log --- changeset/export.go | 22 ++++++++-------------- changeset/import.go | 19 +++++++++++-------- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/changeset/export.go b/changeset/export.go index 9363817..0ae459b 100644 --- a/changeset/export.go +++ b/changeset/export.go @@ -5,6 +5,7 @@ import ( "encoding/binary" "fmt" "io" + "math" "math/bits" "os" "path/filepath" @@ -122,6 +123,12 @@ func dumpChangesetSegment(outputDir string, tree *iavl.ImmutableTree, start int6 fmt.Printf("Error: %v\n", err) return err } + var endingHeader [16]byte + binary.LittleEndian.PutUint64(endingHeader[:], math.MaxUint64) + binary.LittleEndian.PutUint64(endingHeader[8:], uint64(0)) + if _, err := zstdWriter.Write(endingHeader[:]); err != nil { + return err + } return zstdWriter.Flush() } @@ -135,13 +142,12 @@ func WriteChangeSet(writer io.Writer, version int64, cs iavl.ChangeSet) error { return nil } - fmt.Printf("Version: %d\n", version) + fmt.Printf("Writing Version: %d\n", version) var size int items := make([][]byte, 0, len(cs.Pairs)) for _, pair := range cs.Pairs { buf, err := encodeKVPair(pair) if err != nil { - fmt.Printf("Error: %v\n", err) return err } size += len(buf) @@ -153,15 +159,11 @@ func WriteChangeSet(writer io.Writer, version int64, cs iavl.ChangeSet) error { binary.LittleEndian.PutUint64(versionHeader[:], uint64(version)) binary.LittleEndian.PutUint64(versionHeader[8:], uint64(size)) - fmt.Printf("Total Size: %d\n", size) - if _, err := writer.Write(versionHeader[:]); err != nil { - fmt.Printf("Error: %v\n", err) return err } for _, item := range items { if _, err := writer.Write(item); err != nil { - fmt.Printf("Error: %v\n", err) return err } } @@ -173,15 +175,12 @@ func WriteChangeSet(writer io.Writer, version int64, cs iavl.ChangeSet) error { // returns error if key/value length overflows. func encodeKVPair(pair *iavl.KVPair) ([]byte, error) { buf := make([]byte, encodedSizeOfKVPair(pair)) - fmt.Printf("encodedSizeOfKVPair: %d\n", encodedSizeOfKVPair(pair)) offset := 1 keyLen := len(pair.Key) - fmt.Printf("offset: %d\n", offset) written := binary.PutUvarint(buf[offset:], uint64(keyLen)) offset += written - fmt.Printf("written: %d, offset: %d\n", written, offset) copy(buf[offset:], pair.Key) if pair.Delete { @@ -190,19 +189,14 @@ func encodeKVPair(pair *iavl.KVPair) ([]byte, error) { } else { buf[0] = 0 } - fmt.Printf("key: %X\n", pair.Key) offset += keyLen - fmt.Printf("keyLen: %d, offset: %d\n", keyLen, offset) valueLen := len(pair.Value) written = binary.PutUvarint(buf[offset:], uint64(valueLen)) offset += written - fmt.Printf("written: %d, offset: %d\n", written, offset) copy(buf[offset:], pair.Value) - offset += valueLen - fmt.Printf("valueLen: %d, offset: %d\n", valueLen, offset) return buf, nil } diff --git a/changeset/import.go b/changeset/import.go index a302610..6512d49 100644 --- a/changeset/import.go +++ b/changeset/import.go @@ -6,6 +6,7 @@ import ( "github.com/DataDog/zstd" "github.com/cosmos/iavl" "io" + "math" "os" ) @@ -58,12 +59,12 @@ func iterateChangeSet(reader Reader, fn func(version int64, changeset *iavl.Chan for true { offset, version, changeSet, err := readNextChangeset(reader) if err != nil { - if err == io.EOF { - // Reaching here means we are done reading everything in the file - break - } else { - return lastOffset, err - } + return lastOffset, err + } + + if offset == -1 || version == -1 { + // this means we are done + break } shouldStop, err := fn(version, changeSet) lastOffset += offset @@ -83,9 +84,11 @@ func readNextChangeset(reader Reader) (int64, int64, *iavl.ChangeSet, error) { } // Read header version := binary.LittleEndian.Uint64(versionHeader[:8]) - fmt.Printf("Version: %d\n", version) + fmt.Printf("Reading Version: %d\n", version) + if version == math.MaxUint64 { + return -1, -1, nil, nil + } size := int64(binary.LittleEndian.Uint64(versionHeader[8:16])) - fmt.Printf("Size: %d\n", size) if size <= 0 { return 16, int64(version), nil, nil } From a6b97882fe7e23b6a315ea480adfe09f42585d7c Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Sat, 7 Oct 2023 07:39:48 -0700 Subject: [PATCH 55/72] Add chunk file support --- changeset/export.go | 109 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 87 insertions(+), 22 deletions(-) diff --git a/changeset/export.go b/changeset/export.go index 0ae459b..ee94f35 100644 --- a/changeset/export.go +++ b/changeset/export.go @@ -1,6 +1,7 @@ package changeset import ( + "bufio" "context" "encoding/binary" "fmt" @@ -10,6 +11,7 @@ import ( "os" "path/filepath" "sync" + "time" "github.com/alitto/pond" "github.com/cosmos/iavl" @@ -18,7 +20,7 @@ import ( ) const ( - DefaultCacheSize int = 1000000 + DefaultCacheSize int = 10000000 ) type Exporter struct { @@ -50,7 +52,7 @@ func NewExporter( func (exporter *Exporter) Start() error { // use a worker pool with enough buffer to parallelize the export - pool := pond.New(exporter.concurrency, 1024) + pool := pond.New(exporter.concurrency, 10240) defer pool.StopAndWait() // share the iavl tree between tasks to reuse the node cache @@ -63,40 +65,45 @@ func (exporter *Exporter) Start() error { // split into segments var segmentSize = exporter.segmentSize - var groups []*pond.TaskGroupWithContext for i := exporter.start; i < exporter.end; i += segmentSize { end := i + segmentSize if end > exporter.end { end = exporter.end } + var chunkFiles []string group, _ := pool.GroupContext(context.Background()) - startPos := i - endPos := end - group.Submit(func() error { - tree := iavlTreePool.Get().(*iavl.ImmutableTree) - defer iavlTreePool.Put(tree) - err := dumpChangesetSegment(exporter.outputDir, tree, startPos, endPos) - fmt.Printf("Finished exporting segment %d-%d\n", startPos, endPos) - return err - }) - groups = append(groups, group) - } + fmt.Printf("Start exporting segment %d-%d at %s\n", i, end, time.Now().Format(time.RFC3339)) + for _, workRange := range splitWorkLoad(exporter.concurrency, Range{i, end}) { + workRange := workRange + chunkFile := filepath.Join(exporter.outputDir, fmt.Sprintf("tmp-chunk-%d-%d.zst", workRange.Start, workRange.End)) + group.Submit(func() error { + tree := iavlTreePool.Get().(*iavl.ImmutableTree) + defer iavlTreePool.Put(tree) + fmt.Printf("Start exporting chunk %s at %s\n", chunkFile, time.Now().Format(time.RFC3339)) + err := writeChangesetChunks(chunkFile, tree, workRange.Start, workRange.End) + fmt.Printf("Finished exporting chunk %s at %s\n", chunkFile, time.Now().Format(time.RFC3339)) + return err + }) + chunkFiles = append(chunkFiles, chunkFile) + } - // wait for all task groups to complete - for _, group := range groups { if err := group.Wait(); err != nil { - fmt.Printf("Error: %v\n", err) return err } + err := collectChunksToSegment(exporter.outputDir, chunkFiles) + if err != nil { + return err + } + fmt.Printf("Finished exporting segment %d-%d at %s\n", i, end, time.Now().Format(time.RFC3339)) } + return nil } -func dumpChangesetSegment(outputDir string, tree *iavl.ImmutableTree, start int64, end int64) (returnErr error) { +func writeChangesetChunks(chunkFilePath string, tree *iavl.ImmutableTree, start int64, end int64) (returnErr error) { fmt.Printf("Exporting changeset segment %d-%d\n", start, end) - segmentFilePath := filepath.Join(outputDir, fmt.Sprintf("changeset-%d-%d.zst", start, end)) - segmentFile, err := createFile(segmentFilePath) - zstdWriter, err := zstd.NewWriter(segmentFile) + chunkFile, err := createFile(chunkFilePath) + zstdWriter, err := zstd.NewWriter(chunkFile) if err != nil { fmt.Printf("Error: %v\n", err) @@ -108,7 +115,7 @@ func dumpChangesetSegment(outputDir string, tree *iavl.ImmutableTree, start int6 if err != nil { returnErr = err } - if err := segmentFile.Close(); returnErr == nil { + if err := chunkFile.Close(); returnErr == nil { returnErr = err } }() @@ -133,6 +140,47 @@ func dumpChangesetSegment(outputDir string, tree *iavl.ImmutableTree, start int6 return zstdWriter.Flush() } +func collectChunksToSegment(outputFile string, chunkFiles []string) error { + fp, err := createFile(outputFile) + if err != nil { + return err + } + defer fp.Close() + + bufWriter := bufio.NewWriter(fp) + writer, _ := zstd.NewWriter(bufWriter) + + for _, chunkFile := range chunkFiles { + if err := copyTmpFile(writer, chunkFile); err != nil { + return err + } + if err := os.Remove(chunkFile); err != nil { + return err + } + } + + if writer != nil { + if err := writer.Close(); err != nil { + return err + } + } + + return bufWriter.Flush() +} + +// copyTmpFile append the compressed temporary file to writer +func copyTmpFile(writer io.Writer, tmpFile string) error { + fp, err := os.Open(tmpFile) + if err != nil { + return err + } + defer fp.Close() + + reader, _ := zstd.NewReader(fp) + _, err = io.Copy(writer, reader) + return err +} + func createFile(name string) (*os.File, error) { return os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) } @@ -170,6 +218,23 @@ func WriteChangeSet(writer io.Writer, version int64, cs iavl.ChangeSet) error { return nil } +type Range struct { + Start, End int64 +} + +func splitWorkLoad(workers int, full Range) []Range { + var ranges []Range + chunkSize := (full.End - full.Start + int64(workers) - 1) / int64(workers) + for i := full.Start; i < full.End; i += chunkSize { + end := i + chunkSize + if end > full.End { + end = full.End + } + ranges = append(ranges, Range{Start: i, End: end}) + } + return ranges +} + // encodeKVPair encode a key-value pair in change set. // see godoc of `encodedSizeOfKVPair` for layout description, // returns error if key/value length overflows. From ab545cd8ddc769ecf9562dd9a7c5e5d59a3ec02b Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Sat, 7 Oct 2023 07:43:09 -0700 Subject: [PATCH 56/72] Add chunk file support --- changeset/export.go | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/changeset/export.go b/changeset/export.go index ee94f35..fb09186 100644 --- a/changeset/export.go +++ b/changeset/export.go @@ -130,12 +130,6 @@ func writeChangesetChunks(chunkFilePath string, tree *iavl.ImmutableTree, start fmt.Printf("Error: %v\n", err) return err } - var endingHeader [16]byte - binary.LittleEndian.PutUint64(endingHeader[:], math.MaxUint64) - binary.LittleEndian.PutUint64(endingHeader[8:], uint64(0)) - if _, err := zstdWriter.Write(endingHeader[:]); err != nil { - return err - } return zstdWriter.Flush() } @@ -145,11 +139,15 @@ func collectChunksToSegment(outputFile string, chunkFiles []string) error { if err != nil { return err } - defer fp.Close() bufWriter := bufio.NewWriter(fp) writer, _ := zstd.NewWriter(bufWriter) + defer func() { + fp.Close() + writer.Close() + }() + for _, chunkFile := range chunkFiles { if err := copyTmpFile(writer, chunkFile); err != nil { return err @@ -159,10 +157,11 @@ func collectChunksToSegment(outputFile string, chunkFiles []string) error { } } - if writer != nil { - if err := writer.Close(); err != nil { - return err - } + var endingHeader [16]byte + binary.LittleEndian.PutUint64(endingHeader[:], math.MaxUint64) + binary.LittleEndian.PutUint64(endingHeader[8:], uint64(0)) + if _, err := bufWriter.Write(endingHeader[:]); err != nil { + return err } return bufWriter.Flush() From a7fe73c8d16bf2412a8383ae44ba97be39131acf Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Sat, 7 Oct 2023 07:49:31 -0700 Subject: [PATCH 57/72] Add chunk file support --- changeset/export.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/changeset/export.go b/changeset/export.go index fb09186..75f1d89 100644 --- a/changeset/export.go +++ b/changeset/export.go @@ -90,7 +90,8 @@ func (exporter *Exporter) Start() error { if err := group.Wait(); err != nil { return err } - err := collectChunksToSegment(exporter.outputDir, chunkFiles) + segmentFile := filepath.Join(exporter.outputDir, fmt.Sprintf("changeset-%d-%d.zst", i, end)) + err := collectChunksToSegment(segmentFile, chunkFiles) if err != nil { return err } From 352cdf1870107a08429091dbcd55a71732c14b82 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Sat, 7 Oct 2023 09:54:49 -0700 Subject: [PATCH 58/72] Fix logging --- changeset/export.go | 2 +- changeset/import.go | 10 +--------- cmd/changeset/print.go | 2 +- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/changeset/export.go b/changeset/export.go index 75f1d89..3d20372 100644 --- a/changeset/export.go +++ b/changeset/export.go @@ -102,7 +102,7 @@ func (exporter *Exporter) Start() error { } func writeChangesetChunks(chunkFilePath string, tree *iavl.ImmutableTree, start int64, end int64) (returnErr error) { - fmt.Printf("Exporting changeset segment %d-%d\n", start, end) + fmt.Printf("Exporting changeset chunk %d-%d\n", start, end) chunkFile, err := createFile(chunkFilePath) zstdWriter, err := zstd.NewWriter(chunkFile) diff --git a/changeset/import.go b/changeset/import.go index 6512d49..a045e54 100644 --- a/changeset/import.go +++ b/changeset/import.go @@ -84,7 +84,6 @@ func readNextChangeset(reader Reader) (int64, int64, *iavl.ChangeSet, error) { } // Read header version := binary.LittleEndian.Uint64(versionHeader[:8]) - fmt.Printf("Reading Version: %d\n", version) if version == math.MaxUint64 { return -1, -1, nil, nil } @@ -100,7 +99,6 @@ func readNextChangeset(reader Reader) (int64, int64, *iavl.ChangeSet, error) { return 0, 0, nil, err } offset += int64(encodedSizeOfKVPair(pair)) - fmt.Printf("offset: %d\n", offset) changeset.Pairs = append(changeset.Pairs, pair) } if offset != size { @@ -116,13 +114,11 @@ func readKVPair(reader Reader) (*iavl.KVPair, error) { if err != nil { return nil, err } - fmt.Printf("deletion: %d\n", deletion) keyLen, err := binary.ReadUvarint(reader) if err != nil { return nil, err } - fmt.Printf("keyLen: %d\n", keyLen) pair := iavl.KVPair{ Delete: deletion == 1, @@ -131,9 +127,7 @@ func readKVPair(reader Reader) (*iavl.KVPair, error) { if _, err := io.ReadFull(reader, pair.Key); err != nil { return nil, err } - - fmt.Printf("key: %X\n", pair.Key) - + if pair.Delete { return &pair, nil } @@ -143,8 +137,6 @@ func readKVPair(reader Reader) (*iavl.KVPair, error) { return nil, err } - fmt.Printf("valueLen: %d\n", valueLen) - pair.Value = make([]byte, valueLen) if _, err := io.ReadFull(reader, pair.Value); err != nil { return nil, err diff --git a/cmd/changeset/print.go b/cmd/changeset/print.go index a0c6d34..c4e3c71 100644 --- a/cmd/changeset/print.go +++ b/cmd/changeset/print.go @@ -21,7 +21,7 @@ func PrintChangeSetCmd() *cobra.Command { func handlePrintChangeset(cmd *cobra.Command, _ []string) error { inputFile, _ := cmd.Flags().GetString("input-file") if inputFile == "" { - return errors.New("stores is required") + return errors.New("input-file is required") } importer := changeset.NewImporter(inputFile). WithProcessFn(func(version int64, cs *iavl.ChangeSet) (bool, error) { From e032e126a38250a11adc8fc7730bcb0139fbb69b Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Tue, 10 Oct 2023 12:12:12 -0700 Subject: [PATCH 59/72] Fix wrap reader --- changeset/export.go | 1 - changeset/import.go | 4 ++-- changeset/reader.go | 8 ++++---- cmd/changeset/print.go | 15 +++++++-------- go.mod | 2 +- 5 files changed, 14 insertions(+), 16 deletions(-) diff --git a/changeset/export.go b/changeset/export.go index 3d20372..2fbfe5f 100644 --- a/changeset/export.go +++ b/changeset/export.go @@ -190,7 +190,6 @@ func WriteChangeSet(writer io.Writer, version int64, cs iavl.ChangeSet) error { return nil } - fmt.Printf("Writing Version: %d\n", version) var size int items := make([][]byte, 0, len(cs.Pairs)) for _, pair := range cs.Pairs { diff --git a/changeset/import.go b/changeset/import.go index a045e54..53be7f3 100644 --- a/changeset/import.go +++ b/changeset/import.go @@ -51,7 +51,7 @@ func openChangesetFile(fileName string) (Reader, error) { return nil, err } zstdReaderCloser := zstd.NewReader(fp) - return &WrapReader{zstdReaderCloser, fp}, nil + return &WrappedReader{zstdReaderCloser, fp}, nil } func iterateChangeSet(reader Reader, fn func(version int64, changeset *iavl.ChangeSet) (bool, error)) (int64, error) { @@ -127,7 +127,7 @@ func readKVPair(reader Reader) (*iavl.KVPair, error) { if _, err := io.ReadFull(reader, pair.Key); err != nil { return nil, err } - + if pair.Delete { return &pair, nil } diff --git a/changeset/reader.go b/changeset/reader.go index 7be62f8..e15572b 100644 --- a/changeset/reader.go +++ b/changeset/reader.go @@ -4,22 +4,22 @@ import ( "io" ) -type WrapReader struct { +type WrappedReader struct { readerCloser io.ReadCloser closer io.Closer } -func (r *WrapReader) Read(p []byte) (int, error) { +func (r *WrappedReader) Read(p []byte) (int, error) { return r.readerCloser.Read(p) } -func (r *WrapReader) ReadByte() (byte, error) { +func (r *WrappedReader) ReadByte() (byte, error) { var singleByte [1]byte _, err := r.readerCloser.Read(singleByte[:]) return singleByte[0], err } -func (r *WrapReader) Close() error { +func (r *WrappedReader) Close() error { var errs []error if closer, ok := r.readerCloser.(io.Closer); ok { err := closer.Close() diff --git a/cmd/changeset/print.go b/cmd/changeset/print.go index c4e3c71..7b255b3 100644 --- a/cmd/changeset/print.go +++ b/cmd/changeset/print.go @@ -23,15 +23,14 @@ func handlePrintChangeset(cmd *cobra.Command, _ []string) error { if inputFile == "" { return errors.New("input-file is required") } - importer := changeset.NewImporter(inputFile). - WithProcessFn(func(version int64, cs *iavl.ChangeSet) (bool, error) { - if cs != nil { - for _, pair := range cs.Pairs { - fmt.Printf("Version: %d, delete: %t, key: %X, value: %X\n", version, pair.Delete, pair.Key, pair.Value) - } + importer := changeset.NewImporter(inputFile).WithProcessFn(func(version int64, cs *iavl.ChangeSet) (bool, error) { + if cs != nil { + for _, pair := range cs.Pairs { + fmt.Printf("Version: %d, delete: %t, key: %X, value: %X\n", version, pair.Delete, pair.Key, pair.Value) } - return true, nil - }) + } + return true, nil + }) _, err := importer.Start() if err != nil { panic(err) diff --git a/go.mod b/go.mod index 2363ed9..8f59437 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/cosmos/iavl go 1.18 require ( + github.com/DataDog/zstd v1.4.5 github.com/alitto/pond v1.8.3 github.com/confio/ics23/go v0.7.0 github.com/gogo/protobuf v1.3.2 @@ -17,7 +18,6 @@ require ( ) require ( - github.com/DataDog/zstd v1.4.5 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgraph-io/badger/v2 v2.2007.2 // indirect From 18e1d87f0582e068bb0ca319de9d1d960aac7827 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Tue, 10 Oct 2023 14:16:10 -0700 Subject: [PATCH 60/72] Fix go mod --- coverage.txt | 109 +++++++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- 3 files changed, 112 insertions(+), 3 deletions(-) create mode 100644 coverage.txt diff --git a/coverage.txt b/coverage.txt new file mode 100644 index 0000000..7b11066 --- /dev/null +++ b/coverage.txt @@ -0,0 +1,109 @@ +mode: atomic +github.com/cosmos/iavl/cache/cache.go:55.37,61.2 1 11 +github.com/cosmos/iavl/cache/cache.go:63.40,65.41 2 19 +github.com/cosmos/iavl/cache/cache.go:65.41,70.3 4 1 +github.com/cosmos/iavl/cache/cache.go:72.2,75.36 3 18 +github.com/cosmos/iavl/cache/cache.go:75.36,78.3 2 4 +github.com/cosmos/iavl/cache/cache.go:79.2,79.12 1 14 +github.com/cosmos/iavl/cache/cache.go:82.42,83.60 1 9 +github.com/cosmos/iavl/cache/cache.go:83.60,86.3 2 9 +github.com/cosmos/iavl/cache/cache.go:87.2,87.12 1 0 +github.com/cosmos/iavl/cache/cache.go:90.41,93.2 2 9 +github.com/cosmos/iavl/cache/cache.go:95.31,97.2 1 41 +github.com/cosmos/iavl/cache/cache.go:99.44,100.66 1 8 +github.com/cosmos/iavl/cache/cache.go:100.66,102.3 1 5 +github.com/cosmos/iavl/cache/cache.go:103.2,103.12 1 3 +github.com/cosmos/iavl/cache/cache.go:106.49,110.2 3 9 +github.com/cosmos/iavl/internal/encoding/encoding.go:14.26,16.3 1 0 +github.com/cosmos/iavl/internal/encoding/encoding.go:20.26,22.3 1 0 +github.com/cosmos/iavl/internal/encoding/encoding.go:26.26,28.3 1 0 +github.com/cosmos/iavl/internal/encoding/encoding.go:33.50,35.16 2 35 +github.com/cosmos/iavl/internal/encoding/encoding.go:35.16,37.3 1 1 +github.com/cosmos/iavl/internal/encoding/encoding.go:41.2,42.42 2 34 +github.com/cosmos/iavl/internal/encoding/encoding.go:42.42,44.3 1 6 +github.com/cosmos/iavl/internal/encoding/encoding.go:46.2,47.13 2 28 +github.com/cosmos/iavl/internal/encoding/encoding.go:47.13,49.3 1 1 +github.com/cosmos/iavl/internal/encoding/encoding.go:51.2,51.19 1 27 +github.com/cosmos/iavl/internal/encoding/encoding.go:51.19,53.3 1 23 +github.com/cosmos/iavl/internal/encoding/encoding.go:54.2,56.22 3 4 +github.com/cosmos/iavl/internal/encoding/encoding.go:61.52,63.12 2 35 +github.com/cosmos/iavl/internal/encoding/encoding.go:63.12,66.3 1 1 +github.com/cosmos/iavl/internal/encoding/encoding.go:66.8,66.18 1 34 +github.com/cosmos/iavl/internal/encoding/encoding.go:66.18,71.3 2 0 +github.com/cosmos/iavl/internal/encoding/encoding.go:72.2,72.18 1 34 +github.com/cosmos/iavl/internal/encoding/encoding.go:77.50,79.12 2 0 +github.com/cosmos/iavl/internal/encoding/encoding.go:79.12,81.3 1 0 +github.com/cosmos/iavl/internal/encoding/encoding.go:81.8,81.18 1 0 +github.com/cosmos/iavl/internal/encoding/encoding.go:81.18,86.3 2 0 +github.com/cosmos/iavl/internal/encoding/encoding.go:87.2,87.18 1 0 +github.com/cosmos/iavl/internal/encoding/encoding.go:91.48,93.16 2 0 +github.com/cosmos/iavl/internal/encoding/encoding.go:93.16,95.3 1 0 +github.com/cosmos/iavl/internal/encoding/encoding.go:96.2,97.12 2 0 +github.com/cosmos/iavl/internal/encoding/encoding.go:101.50,112.2 7 0 +github.com/cosmos/iavl/internal/encoding/encoding.go:115.37,117.2 1 0 +github.com/cosmos/iavl/internal/encoding/encoding.go:120.49,130.2 5 0 +github.com/cosmos/iavl/internal/encoding/encoding.go:133.38,134.12 1 0 +github.com/cosmos/iavl/internal/encoding/encoding.go:134.12,136.3 1 0 +github.com/cosmos/iavl/internal/encoding/encoding.go:137.2,137.32 1 0 +github.com/cosmos/iavl/internal/encoding/encoding.go:141.47,158.2 5 0 +github.com/cosmos/iavl/internal/encoding/encoding.go:161.36,164.2 2 0 +github.com/cosmos/iavl/internal/bytes/bytes.go:13.46,15.2 1 1 +github.com/cosmos/iavl/internal/bytes/bytes.go:18.50,21.2 2 1 +github.com/cosmos/iavl/internal/bytes/bytes.go:24.50,31.2 6 3 +github.com/cosmos/iavl/internal/bytes/bytes.go:34.54,35.65 1 3 +github.com/cosmos/iavl/internal/bytes/bytes.go:35.65,37.3 1 0 +github.com/cosmos/iavl/internal/bytes/bytes.go:38.2,41.16 4 3 +github.com/cosmos/iavl/internal/bytes/bytes.go:41.16,43.3 1 0 +github.com/cosmos/iavl/internal/bytes/bytes.go:44.2,45.12 2 3 +github.com/cosmos/iavl/internal/bytes/bytes.go:49.35,51.2 1 0 +github.com/cosmos/iavl/internal/bytes/bytes.go:53.36,55.2 1 0 +github.com/cosmos/iavl/internal/bytes/bytes.go:57.51,58.14 1 0 +github.com/cosmos/iavl/internal/bytes/bytes.go:59.11,60.41 1 0 +github.com/cosmos/iavl/internal/bytes/bytes.go:61.10,62.49 1 0 +github.com/cosmos/iavl/internal/bytes/string.go:10.40,18.2 7 5 +github.com/cosmos/iavl/internal/bytes/string.go:24.40,26.2 1 5 +github.com/cosmos/iavl/internal/rand/random.go:28.13,31.2 2 1 +github.com/cosmos/iavl/internal/rand/random.go:33.22,37.2 3 1 +github.com/cosmos/iavl/internal/rand/random.go:39.23,42.25 3 2 +github.com/cosmos/iavl/internal/rand/random.go:42.25,45.3 2 16 +github.com/cosmos/iavl/internal/rand/random.go:46.2,46.22 1 2 +github.com/cosmos/iavl/internal/rand/random.go:49.34,51.2 1 102 +github.com/cosmos/iavl/internal/rand/random.go:56.23,58.2 1 0 +github.com/cosmos/iavl/internal/rand/random.go:60.33,62.2 1 1 +github.com/cosmos/iavl/internal/rand/random.go:64.20,66.2 1 100 +github.com/cosmos/iavl/internal/rand/random.go:68.24,70.2 1 100 +github.com/cosmos/iavl/internal/rand/random.go:72.30,74.2 1 1 +github.com/cosmos/iavl/internal/rand/random.go:76.28,78.2 1 100 +github.com/cosmos/iavl/internal/rand/random.go:83.33,87.2 3 0 +github.com/cosmos/iavl/internal/rand/random.go:90.39,93.6 2 1 +github.com/cosmos/iavl/internal/rand/random.go:93.6,95.27 2 26 +github.com/cosmos/iavl/internal/rand/random.go:95.27,97.15 2 256 +github.com/cosmos/iavl/internal/rand/random.go:97.15,99.13 2 13 +github.com/cosmos/iavl/internal/rand/random.go:100.10,102.29 2 243 +github.com/cosmos/iavl/internal/rand/random.go:102.29,103.21 1 1 +github.com/cosmos/iavl/internal/rand/random.go:105.5,105.14 1 242 +github.com/cosmos/iavl/internal/rand/random.go:110.2,110.22 1 1 +github.com/cosmos/iavl/internal/rand/random.go:113.32,115.2 1 0 +github.com/cosmos/iavl/internal/rand/random.go:117.32,122.2 4 0 +github.com/cosmos/iavl/internal/rand/random.go:124.32,126.2 1 0 +github.com/cosmos/iavl/internal/rand/random.go:128.28,133.2 4 0 +github.com/cosmos/iavl/internal/rand/random.go:135.30,137.2 1 0 +github.com/cosmos/iavl/internal/rand/random.go:139.30,141.2 1 0 +github.com/cosmos/iavl/internal/rand/random.go:143.30,145.2 1 0 +github.com/cosmos/iavl/internal/rand/random.go:147.26,152.2 4 343 +github.com/cosmos/iavl/internal/rand/random.go:154.30,159.2 4 100 +github.com/cosmos/iavl/internal/rand/random.go:161.38,166.2 4 0 +github.com/cosmos/iavl/internal/rand/random.go:168.30,173.2 4 26 +github.com/cosmos/iavl/internal/rand/random.go:175.38,180.2 4 0 +github.com/cosmos/iavl/internal/rand/random.go:182.34,187.2 4 0 +github.com/cosmos/iavl/internal/rand/random.go:189.34,194.2 4 0 +github.com/cosmos/iavl/internal/rand/random.go:196.33,198.2 1 0 +github.com/cosmos/iavl/internal/rand/random.go:202.36,206.31 2 1 +github.com/cosmos/iavl/internal/rand/random.go:206.31,208.3 1 243 +github.com/cosmos/iavl/internal/rand/random.go:209.2,209.11 1 1 +github.com/cosmos/iavl/internal/rand/random.go:214.32,219.2 4 0 +github.com/cosmos/iavl/internal/rand/random.go:222.28,226.2 1 0 +github.com/cosmos/iavl/internal/rand/random.go:229.34,234.2 4 100 +github.com/cosmos/iavl/internal/rand/random.go:238.38,241.16 3 2 +github.com/cosmos/iavl/internal/rand/random.go:241.16,242.13 1 0 +github.com/cosmos/iavl/internal/rand/random.go:244.2,244.10 1 2 diff --git a/go.mod b/go.mod index 8f59437..085148e 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/cosmos/iavl go 1.18 require ( - github.com/DataDog/zstd v1.4.5 + github.com/DataDog/zstd v1.5.5 github.com/alitto/pond v1.8.3 github.com/confio/ics23/go v0.7.0 github.com/gogo/protobuf v1.3.2 diff --git a/go.sum b/go.sum index 143e6a5..0d81a9b 100644 --- a/go.sum +++ b/go.sum @@ -76,8 +76,8 @@ github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d/go.mod h1: github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= -github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= +github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs= github.com/GaijinEntertainment/go-exhaustruct/v2 v2.1.0/go.mod h1:LGOGuvEgCfCQsy3JF2tRmpGDpzA53iZfyGEWSPwQ6/4= github.com/HdrHistogram/hdrhistogram-go v1.1.0/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= From ba73621af327f61e461bb6a5ba55291529701be6 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Tue, 10 Oct 2023 14:19:02 -0700 Subject: [PATCH 61/72] Add log --- changeset/import.go | 4 +- coverage.txt | 109 -------------------------------------------- 2 files changed, 3 insertions(+), 110 deletions(-) delete mode 100644 coverage.txt diff --git a/changeset/import.go b/changeset/import.go index 53be7f3..ba28200 100644 --- a/changeset/import.go +++ b/changeset/import.go @@ -56,14 +56,16 @@ func openChangesetFile(fileName string) (Reader, error) { func iterateChangeSet(reader Reader, fn func(version int64, changeset *iavl.ChangeSet) (bool, error)) (int64, error) { var lastOffset int64 - for true { + for { offset, version, changeSet, err := readNextChangeset(reader) if err != nil { + fmt.Printf("Error: %v\n", err) return lastOffset, err } if offset == -1 || version == -1 { // this means we are done + fmt.Printf("Done\n") break } shouldStop, err := fn(version, changeSet) diff --git a/coverage.txt b/coverage.txt deleted file mode 100644 index 7b11066..0000000 --- a/coverage.txt +++ /dev/null @@ -1,109 +0,0 @@ -mode: atomic -github.com/cosmos/iavl/cache/cache.go:55.37,61.2 1 11 -github.com/cosmos/iavl/cache/cache.go:63.40,65.41 2 19 -github.com/cosmos/iavl/cache/cache.go:65.41,70.3 4 1 -github.com/cosmos/iavl/cache/cache.go:72.2,75.36 3 18 -github.com/cosmos/iavl/cache/cache.go:75.36,78.3 2 4 -github.com/cosmos/iavl/cache/cache.go:79.2,79.12 1 14 -github.com/cosmos/iavl/cache/cache.go:82.42,83.60 1 9 -github.com/cosmos/iavl/cache/cache.go:83.60,86.3 2 9 -github.com/cosmos/iavl/cache/cache.go:87.2,87.12 1 0 -github.com/cosmos/iavl/cache/cache.go:90.41,93.2 2 9 -github.com/cosmos/iavl/cache/cache.go:95.31,97.2 1 41 -github.com/cosmos/iavl/cache/cache.go:99.44,100.66 1 8 -github.com/cosmos/iavl/cache/cache.go:100.66,102.3 1 5 -github.com/cosmos/iavl/cache/cache.go:103.2,103.12 1 3 -github.com/cosmos/iavl/cache/cache.go:106.49,110.2 3 9 -github.com/cosmos/iavl/internal/encoding/encoding.go:14.26,16.3 1 0 -github.com/cosmos/iavl/internal/encoding/encoding.go:20.26,22.3 1 0 -github.com/cosmos/iavl/internal/encoding/encoding.go:26.26,28.3 1 0 -github.com/cosmos/iavl/internal/encoding/encoding.go:33.50,35.16 2 35 -github.com/cosmos/iavl/internal/encoding/encoding.go:35.16,37.3 1 1 -github.com/cosmos/iavl/internal/encoding/encoding.go:41.2,42.42 2 34 -github.com/cosmos/iavl/internal/encoding/encoding.go:42.42,44.3 1 6 -github.com/cosmos/iavl/internal/encoding/encoding.go:46.2,47.13 2 28 -github.com/cosmos/iavl/internal/encoding/encoding.go:47.13,49.3 1 1 -github.com/cosmos/iavl/internal/encoding/encoding.go:51.2,51.19 1 27 -github.com/cosmos/iavl/internal/encoding/encoding.go:51.19,53.3 1 23 -github.com/cosmos/iavl/internal/encoding/encoding.go:54.2,56.22 3 4 -github.com/cosmos/iavl/internal/encoding/encoding.go:61.52,63.12 2 35 -github.com/cosmos/iavl/internal/encoding/encoding.go:63.12,66.3 1 1 -github.com/cosmos/iavl/internal/encoding/encoding.go:66.8,66.18 1 34 -github.com/cosmos/iavl/internal/encoding/encoding.go:66.18,71.3 2 0 -github.com/cosmos/iavl/internal/encoding/encoding.go:72.2,72.18 1 34 -github.com/cosmos/iavl/internal/encoding/encoding.go:77.50,79.12 2 0 -github.com/cosmos/iavl/internal/encoding/encoding.go:79.12,81.3 1 0 -github.com/cosmos/iavl/internal/encoding/encoding.go:81.8,81.18 1 0 -github.com/cosmos/iavl/internal/encoding/encoding.go:81.18,86.3 2 0 -github.com/cosmos/iavl/internal/encoding/encoding.go:87.2,87.18 1 0 -github.com/cosmos/iavl/internal/encoding/encoding.go:91.48,93.16 2 0 -github.com/cosmos/iavl/internal/encoding/encoding.go:93.16,95.3 1 0 -github.com/cosmos/iavl/internal/encoding/encoding.go:96.2,97.12 2 0 -github.com/cosmos/iavl/internal/encoding/encoding.go:101.50,112.2 7 0 -github.com/cosmos/iavl/internal/encoding/encoding.go:115.37,117.2 1 0 -github.com/cosmos/iavl/internal/encoding/encoding.go:120.49,130.2 5 0 -github.com/cosmos/iavl/internal/encoding/encoding.go:133.38,134.12 1 0 -github.com/cosmos/iavl/internal/encoding/encoding.go:134.12,136.3 1 0 -github.com/cosmos/iavl/internal/encoding/encoding.go:137.2,137.32 1 0 -github.com/cosmos/iavl/internal/encoding/encoding.go:141.47,158.2 5 0 -github.com/cosmos/iavl/internal/encoding/encoding.go:161.36,164.2 2 0 -github.com/cosmos/iavl/internal/bytes/bytes.go:13.46,15.2 1 1 -github.com/cosmos/iavl/internal/bytes/bytes.go:18.50,21.2 2 1 -github.com/cosmos/iavl/internal/bytes/bytes.go:24.50,31.2 6 3 -github.com/cosmos/iavl/internal/bytes/bytes.go:34.54,35.65 1 3 -github.com/cosmos/iavl/internal/bytes/bytes.go:35.65,37.3 1 0 -github.com/cosmos/iavl/internal/bytes/bytes.go:38.2,41.16 4 3 -github.com/cosmos/iavl/internal/bytes/bytes.go:41.16,43.3 1 0 -github.com/cosmos/iavl/internal/bytes/bytes.go:44.2,45.12 2 3 -github.com/cosmos/iavl/internal/bytes/bytes.go:49.35,51.2 1 0 -github.com/cosmos/iavl/internal/bytes/bytes.go:53.36,55.2 1 0 -github.com/cosmos/iavl/internal/bytes/bytes.go:57.51,58.14 1 0 -github.com/cosmos/iavl/internal/bytes/bytes.go:59.11,60.41 1 0 -github.com/cosmos/iavl/internal/bytes/bytes.go:61.10,62.49 1 0 -github.com/cosmos/iavl/internal/bytes/string.go:10.40,18.2 7 5 -github.com/cosmos/iavl/internal/bytes/string.go:24.40,26.2 1 5 -github.com/cosmos/iavl/internal/rand/random.go:28.13,31.2 2 1 -github.com/cosmos/iavl/internal/rand/random.go:33.22,37.2 3 1 -github.com/cosmos/iavl/internal/rand/random.go:39.23,42.25 3 2 -github.com/cosmos/iavl/internal/rand/random.go:42.25,45.3 2 16 -github.com/cosmos/iavl/internal/rand/random.go:46.2,46.22 1 2 -github.com/cosmos/iavl/internal/rand/random.go:49.34,51.2 1 102 -github.com/cosmos/iavl/internal/rand/random.go:56.23,58.2 1 0 -github.com/cosmos/iavl/internal/rand/random.go:60.33,62.2 1 1 -github.com/cosmos/iavl/internal/rand/random.go:64.20,66.2 1 100 -github.com/cosmos/iavl/internal/rand/random.go:68.24,70.2 1 100 -github.com/cosmos/iavl/internal/rand/random.go:72.30,74.2 1 1 -github.com/cosmos/iavl/internal/rand/random.go:76.28,78.2 1 100 -github.com/cosmos/iavl/internal/rand/random.go:83.33,87.2 3 0 -github.com/cosmos/iavl/internal/rand/random.go:90.39,93.6 2 1 -github.com/cosmos/iavl/internal/rand/random.go:93.6,95.27 2 26 -github.com/cosmos/iavl/internal/rand/random.go:95.27,97.15 2 256 -github.com/cosmos/iavl/internal/rand/random.go:97.15,99.13 2 13 -github.com/cosmos/iavl/internal/rand/random.go:100.10,102.29 2 243 -github.com/cosmos/iavl/internal/rand/random.go:102.29,103.21 1 1 -github.com/cosmos/iavl/internal/rand/random.go:105.5,105.14 1 242 -github.com/cosmos/iavl/internal/rand/random.go:110.2,110.22 1 1 -github.com/cosmos/iavl/internal/rand/random.go:113.32,115.2 1 0 -github.com/cosmos/iavl/internal/rand/random.go:117.32,122.2 4 0 -github.com/cosmos/iavl/internal/rand/random.go:124.32,126.2 1 0 -github.com/cosmos/iavl/internal/rand/random.go:128.28,133.2 4 0 -github.com/cosmos/iavl/internal/rand/random.go:135.30,137.2 1 0 -github.com/cosmos/iavl/internal/rand/random.go:139.30,141.2 1 0 -github.com/cosmos/iavl/internal/rand/random.go:143.30,145.2 1 0 -github.com/cosmos/iavl/internal/rand/random.go:147.26,152.2 4 343 -github.com/cosmos/iavl/internal/rand/random.go:154.30,159.2 4 100 -github.com/cosmos/iavl/internal/rand/random.go:161.38,166.2 4 0 -github.com/cosmos/iavl/internal/rand/random.go:168.30,173.2 4 26 -github.com/cosmos/iavl/internal/rand/random.go:175.38,180.2 4 0 -github.com/cosmos/iavl/internal/rand/random.go:182.34,187.2 4 0 -github.com/cosmos/iavl/internal/rand/random.go:189.34,194.2 4 0 -github.com/cosmos/iavl/internal/rand/random.go:196.33,198.2 1 0 -github.com/cosmos/iavl/internal/rand/random.go:202.36,206.31 2 1 -github.com/cosmos/iavl/internal/rand/random.go:206.31,208.3 1 243 -github.com/cosmos/iavl/internal/rand/random.go:209.2,209.11 1 1 -github.com/cosmos/iavl/internal/rand/random.go:214.32,219.2 4 0 -github.com/cosmos/iavl/internal/rand/random.go:222.28,226.2 1 0 -github.com/cosmos/iavl/internal/rand/random.go:229.34,234.2 4 100 -github.com/cosmos/iavl/internal/rand/random.go:238.38,241.16 3 2 -github.com/cosmos/iavl/internal/rand/random.go:241.16,242.13 1 0 -github.com/cosmos/iavl/internal/rand/random.go:244.2,244.10 1 2 From af433797d67ec06e1b59c578b7263880d21c2977 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Tue, 10 Oct 2023 14:26:40 -0700 Subject: [PATCH 62/72] Fix print command --- cmd/changeset/print.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/changeset/print.go b/cmd/changeset/print.go index 7b255b3..adc8fe1 100644 --- a/cmd/changeset/print.go +++ b/cmd/changeset/print.go @@ -24,12 +24,12 @@ func handlePrintChangeset(cmd *cobra.Command, _ []string) error { return errors.New("input-file is required") } importer := changeset.NewImporter(inputFile).WithProcessFn(func(version int64, cs *iavl.ChangeSet) (bool, error) { - if cs != nil { + if cs != nil && version >= 0 { for _, pair := range cs.Pairs { fmt.Printf("Version: %d, delete: %t, key: %X, value: %X\n", version, pair.Delete, pair.Key, pair.Value) } } - return true, nil + return false, nil }) _, err := importer.Start() if err != nil { From 3116486d092a538bd3fbd5e1b9325fa3f86737a9 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Tue, 10 Oct 2023 14:52:39 -0700 Subject: [PATCH 63/72] Fix go mod --- changeset/export.go | 4 ++-- go.mod | 4 ++-- go.sum | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/changeset/export.go b/changeset/export.go index 2fbfe5f..a0124fb 100644 --- a/changeset/export.go +++ b/changeset/export.go @@ -161,11 +161,11 @@ func collectChunksToSegment(outputFile string, chunkFiles []string) error { var endingHeader [16]byte binary.LittleEndian.PutUint64(endingHeader[:], math.MaxUint64) binary.LittleEndian.PutUint64(endingHeader[8:], uint64(0)) - if _, err := bufWriter.Write(endingHeader[:]); err != nil { + if _, err := writer.Write(endingHeader[:]); err != nil { return err } - return bufWriter.Flush() + return writer.Flush() } // copyTmpFile append the compressed temporary file to writer diff --git a/go.mod b/go.mod index 085148e..ba50b6f 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,12 @@ module github.com/cosmos/iavl go 1.18 require ( - github.com/DataDog/zstd v1.5.5 + github.com/DataDog/zstd v1.4.1 github.com/alitto/pond v1.8.3 github.com/confio/ics23/go v0.7.0 github.com/gogo/protobuf v1.3.2 github.com/golang/mock v1.6.0 - github.com/klauspost/compress v1.15.1 + github.com/klauspost/compress v1.17.0 github.com/pkg/errors v0.9.1 github.com/spf13/cobra v1.5.0 github.com/stretchr/testify v1.8.1 diff --git a/go.sum b/go.sum index 0d81a9b..d875e63 100644 --- a/go.sum +++ b/go.sum @@ -75,9 +75,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d/go.mod h1:URdX5+vg25ts3aCh8H5IFZybJYKWhJHYMTnf+ULtoC4= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= -github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs= github.com/GaijinEntertainment/go-exhaustruct/v2 v2.1.0/go.mod h1:LGOGuvEgCfCQsy3JF2tRmpGDpzA53iZfyGEWSPwQ6/4= github.com/HdrHistogram/hdrhistogram-go v1.1.0/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= @@ -577,8 +576,9 @@ github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6 github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= +github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= From 3fc52ca31c435e2aa9501c944534251c9fecb44b Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Tue, 10 Oct 2023 15:09:36 -0700 Subject: [PATCH 64/72] Add logs --- changeset/export.go | 1 + 1 file changed, 1 insertion(+) diff --git a/changeset/export.go b/changeset/export.go index a0124fb..fb01b3a 100644 --- a/changeset/export.go +++ b/changeset/export.go @@ -202,6 +202,7 @@ func WriteChangeSet(writer io.Writer, version int64, cs iavl.ChangeSet) error { } // Write header + fmt.Printf("Writing version %d with %d items at %s\n", version, len(items), time.Now().Format(time.RFC3339)) var versionHeader [16]byte binary.LittleEndian.PutUint64(versionHeader[:], uint64(version)) binary.LittleEndian.PutUint64(versionHeader[8:], uint64(size)) From 47c352f8e783f31cfbe1f1aac0be4ebbd21abe82 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Tue, 10 Oct 2023 15:18:48 -0700 Subject: [PATCH 65/72] Fix flush --- changeset/export.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changeset/export.go b/changeset/export.go index fb01b3a..39c3a84 100644 --- a/changeset/export.go +++ b/changeset/export.go @@ -165,7 +165,7 @@ func collectChunksToSegment(outputFile string, chunkFiles []string) error { return err } - return writer.Flush() + return bufWriter.Flush() } // copyTmpFile append the compressed temporary file to writer From 966af358aeee2ac189b467bd0b4510bb588293e0 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Tue, 10 Oct 2023 15:29:05 -0700 Subject: [PATCH 66/72] Add logs --- changeset/import.go | 1 + 1 file changed, 1 insertion(+) diff --git a/changeset/import.go b/changeset/import.go index ba28200..e3460c6 100644 --- a/changeset/import.go +++ b/changeset/import.go @@ -86,6 +86,7 @@ func readNextChangeset(reader Reader) (int64, int64, *iavl.ChangeSet, error) { } // Read header version := binary.LittleEndian.Uint64(versionHeader[:8]) + fmt.Printf("Reading version %d\n", version) if version == math.MaxUint64 { return -1, -1, nil, nil } From 1d713c6f4bb578cb76dfadcccc66b7c7280a8ed7 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Tue, 10 Oct 2023 15:30:23 -0700 Subject: [PATCH 67/72] Add logs --- changeset/import.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changeset/import.go b/changeset/import.go index e3460c6..93a7b71 100644 --- a/changeset/import.go +++ b/changeset/import.go @@ -86,11 +86,11 @@ func readNextChangeset(reader Reader) (int64, int64, *iavl.ChangeSet, error) { } // Read header version := binary.LittleEndian.Uint64(versionHeader[:8]) - fmt.Printf("Reading version %d\n", version) if version == math.MaxUint64 { return -1, -1, nil, nil } size := int64(binary.LittleEndian.Uint64(versionHeader[8:16])) + fmt.Printf("Reading version: %d, size: %d\n", version, size) if size <= 0 { return 16, int64(version), nil, nil } From 9ea4bcfa160709e4027f70ed48e629a23e2efcc6 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Tue, 10 Oct 2023 15:38:44 -0700 Subject: [PATCH 68/72] Add logs --- changeset/export.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/changeset/export.go b/changeset/export.go index 39c3a84..3fdf6af 100644 --- a/changeset/export.go +++ b/changeset/export.go @@ -6,7 +6,6 @@ import ( "encoding/binary" "fmt" "io" - "math" "math/bits" "os" "path/filepath" @@ -158,13 +157,6 @@ func collectChunksToSegment(outputFile string, chunkFiles []string) error { } } - var endingHeader [16]byte - binary.LittleEndian.PutUint64(endingHeader[:], math.MaxUint64) - binary.LittleEndian.PutUint64(endingHeader[8:], uint64(0)) - if _, err := writer.Write(endingHeader[:]); err != nil { - return err - } - return bufWriter.Flush() } @@ -205,7 +197,7 @@ func WriteChangeSet(writer io.Writer, version int64, cs iavl.ChangeSet) error { fmt.Printf("Writing version %d with %d items at %s\n", version, len(items), time.Now().Format(time.RFC3339)) var versionHeader [16]byte binary.LittleEndian.PutUint64(versionHeader[:], uint64(version)) - binary.LittleEndian.PutUint64(versionHeader[8:], uint64(size)) + binary.LittleEndian.PutUint64(versionHeader[8:16], uint64(size)) if _, err := writer.Write(versionHeader[:]); err != nil { return err From be0d57182a2b8fd50bf7b43ce80ae5b1e2f2b0c7 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Tue, 10 Oct 2023 15:46:25 -0700 Subject: [PATCH 69/72] Add logs --- changeset/export.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/changeset/export.go b/changeset/export.go index 3fdf6af..53fff47 100644 --- a/changeset/export.go +++ b/changeset/export.go @@ -143,10 +143,7 @@ func collectChunksToSegment(outputFile string, chunkFiles []string) error { bufWriter := bufio.NewWriter(fp) writer, _ := zstd.NewWriter(bufWriter) - defer func() { - fp.Close() - writer.Close() - }() + defer fp.Close() for _, chunkFile := range chunkFiles { if err := copyTmpFile(writer, chunkFile); err != nil { @@ -156,7 +153,10 @@ func collectChunksToSegment(outputFile string, chunkFiles []string) error { return err } } - + err = writer.Close() + if err != nil { + return err + } return bufWriter.Flush() } From cac4c5e85b986cc2022582b4f92279ee8390f2ab Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Tue, 10 Oct 2023 15:51:42 -0700 Subject: [PATCH 70/72] Fix ending header --- changeset/export.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/changeset/export.go b/changeset/export.go index 53fff47..dea769e 100644 --- a/changeset/export.go +++ b/changeset/export.go @@ -6,6 +6,7 @@ import ( "encoding/binary" "fmt" "io" + "math" "math/bits" "os" "path/filepath" @@ -153,6 +154,15 @@ func collectChunksToSegment(outputFile string, chunkFiles []string) error { return err } } + + // Write ending header + var endingHeader [16]byte + binary.LittleEndian.PutUint64(endingHeader[:], uint64(math.MaxUint64)) + binary.LittleEndian.PutUint64(endingHeader[8:16], uint64(0)) + if _, err := writer.Write(endingHeader[:]); err != nil { + return err + } + err = writer.Close() if err != nil { return err From 316ab35026950030753c1ec7631d40adc04843d6 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Tue, 10 Oct 2023 16:01:39 -0700 Subject: [PATCH 71/72] Fix logs --- changeset/export.go | 2 +- changeset/import.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/changeset/export.go b/changeset/export.go index dea769e..04d1af3 100644 --- a/changeset/export.go +++ b/changeset/export.go @@ -162,7 +162,7 @@ func collectChunksToSegment(outputFile string, chunkFiles []string) error { if _, err := writer.Write(endingHeader[:]); err != nil { return err } - + err = writer.Close() if err != nil { return err diff --git a/changeset/import.go b/changeset/import.go index 93a7b71..e258af8 100644 --- a/changeset/import.go +++ b/changeset/import.go @@ -65,7 +65,6 @@ func iterateChangeSet(reader Reader, fn func(version int64, changeset *iavl.Chan if offset == -1 || version == -1 { // this means we are done - fmt.Printf("Done\n") break } shouldStop, err := fn(version, changeSet) From 9365ed91cec98e415c49ba87a18ce3d5c72a0ef8 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Thu, 7 Dec 2023 17:22:36 -0800 Subject: [PATCH 72/72] Add prints --- immutable_tree.go | 5 +- mutable_tree.go | 6 ++ mutable_tree_test.go | 109 ++++++++++++++++++++++++++++++++ node.go | 4 +- nodedb.go | 3 +- proto/memiavl/commit_info.proto | 26 ++++++++ proto/memiavl/wal.proto | 35 ++++++++++ scripts/protocgen.sh | 4 ++ 8 files changed, 188 insertions(+), 4 deletions(-) create mode 100644 proto/memiavl/commit_info.proto create mode 100644 proto/memiavl/wal.proto diff --git a/immutable_tree.go b/immutable_tree.go index 3bd958c..ed8fc39 100644 --- a/immutable_tree.go +++ b/immutable_tree.go @@ -86,14 +86,15 @@ func (t *ImmutableTree) renderNode(node *Node, indent string, depth int, encoder } // handle leaf if node.isLeaf() { - here := fmt.Sprintf("%s%s", prefix, encoder(node.GetNodeKey(), depth, true)) + here := fmt.Sprintf("%s|key:%s,val:%s,ver:%d,height:%d", prefix, encoder(node.GetNodeKey(), depth, true), string(node.GetValue()), node.GetVersion(), node.height) return []string{here}, nil } // recurse on inner node - here := fmt.Sprintf("%s%s", prefix, encoder(node.GetHash(), depth, false)) + here := fmt.Sprintf("%s%s|ver:%d|h:%d", prefix, encoder(node.GetNodeKey(), depth, false), node.version, node.height) rightNode, err := node.getRightNode(t) + if err != nil { return nil, err } diff --git a/mutable_tree.go b/mutable_tree.go index 269bb04..9ab8f80 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -168,6 +168,7 @@ func (tree *MutableTree) Set(key, value []byte) (updated bool, err error) { return false, err } err = tree.addOrphans(orphaned) + if err != nil { return updated, err } @@ -271,6 +272,10 @@ func (tree *MutableTree) set(key []byte, value []byte) (orphans []*Node, updated orphans = tree.prepareOrphansSlice() tree.ITree.root, updated, err = tree.recursiveSet(tree.ITree.root, key, value, &orphans) + if err != nil { + return nil, updated, err + } + return orphans, updated, err } @@ -1405,6 +1410,7 @@ func (tree *MutableTree) addOrphans(orphans []*Node) error { if len(node.GetHash()) == 0 { return fmt.Errorf("expected to find node hash, but was empty") } + fmt.Printf("adding orphan: %s\n", node.String()) tree.orphans[unsafeToStr(node.GetHash())] = node.GetVersion() } return nil diff --git a/mutable_tree_test.go b/mutable_tree_test.go index 6eeaedb..721eec2 100644 --- a/mutable_tree_test.go +++ b/mutable_tree_test.go @@ -7,6 +7,7 @@ import ( "runtime" "sort" "strconv" + "strings" "testing" "github.com/cosmos/iavl/internal/encoding" @@ -1481,3 +1482,111 @@ func TestSaveCurrentVersion_ChangedHash(t *testing.T) { _, version, err = tree.SaveCurrentVersion() require.Error(t, err) } + +func TestAddPrint(t *testing.T) { + tree := setupMutableTree(t) + tree.Set([]byte("1"), []byte("1")) + tree.SaveVersion() + shape, err := tree.ITree.RenderShape("\t", myNodeEncoder) + require.NoError(t, err) + fmt.Printf("%s\n", strings.Join(shape, "\n")) + fmt.Printf("===========================================\n") + tree.Set([]byte("2"), []byte("2")) + tree.SaveVersion() + shape, err = tree.ITree.RenderShape("\t", myNodeEncoder) + require.NoError(t, err) + fmt.Printf("%s\n", strings.Join(shape, "\n")) + fmt.Printf("===========================================\n") + tree.Set([]byte("3"), []byte("3")) + tree.SaveVersion() + shape, _ = tree.ITree.RenderShape("\t", myNodeEncoder) + fmt.Printf("%s\n", strings.Join(shape, "\n")) + fmt.Printf("===========================================\n") + tree.Set([]byte("4"), []byte("4")) + tree.SaveVersion() + shape, _ = tree.ITree.RenderShape("\t", myNodeEncoder) + fmt.Printf("%s\n", strings.Join(shape, "\n")) + fmt.Printf("===========================================\n") + tree.Set([]byte("5"), []byte("5")) + tree.SaveVersion() + shape, err = tree.ITree.RenderShape("\t", myNodeEncoder) + fmt.Printf("%s\n", strings.Join(shape, "\n")) + fmt.Printf("===========================================\n") + tree.Set([]byte("6"), []byte("6")) + tree.SaveVersion() + shape, _ = tree.ITree.RenderShape("\t", myNodeEncoder) + fmt.Printf("%s", strings.Join(shape, "\n")) + fmt.Printf("===========================================\n") + tree.Set([]byte("7"), []byte("7")) + tree.SaveVersion() + shape, _ = tree.ITree.RenderShape("\t", myNodeEncoder) + fmt.Printf("%s", strings.Join(shape, "\n")) + fmt.Printf("===========================================\n") + tree.Set([]byte("8"), []byte("9")) + tree.SaveVersion() + shape, _ = tree.ITree.RenderShape("\t", myNodeEncoder) + fmt.Printf("%s", strings.Join(shape, "\n")) + fmt.Printf("===========================================\n") + tree.Set([]byte("9"), []byte("9")) + tree.SaveVersion() + shape, _ = tree.ITree.RenderShape("\t", myNodeEncoder) + fmt.Printf("%s", strings.Join(shape, "\n")) + fmt.Printf("===========================================\n") +} + +func TestAddVersions(t *testing.T) { + tree := setupMutableTree(t) + + // step 1 + tree.Set([]byte("1"), []byte("1")) + tree.SaveVersion() + tree.ndb.traverse(func(key, value []byte) error { + fmt.Printf("db key: %s\n", key) + return nil + }) + shape, err := tree.ITree.RenderShape("\t", myNodeEncoder) + require.NoError(t, err) + fmt.Printf("%s\n", strings.Join(shape, "\n")) + fmt.Printf("=========================\n") + + // step 2 + tree.Set([]byte("2"), []byte("2")) + tree.SaveVersion() + tree.ndb.traverse(func(key, value []byte) error { + fmt.Printf("db key: %s\n", key) + return nil + }) + fmt.Printf("tree version: %d\n", tree.Version()) + shape, err = tree.ITree.RenderShape("\t", myNodeEncoder) + fmt.Printf("%s\n", strings.Join(shape, "\n")) + fmt.Printf("=========================\n") + + // step 3 + tree.SaveVersion() + tree.ndb.traverse(func(key, value []byte) error { + fmt.Printf("db key: %s\n", key) + return nil + }) + + // step 4 + tree.Set([]byte("2"), []byte("4")) + tree.SaveVersion() + tree.ndb.traverse(func(key, value []byte) error { + fmt.Printf("db key: %s\n", key) + return nil + }) + fmt.Printf("tree version: %d\n", tree.Version()) + shape, err = tree.ITree.RenderShape("\t", myNodeEncoder) + fmt.Printf("%s\n", strings.Join(shape, "\n")) + fmt.Printf("=========================\n") + +} + +// defaultNodeEncoder can encode any node unless the client overrides it +func myNodeEncoder(key []byte, depth int, isLeaf bool) string { + if isLeaf { + return fmt.Sprintf("%s-%s", "LF", string(key)) + } + return fmt.Sprintf("%s-%s", "BR", string(key)) + +} diff --git a/node.go b/node.go index ea413ae..4ba5490 100644 --- a/node.go +++ b/node.go @@ -254,9 +254,11 @@ func (node *Node) String() string { if len(node.hash) > 0 { hashstr = fmt.Sprintf("%X", node.hash) } - return fmt.Sprintf("Node{%s:%s@%d %X;%X}#%s", + return fmt.Sprintf("Node key:%s, value:%s, height: %d, size:%d, version:%d, lh:%X, rh:%X, #%s", ColoredBytes(node.key, Green, Blue), ColoredBytes(node.value, Cyan, Blue), + node.height, + node.size, node.version, node.leftHash, node.rightHash, hashstr) diff --git a/nodedb.go b/nodedb.go index 019a4ec..b28b7fb 100644 --- a/nodedb.go +++ b/nodedb.go @@ -640,7 +640,7 @@ func (ndb *nodeDB) SaveOrphans(version int64, orphans map[string]int64) error { } for hash, fromVersion := range orphans { - logger.Debug("SAVEORPHAN %v-%v %X\n", fromVersion, toVersion, hash) + //fmt.Printf("SAVEORPHAN %v-%v %X\n", fromVersion, toVersion, hash) err := ndb.saveOrphan([]byte(hash), fromVersion, toVersion) if err != nil { return err @@ -916,6 +916,7 @@ func (ndb *nodeDB) SaveRoot(root *Node, version int64) error { if len(root.GetHash()) == 0 { return ErrRootMissingHash } + fmt.Printf("Saving root %s at version %d\n", string(root.key), version) return ndb.saveRoot(root.GetHash(), version) } diff --git a/proto/memiavl/commit_info.proto b/proto/memiavl/commit_info.proto new file mode 100644 index 0000000..1cf4518 --- /dev/null +++ b/proto/memiavl/commit_info.proto @@ -0,0 +1,26 @@ +syntax = "proto3"; +package memiavl; + +option go_package = "github.com/sei-protocol/sei-db"; + +// CommitInfo defines commit information used by the multi-store when committing +// a version/height. +message CommitInfo { + int64 version = 1; + repeated StoreInfo store_infos = 2; +} + +// StoreInfo defines store-specific commit information. It contains a reference +// between a store name and the commit ID. +message StoreInfo { + string name = 1; + CommitID commit_id = 2; +} + +// CommitID defines the committment information when a specific store is +// committed. +message CommitID { + + int64 version = 1; + bytes hash = 2; +} diff --git a/proto/memiavl/wal.proto b/proto/memiavl/wal.proto new file mode 100644 index 0000000..a7f94d0 --- /dev/null +++ b/proto/memiavl/wal.proto @@ -0,0 +1,35 @@ +syntax = "proto3"; +package memiavl; + +option go_package = "github.com/sei-protocol/sei-db"; + +import "iavl/changeset.proto"; +import "memiavl/commit_info.proto"; + +// NamedChangeSet combine a tree name with the changeset +message NamedChangeSet { + iavl.ChangeSet changeset = 1; + string name = 2; +} + +// TreeNameUpgrade defines upgrade of tree names: +// - New tree: { name: "tree" } +// - Delete tree: { name: "tree", delete: true } +// - Rename tree: { name: "new-tree", rename_from: "old-tree" } +message TreeNameUpgrade { + string name = 1; + string rename_from = 2; + bool delete = 3; +} + +// WALEntry is a single Write-Ahead-Log entry +message WALEntry { + repeated NamedChangeSet changesets = 1; + repeated TreeNameUpgrade upgrades = 2; +} + +// MultiTreeMetadata stores the metadata for MultiTree +message MultiTreeMetadata { + CommitInfo commit_info = 1; + int64 initial_version = 2; +} diff --git a/scripts/protocgen.sh b/scripts/protocgen.sh index 4bf62a2..e5a9de0 100644 --- a/scripts/protocgen.sh +++ b/scripts/protocgen.sh @@ -5,3 +5,7 @@ set -eo pipefail buf generate --path proto/iavl mv ./proto/iavl/*.go ./proto + +buf generate --path proto/memiavl + +mv ./proto/memiavl/*.go ./proto \ No newline at end of file