From 2ead2539c00043484fe9634ea2f0416ff4fc5f11 Mon Sep 17 00:00:00 2001 From: Roy Lee Date: Sat, 31 Jul 2021 21:42:39 -0700 Subject: [PATCH] [lbry] Rework claimtrie CLIs 1. Ditch in-house block repo, and use btcd blocks database. 2. Commands switch from args to flags. 3. Revive chain recording and replaying, which will be part of CI pipeline. 4. Refactor cleanup the CLI skeletons. 5. Support DataDir, and testnet/regtes (not tested yet). TODOs: 1. Remove hardcoded test/development params, and pass them from flags. 2. Make output more sensible. 3. Add debug level flag. 4. Add MerkleTrie implementation switch. 5. Refactor periodic progess/status reporting for long run-time tasks. ... --- claimtrie/claimtrie.go | 8 +- claimtrie/cmd/cmd/block.go | 192 ++++-------- claimtrie/cmd/cmd/chain.go | 528 +++++++++++++++++++++++--------- claimtrie/cmd/cmd/helper.go | 62 ++++ claimtrie/cmd/cmd/merkletrie.go | 105 +++++++ claimtrie/cmd/cmd/node.go | 205 +++++++------ claimtrie/cmd/cmd/root.go | 46 ++- claimtrie/cmd/cmd/temporal.go | 92 +++--- claimtrie/cmd/cmd/ui.go | 20 +- go.mod | 1 + 10 files changed, 832 insertions(+), 427 deletions(-) create mode 100644 claimtrie/cmd/cmd/helper.go create mode 100644 claimtrie/cmd/cmd/merkletrie.go diff --git a/claimtrie/claimtrie.go b/claimtrie/claimtrie.go index 28b6f010..4ba666d2 100644 --- a/claimtrie/claimtrie.go +++ b/claimtrie/claimtrie.go @@ -247,9 +247,11 @@ func (ct *ClaimTrie) AppendBlock() error { updateNames = append(updateNames, newName) // TODO: make sure using the temporalRepo batch is actually faster updateHeights = append(updateHeights, nextUpdate) } - err = ct.temporalRepo.SetNodesAt(updateNames, updateHeights) - if err != nil { - return errors.Wrap(err, "temporal repo set") + if len(updateNames) != 0 { + err = ct.temporalRepo.SetNodesAt(updateNames, updateHeights) + if err != nil { + return errors.Wrap(err, "temporal repo set") + } } hitFork := ct.updateTrieForHashForkIfNecessary() diff --git a/claimtrie/cmd/cmd/block.go b/claimtrie/cmd/cmd/block.go index 5a2db0f4..d0ee7719 100644 --- a/claimtrie/cmd/cmd/block.go +++ b/claimtrie/cmd/cmd/block.go @@ -2,157 +2,97 @@ package cmd import ( "fmt" - "log" - "path/filepath" - "strconv" - - "github.com/btcsuite/btcd/claimtrie/block/blockrepo" - "github.com/btcsuite/btcd/claimtrie/merkletrie" - "github.com/btcsuite/btcd/claimtrie/merkletrie/merkletrierepo" - "github.com/btcsuite/btcd/claimtrie/temporal/temporalrepo" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" ) func init() { - rootCmd.AddCommand(blockCmd) - - blockCmd.AddCommand(blockLastCmd) - blockCmd.AddCommand(blockListCmd) - blockCmd.AddCommand(blockNameCmd) + rootCmd.AddCommand(NewBlocCommands()) } -var blockCmd = &cobra.Command{ - Use: "block", - Short: "Block related commands", +func NewBlocCommands() *cobra.Command { + + cmd := &cobra.Command{ + Use: "block", + Short: "Block related commands", + } + + cmd.AddCommand(NewBlockBestCommand()) + cmd.AddCommand(NewBlockListCommand()) + + return cmd } -var blockLastCmd = &cobra.Command{ - Use: "last", - Short: "Show the Merkle Hash of the last block", - RunE: func(cmd *cobra.Command, args []string) error { +func NewBlockBestCommand() *cobra.Command { - repo, err := blockrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ReportedBlockRepoPebble.Path)) - if err != nil { - log.Fatalf("can't open reported block repo: %s", err) - } + cmd := &cobra.Command{ + Use: "best", + Short: "Show the height and hash of the best block", + RunE: func(cmd *cobra.Command, args []string) error { - last, err := repo.Load() - if err != nil { - return fmt.Errorf("load previous height") - } - - hash, err := repo.Get(last) - if err != nil { - return fmt.Errorf("load changes from repo: %w", err) - } - - fmt.Printf("blk %-7d: %s\n", last, hash.String()) - - return nil - }, -} - -var blockListCmd = &cobra.Command{ - Use: "list []", - Short: "List the Merkle Hash of block in a range of heights", - Args: cobra.RangeArgs(1, 2), - RunE: func(cmd *cobra.Command, args []string) error { - - repo, err := blockrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ReportedBlockRepoPebble.Path)) - if err != nil { - log.Fatalf("can't open reported block repo: %s", err) - } - - fromHeight, err := strconv.Atoi(args[0]) - if err != nil { - return fmt.Errorf("invalid args") - } - - toHeight := fromHeight + 1 - if len(args) == 2 { - toHeight, err = strconv.Atoi(args[1]) + db, err := loadBlocksDB() if err != nil { - return fmt.Errorf("invalid args") + return errors.Wrapf(err, "load blocks database") } - } + defer db.Close() - last, err := repo.Load() - if err != nil { - return fmt.Errorf("load previous height") - } - - if toHeight >= int(last) { - toHeight = int(last) - } - - for i := fromHeight; i < toHeight; i++ { - hash, err := repo.Get(int32(i)) + chain, err := loadChain(db) if err != nil { - return fmt.Errorf("load changes from repo: %w", err) + return errors.Wrapf(err, "load chain") } - fmt.Printf("blk %-7d: %s\n", i, hash.String()) - } - return nil - }, + state := chain.BestSnapshot() + fmt.Printf("Block %7d: %s\n", state.Height, state.Hash.String()) + + return nil + }, + } + + return cmd } -var blockNameCmd = &cobra.Command{ - Use: "vertex ", - Short: "List the claim and child hashes at vertex name of block at height", - Args: cobra.RangeArgs(2, 2), - RunE: func(cmd *cobra.Command, args []string) error { +func NewBlockListCommand() *cobra.Command { - repo, err := blockrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.BlockRepoPebble.Path)) - if err != nil { - return fmt.Errorf("can't open reported block repo: %w", err) - } - defer repo.Close() + var fromHeight int32 + var toHeight int32 - height, err := strconv.Atoi(args[0]) - if err != nil { - return fmt.Errorf("invalid args") - } + cmd := &cobra.Command{ + Use: "list", + Short: "List merkle hash of blocks between ", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { - last, err := repo.Load() - if err != nil { - return fmt.Errorf("load previous height: %w", err) - } - - if last < int32(height) { - return fmt.Errorf("requested height is unavailable") - } - - hash, err := repo.Get(int32(height)) - if err != nil { - return fmt.Errorf("load previous height: %w", err) - } - - trieRepo, err := merkletrierepo.NewPebble(filepath.Join(cfg.DataDir, cfg.MerkleTrieRepoPebble.Path)) - if err != nil { - return fmt.Errorf("can't open merkle trie repo: %w", err) - } - - trie := merkletrie.NewPersistentTrie(nil, trieRepo) - defer trie.Close() - trie.SetRoot(hash, nil) - if len(args) > 1 { - trie.Dump(args[1]) - } else { - tmpRepo, err := temporalrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.TemporalRepoPebble.Path)) + db, err := loadBlocksDB() if err != nil { - return fmt.Errorf("can't open temporal repo: %w", err) + return errors.Wrapf(err, "load blocks database") } - nodes, err := tmpRepo.NodesAt(int32(height)) + defer db.Close() + + chain, err := loadChain(db) if err != nil { - return fmt.Errorf("can't read temporal repo at %d: %w", height, err) + return errors.Wrapf(err, "load chain") } - for _, name := range nodes { - fmt.Printf("Name: %s, ", string(name)) - trie.Dump(string(name)) + + if toHeight > chain.BestSnapshot().Height { + toHeight = chain.BestSnapshot().Height } - } - return nil - }, + + for ht := fromHeight; ht <= toHeight; ht++ { + hash, err := chain.BlockHashByHeight(ht) + if err != nil { + return errors.Wrapf(err, "load hash for %d", ht) + } + fmt.Printf("Block %7d: %s\n", ht, hash.String()) + } + + return nil + }, + } + + cmd.Flags().Int32Var(&fromHeight, "from", 0, "From height (inclusive)") + cmd.Flags().Int32Var(&toHeight, "to", 0, "To height (inclusive)") + cmd.Flags().SortFlags = false + + return cmd } diff --git a/claimtrie/cmd/cmd/chain.go b/claimtrie/cmd/cmd/chain.go index 36e75a93..626ce852 100644 --- a/claimtrie/cmd/cmd/chain.go +++ b/claimtrie/cmd/cmd/chain.go @@ -2,191 +2,423 @@ package cmd import ( "fmt" - "math" "os" "path/filepath" - "strconv" + "sync" + "time" + "github.com/btcsuite/btcd/blockchain" "github.com/btcsuite/btcd/claimtrie" - "github.com/btcsuite/btcd/claimtrie/block" - "github.com/btcsuite/btcd/claimtrie/block/blockrepo" "github.com/btcsuite/btcd/claimtrie/chain/chainrepo" "github.com/btcsuite/btcd/claimtrie/change" "github.com/btcsuite/btcd/claimtrie/config" + "github.com/btcsuite/btcd/database" + _ "github.com/btcsuite/btcd/database/ffldb" + "github.com/btcsuite/btcd/txscript" + "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcutil" + + "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble" "github.com/spf13/cobra" ) func init() { - rootCmd.AddCommand(chainCmd) - - chainCmd.AddCommand(chainDumpCmd) - chainCmd.AddCommand(chainReplayCmd) + rootCmd.AddCommand(NewChainCommands()) } -var chainCmd = &cobra.Command{ - Use: "chain", - Short: "chain related command", +func NewChainCommands() *cobra.Command { + + cmd := &cobra.Command{ + Use: "chain", + Short: "chain related command", + } + + cmd.AddCommand(NewChainDumpCommand()) + cmd.AddCommand(NewChainReplayCommand()) + cmd.AddCommand(NewChainConvertCommand()) + + return cmd } -var chainDumpCmd = &cobra.Command{ - Use: "dump []", - Short: "dump changes from to []", - Args: cobra.RangeArgs(1, 2), - RunE: func(cmd *cobra.Command, args []string) error { +func NewChainDumpCommand() *cobra.Command { - fromHeight, err := strconv.Atoi(args[0]) - if err != nil { - return fmt.Errorf("invalid args") - } + var fromHeight int32 + var toHeight int32 - toHeight := fromHeight + 1 - if len(args) == 2 { - toHeight, err = strconv.Atoi(args[1]) + cmd := &cobra.Command{ + Use: "dump", + Short: "Dump the chain changes between and ", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + + dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.ChainRepoPebble.Path) + log.Debugf("Open chain repo: %q", dbPath) + chainRepo, err := chainrepo.NewPebble(dbPath) if err != nil { - return fmt.Errorf("invalid args") - } - } - - chainRepo, err := chainrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ChainRepoPebble.Path)) - if err != nil { - return fmt.Errorf("open node repo: %w", err) - } - - for height := fromHeight; height < toHeight; height++ { - changes, err := chainRepo.Load(int32(height)) - if err == pebble.ErrNotFound { - continue - } - if err != nil { - return fmt.Errorf("load commands: %w", err) + return errors.Wrapf(err, "open chain repo") } - for _, chg := range changes { - if int(chg.Height) > height { - break + for height := fromHeight; height <= toHeight; height++ { + changes, err := chainRepo.Load(height) + if errors.Is(err, pebble.ErrNotFound) { + continue } - showChange(chg) - } - } - - return nil - }, -} - -var chainReplayCmd = &cobra.Command{ - Use: "replay ", - Short: "Replay the chain up to ", - Args: cobra.RangeArgs(0, 1), - RunE: func(cmd *cobra.Command, args []string) error { - - fmt.Printf("not working until we pass record flag to claimtrie\n") - - fromHeight := 2 - toHeight := int(math.MaxInt32) - - var err error - if len(args) == 1 { - toHeight, err = strconv.Atoi(args[0]) - if err != nil { - return fmt.Errorf("invalid args") - } - } - - err = os.RemoveAll(filepath.Join(cfg.DataDir, cfg.NodeRepoPebble.Path)) - if err != nil { - return fmt.Errorf("delete node repo: %w", err) - } - - fmt.Printf("Deleted node repo\n") - - chainRepo, err := chainrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ChainRepoPebble.Path)) - if err != nil { - return fmt.Errorf("open change repo: %w", err) - } - - reportedBlockRepo, err := blockrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ReportedBlockRepoPebble.Path)) - if err != nil { - return fmt.Errorf("open block repo: %w", err) - } - - cfg := config.DefaultConfig - ct, err := claimtrie.New(cfg) - if err != nil { - return fmt.Errorf("create claimtrie: %w", err) - } - defer ct.Close() - - err = ct.ResetHeight(int32(fromHeight - 1)) - if err != nil { - return fmt.Errorf("reset claimtrie height: %w", err) - } - - for height := int32(fromHeight); height < int32(toHeight); height++ { - - changes, err := chainRepo.Load(height) - if err == pebble.ErrNotFound { - // do nothing. - } else if err != nil { - return fmt.Errorf("load from change repo: %w", err) - } - - for _, chg := range changes { - - switch chg.Type { - case change.AddClaim: - err = ct.AddClaim(chg.Name, chg.OutPoint, chg.ClaimID, chg.Amount) - - case change.UpdateClaim: - err = ct.UpdateClaim(chg.Name, chg.OutPoint, chg.Amount, chg.ClaimID) - - case change.SpendClaim: - err = ct.SpendClaim(chg.Name, chg.OutPoint, chg.ClaimID) - - case change.AddSupport: - err = ct.AddSupport(chg.Name, chg.OutPoint, chg.Amount, chg.ClaimID) - - case change.SpendSupport: - err = ct.SpendSupport(chg.Name, chg.OutPoint, chg.ClaimID) - - default: - err = fmt.Errorf("invalid change: %v", chg) - } - if err != nil { - return fmt.Errorf("execute change %v: %w", chg, err) + return errors.Wrapf(err, "load charnges for height: %d") + } + for _, chg := range changes { + showChange(chg) } } - err = appendBlock(ct, reportedBlockRepo) - if err != nil { - return err - } - if ct.Height()%1000 == 0 { - fmt.Printf("block: %d\n", ct.Height()) - } - } - return nil - }, + return nil + }, + } + + cmd.Flags().Int32Var(&fromHeight, "from", 0, "From height (inclusive)") + cmd.Flags().Int32Var(&toHeight, "to", 0, "To height (inclusive)") + cmd.Flags().SortFlags = false + + return cmd } -func appendBlock(ct *claimtrie.ClaimTrie, blockRepo block.Repo) error { +func NewChainReplayCommand() *cobra.Command { + + var fromHeight int32 + var toHeight int32 + + cmd := &cobra.Command{ + Use: "replay", + Short: "Replay the chain changes between and ", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + + for _, dbName := range []string{ + cfg.BlockRepoPebble.Path, + cfg.NodeRepoPebble.Path, + cfg.MerkleTrieRepoPebble.Path, + cfg.TemporalRepoPebble.Path, + } { + dbPath := filepath.Join(dataDir, netName, "claim_dbs", dbName) + log.Debugf("Delete repo: %q", dbPath) + err := os.RemoveAll(dbPath) + if err != nil { + return errors.Wrapf(err, "delete repo: %q", dbPath) + } + } + + dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.ChainRepoPebble.Path) + log.Debugf("Open chain repo: %q", dbPath) + chainRepo, err := chainrepo.NewPebble(dbPath) + if err != nil { + return errors.Wrapf(err, "open chain repo") + } + + cfg := config.DefaultConfig + cfg.RamTrie = true + cfg.DataDir = filepath.Join(dataDir, netName) + + ct, err := claimtrie.New(cfg) + if err != nil { + return errors.Wrapf(err, "create claimtrie") + } + defer ct.Close() + + db, err := loadBlocksDB() + if err != nil { + return errors.Wrapf(err, "load blocks database") + } + + chain, err := loadChain(db) + if err != nil { + return errors.Wrapf(err, "load chain") + } + + for ht := fromHeight; ht < toHeight; ht++ { + + changes, err := chainRepo.Load(ht + 1) + if errors.Is(err, pebble.ErrNotFound) { + // do nothing. + } else if err != nil { + return errors.Wrapf(err, "load changes for block %d", ht) + } + + for _, chg := range changes { + + switch chg.Type { + case change.AddClaim: + err = ct.AddClaim(chg.Name, chg.OutPoint, chg.ClaimID, chg.Amount) + case change.UpdateClaim: + err = ct.UpdateClaim(chg.Name, chg.OutPoint, chg.Amount, chg.ClaimID) + case change.SpendClaim: + err = ct.SpendClaim(chg.Name, chg.OutPoint, chg.ClaimID) + case change.AddSupport: + err = ct.AddSupport(chg.Name, chg.OutPoint, chg.Amount, chg.ClaimID) + case change.SpendSupport: + err = ct.SpendSupport(chg.Name, chg.OutPoint, chg.ClaimID) + default: + err = errors.Errorf("invalid change type: %v", chg) + } + + if err != nil { + return errors.Wrapf(err, "execute change %v", chg) + } + } + err = appendBlock(ct, chain) + if err != nil { + return errors.Wrapf(err, "appendBlock") + } + + if ct.Height()%1000 == 0 { + fmt.Printf("block: %d\n", ct.Height()) + } + } + + return nil + }, + } + + // FIXME + cmd.Flags().Int32Var(&fromHeight, "from", 0, "From height") + cmd.Flags().Int32Var(&toHeight, "to", 0, "To height") + cmd.Flags().SortFlags = false + + return cmd +} + +func appendBlock(ct *claimtrie.ClaimTrie, chain *blockchain.BlockChain) error { err := ct.AppendBlock() if err != nil { - return fmt.Errorf("append block: %w", err) + return errors.Wrapf(err, "append block: %w") } - height := ct.Height() - - hash, err := blockRepo.Get(height) + block, err := chain.BlockByHeight(ct.Height()) if err != nil { - return fmt.Errorf("load from block repo: %w", err) + return errors.Wrapf(err, "load from block repo: %w") } + hash := block.MsgBlock().Header.ClaimTrie - if *ct.MerkleHash() != *hash { - return fmt.Errorf("hash mismatched at height %5d: exp: %s, got: %s", height, hash, ct.MerkleHash()) + if *ct.MerkleHash() != hash { + return errors.Errorf("hash mismatched at height %5d: exp: %s, got: %s", ct.Height(), hash, ct.MerkleHash()) } return nil } + +func NewChainConvertCommand() *cobra.Command { + + var height int32 + + cmd := &cobra.Command{ + Use: "convert", + Short: "convert changes from to ", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + + db, err := loadBlocksDB() + if err != nil { + return errors.Wrapf(err, "load block db") + } + defer db.Close() + + chain, err := loadChain(db) + if err != nil { + return errors.Wrapf(err, "load block db") + } + + converter := chainConverter{ + db: db, + chain: chain, + blockChan: make(chan *btcutil.Block, 1000), + changesChan: make(chan []change.Change, 1000), + wg: &sync.WaitGroup{}, + } + + startTime := time.Now() + err = converter.start() + if err != nil { + return errors.Wrapf(err, "start Converter") + } + + converter.wait() + log.Infof("Convert chain: took %s", time.Since(startTime)) + + return nil + }, + } + + cmd.Flags().Int32Var(&height, "height", 0, "Height") + + return cmd +} + +type chainConverter struct { + db database.DB + chain *blockchain.BlockChain + + blockChan chan *btcutil.Block + changesChan chan []change.Change + + wg *sync.WaitGroup + + statBlocksFetched int + statBlocksProcessed int + statChangesSaved int +} + +func (cc *chainConverter) wait() { + cc.wg.Wait() +} + +func (cb *chainConverter) start() error { + + go cb.reportStats() + + cb.wg.Add(3) + go cb.getBlock() + go cb.processBlock() + go cb.saveChanges() + + return nil +} + +func (cb *chainConverter) getBlock() { + defer cb.wg.Done() + defer close(cb.blockChan) + + toHeight := int32(200000) + fmt.Printf("blocks: %d\n", cb.chain.BestSnapshot().Height) + + if toHeight > cb.chain.BestSnapshot().Height { + toHeight = cb.chain.BestSnapshot().Height + + } + + for ht := int32(0); ht < toHeight; ht++ { + block, err := cb.chain.BlockByHeight(ht) + if err != nil { + log.Errorf("load changes from repo: %w", err) + return + } + cb.statBlocksFetched++ + cb.blockChan <- block + } +} + +func (cb *chainConverter) processBlock() { + defer cb.wg.Done() + defer close(cb.changesChan) + + view := blockchain.NewUtxoViewpoint() + for block := range cb.blockChan { + var changes []change.Change + for _, tx := range block.Transactions() { + view.AddTxOuts(tx, block.Height()) + + if blockchain.IsCoinBase(tx) { + continue + } + + for _, txIn := range tx.MsgTx().TxIn { + op := txIn.PreviousOutPoint + e := view.LookupEntry(op) + if e == nil { + log.Criticalf("Missing input in view for %s", op.String()) + } + cs, err := txscript.DecodeClaimScript(e.PkScript()) + if err == txscript.ErrNotClaimScript { + continue + } + if err != nil { + log.Criticalf("Can't parse claim script: %s", err) + } + + chg := change.Change{ + Height: block.Height(), + Name: cs.Name(), + OutPoint: txIn.PreviousOutPoint, + } + + switch cs.Opcode() { + case txscript.OP_CLAIMNAME: + chg.Type = change.SpendClaim + chg.ClaimID = change.NewClaimID(chg.OutPoint) + case txscript.OP_UPDATECLAIM: + chg.Type = change.SpendClaim + copy(chg.ClaimID[:], cs.ClaimID()) + case txscript.OP_SUPPORTCLAIM: + chg.Type = change.SpendSupport + copy(chg.ClaimID[:], cs.ClaimID()) + } + + changes = append(changes, chg) + } + + op := *wire.NewOutPoint(tx.Hash(), 0) + for i, txOut := range tx.MsgTx().TxOut { + cs, err := txscript.DecodeClaimScript(txOut.PkScript) + if err == txscript.ErrNotClaimScript { + continue + } + + op.Index = uint32(i) + chg := change.Change{ + Height: block.Height(), + Name: cs.Name(), + OutPoint: op, + Amount: txOut.Value, + } + + switch cs.Opcode() { + case txscript.OP_CLAIMNAME: + chg.Type = change.AddClaim + chg.ClaimID = change.NewClaimID(op) + case txscript.OP_SUPPORTCLAIM: + chg.Type = change.AddSupport + copy(chg.ClaimID[:], cs.ClaimID()) + case txscript.OP_UPDATECLAIM: + chg.Type = change.UpdateClaim + copy(chg.ClaimID[:], cs.ClaimID()) + } + changes = append(changes, chg) + } + } + cb.statBlocksProcessed++ + + if len(changes) != 0 { + cb.changesChan <- changes + } + } +} + +func (cb *chainConverter) saveChanges() { + defer cb.wg.Done() + + dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.ChainRepoPebble.Path) + chainRepo, err := chainrepo.NewPebble(dbPath) + if err != nil { + log.Errorf("open chain repo: %s", err) + return + } + defer chainRepo.Close() + + for changes := range cb.changesChan { + err = chainRepo.Save(changes[0].Height, changes) + if err != nil { + log.Errorf("save to chain repo: %s", err) + return + } + cb.statChangesSaved++ + } +} + +func (cb *chainConverter) reportStats() { + tick := time.NewTicker(5 * time.Second) + for range tick.C { + log.Infof("block : %7d / %7d, changes saved: %d", + cb.statBlocksFetched, cb.statBlocksProcessed, cb.statChangesSaved) + + } +} diff --git a/claimtrie/cmd/cmd/helper.go b/claimtrie/cmd/cmd/helper.go new file mode 100644 index 00000000..e71b4bbb --- /dev/null +++ b/claimtrie/cmd/cmd/helper.go @@ -0,0 +1,62 @@ +package cmd + +import ( + "path/filepath" + "time" + + "github.com/btcsuite/btcd/blockchain" + "github.com/btcsuite/btcd/chaincfg" + "github.com/btcsuite/btcd/database" + "github.com/btcsuite/btcd/txscript" + + "github.com/cockroachdb/errors" +) + +func loadBlocksDB() (database.DB, error) { + + dbPath := filepath.Join(dataDir, netName, "blocks_ffldb") + log.Infof("Loading blocks database: %s", dbPath) + db, err := database.Open("ffldb", dbPath, chainPramas().Net) + if err != nil { + return nil, errors.Wrapf(err, "open blocks database") + } + + return db, nil +} + +func loadChain(db database.DB) (*blockchain.BlockChain, error) { + paramsCopy := chaincfg.MainNetParams + + log.Infof("Loading chain from database") + + startTime := time.Now() + chain, err := blockchain.New(&blockchain.Config{ + DB: db, + ChainParams: ¶msCopy, + TimeSource: blockchain.NewMedianTime(), + SigCache: txscript.NewSigCache(1000), + }) + if err != nil { + return nil, errors.Wrapf(err, "create blockchain") + } + + log.Infof("Loaded chain from database (%s)", time.Since(startTime)) + + return chain, err + +} + +func chainPramas() chaincfg.Params { + + // Make a copy so the user won't modify the global instance. + params := chaincfg.MainNetParams + switch netName { + case "mainnet": + params = chaincfg.MainNetParams + case "testnet": + params = chaincfg.TestNet3Params + case "regtest": + params = chaincfg.RegressionNetParams + } + return params +} diff --git a/claimtrie/cmd/cmd/merkletrie.go b/claimtrie/cmd/cmd/merkletrie.go new file mode 100644 index 00000000..4d6b4079 --- /dev/null +++ b/claimtrie/cmd/cmd/merkletrie.go @@ -0,0 +1,105 @@ +package cmd + +import ( + "fmt" + "path/filepath" + + "github.com/btcsuite/btcd/claimtrie/merkletrie" + "github.com/btcsuite/btcd/claimtrie/merkletrie/merkletrierepo" + "github.com/btcsuite/btcd/claimtrie/temporal/temporalrepo" + + "github.com/cockroachdb/errors" + "github.com/spf13/cobra" +) + +func init() { + rootCmd.AddCommand(NewTrieCommands()) +} + +func NewTrieCommands() *cobra.Command { + + cmd := &cobra.Command{ + Use: "trie", + Short: "MerkleTrie related commands", + } + + cmd.AddCommand(NewTrieNameCommand()) + + return cmd +} + +func NewTrieNameCommand() *cobra.Command { + + var height int32 + var name string + + cmd := &cobra.Command{ + Use: "name", + Short: "List the claim and child hashes at vertex name of block at height", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + + db, err := loadBlocksDB() + if err != nil { + return errors.Wrapf(err, "load blocks database") + } + defer db.Close() + + chain, err := loadChain(db) + if err != nil { + return errors.Wrapf(err, "load chain") + } + + state := chain.BestSnapshot() + fmt.Printf("Block %7d: %s\n", state.Height, state.Hash.String()) + + if height > state.Height { + return errors.New("requested height is unavailable") + } + + hash := state.Hash + + dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.MerkleTrieRepoPebble.Path) + log.Debugf("Open merkletrie repo: %q", dbPath) + trieRepo, err := merkletrierepo.NewPebble(dbPath) + if err != nil { + return errors.Wrapf(err, "open merkle trie repo") + } + + trie := merkletrie.NewPersistentTrie(nil, trieRepo) + defer trie.Close() + + trie.SetRoot(&hash, nil) + + if len(name) > 1 { + trie.Dump(name) + return nil + } + + dbPath = filepath.Join(dataDir, netName, "claim_dbs", cfg.TemporalRepoPebble.Path) + log.Debugf("Open temporal repo: %q", dbPath) + tmpRepo, err := temporalrepo.NewPebble(dbPath) + if err != nil { + return errors.Wrapf(err, "open temporal repo") + } + + nodes, err := tmpRepo.NodesAt(height) + if err != nil { + return errors.Wrapf(err, "read temporal repo at %d", height) + } + + for _, name := range nodes { + fmt.Printf("Name: %s, ", string(name)) + trie.Dump(string(name)) + } + + return nil + }, + } + + cmd.Flags().Int32Var(&height, "height", 0, "Height") + cmd.Flags().StringVar(&name, "name", "", "Name") + cmd.Flags().SortFlags = false + + return cmd +} diff --git a/claimtrie/cmd/cmd/node.go b/claimtrie/cmd/cmd/node.go index bcd1183b..49049916 100644 --- a/claimtrie/cmd/cmd/node.go +++ b/claimtrie/cmd/cmd/node.go @@ -4,120 +4,151 @@ import ( "fmt" "math" "path/filepath" - "strconv" "github.com/btcsuite/btcd/claimtrie/change" "github.com/btcsuite/btcd/claimtrie/node" "github.com/btcsuite/btcd/claimtrie/node/noderepo" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" ) func init() { - rootCmd.AddCommand(nodeCmd) - - nodeCmd.AddCommand(nodeDumpCmd) - nodeCmd.AddCommand(nodeReplayCmd) - nodeCmd.AddCommand(nodeChildrenCmd) + rootCmd.AddCommand(NewNodeCommands()) } -var nodeCmd = &cobra.Command{ - Use: "node", - Short: "Replay the application of changes on a node up to certain height", +func NewNodeCommands() *cobra.Command { + + cmd := &cobra.Command{ + Use: "node", + Short: "Replay the application of changes on a node up to certain height", + } + + cmd.AddCommand(NewNodeDumpCommand()) + cmd.AddCommand(NewNodeReplayCommand()) + cmd.AddCommand(NewNodeChildrenCommand()) + + return cmd } -var nodeDumpCmd = &cobra.Command{ - Use: "dump []", - Short: "Replay the application of changes on a node up to certain height", - Args: cobra.RangeArgs(1, 2), - RunE: func(cmd *cobra.Command, args []string) error { +func NewNodeDumpCommand() *cobra.Command { - repo, err := noderepo.NewPebble(filepath.Join(cfg.DataDir, cfg.NodeRepoPebble.Path)) - if err != nil { - return fmt.Errorf("open node repo: %w", err) - } + var name string + var height int32 - name := args[0] - height := math.MaxInt32 + cmd := &cobra.Command{ + Use: "dump", + Short: "Replay the application of changes on a node up to certain height", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { - if len(args) == 2 { - height, err = strconv.Atoi(args[1]) + dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.NodeRepoPebble.Path) + log.Debugf("Open node repo: %q", dbPath) + repo, err := noderepo.NewPebble(dbPath) if err != nil { - return fmt.Errorf("invalid args") + return errors.Wrapf(err, "open node repo") } - } - changes, err := repo.LoadChanges([]byte(name)) - if err != nil { - return fmt.Errorf("load commands: %w", err) - } - - for _, chg := range changes { - if int(chg.Height) > height { - break - } - showChange(chg) - } - - return nil - }, -} - -var nodeReplayCmd = &cobra.Command{ - Use: "replay []", - Short: "Replay the application of changes on a node up to certain height", - Args: cobra.RangeArgs(1, 2), - RunE: func(cmd *cobra.Command, args []string) error { - - repo, err := noderepo.NewPebble(filepath.Join(cfg.DataDir, cfg.NodeRepoPebble.Path)) - if err != nil { - return fmt.Errorf("open node repo: %w", err) - } - - name := []byte(args[0]) - height := math.MaxInt32 - - if len(args) == 2 { - height, err = strconv.Atoi(args[1]) + changes, err := repo.LoadChanges([]byte(name)) if err != nil { - return fmt.Errorf("invalid args") + return errors.Wrapf(err, "load commands") } - } - bm, err := node.NewBaseManager(repo) - if err != nil { - return fmt.Errorf("create node manager: %w", err) - } - nm := node.NewNormalizingManager(bm) + for _, chg := range changes { + if chg.Height > height { + break + } + showChange(chg) + } - n, err := nm.NodeAt(int32(height), name) - if err != nil || n == nil { - return fmt.Errorf("get node: %w", err) - } + return nil + }, + } - showNode(n) - return nil - }, + cmd.Flags().StringVar(&name, "name", "", "Name") + cmd.MarkFlagRequired("name") + cmd.Flags().Int32Var(&height, "height", math.MaxInt32, "Height") + + return cmd } -var nodeChildrenCmd = &cobra.Command{ - Use: "children ", - Short: "Show all the children names of a given node name", - Args: cobra.RangeArgs(1, 1), - RunE: func(cmd *cobra.Command, args []string) error { +func NewNodeReplayCommand() *cobra.Command { - repo, err := noderepo.NewPebble(filepath.Join(cfg.DataDir, cfg.NodeRepoPebble.Path)) - if err != nil { - return fmt.Errorf("open node repo: %w", err) - } + var name string + var height int32 - repo.IterateChildren([]byte(args[0]), func(changes []change.Change) bool { - // TODO: dump all the changes? - fmt.Printf("Name: %s, Height: %d, %d\n", changes[0].Name, changes[0].Height, - changes[len(changes)-1].Height) - return true - }) + cmd := &cobra.Command{ + Use: "replay", + Short: "Replay the changes of up to ", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { - return nil - }, + dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.NodeRepoPebble.Path) + log.Debugf("Open node repo: %q", dbPath) + repo, err := noderepo.NewPebble(dbPath) + if err != nil { + return errors.Wrapf(err, "open node repo") + } + + bm, err := node.NewBaseManager(repo) + if err != nil { + return errors.Wrapf(err, "create node manager") + } + + nm := node.NewNormalizingManager(bm) + + n, err := nm.NodeAt(height, []byte(name)) + if err != nil || n == nil { + return errors.Wrapf(err, "get node: %s", name) + } + + showNode(n) + return nil + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Name") + cmd.MarkFlagRequired("name") + cmd.Flags().Int32Var(&height, "height", 0, "Height (inclusive)") + cmd.Flags().SortFlags = false + + return cmd +} + +func NewNodeChildrenCommand() *cobra.Command { + + var name string + + cmd := &cobra.Command{ + Use: "children", + Short: "Show all the children names of a given node name", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + + dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.NodeRepoPebble.Path) + log.Debugf("Open node repo: %q", dbPath) + repo, err := noderepo.NewPebble(dbPath) + if err != nil { + return errors.Wrapf(err, "open node repo") + } + + fn := func(changes []change.Change) bool { + fmt.Printf("Name: %s, Height: %d, %d\n", changes[0].Name, changes[0].Height, + changes[len(changes)-1].Height) + return true + } + + err = repo.IterateChildren([]byte(name), fn) + if err != nil { + return errors.Wrapf(err, "iterate children: %s", name) + } + + return nil + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Name") + cmd.MarkFlagRequired("name") + + return cmd } diff --git a/claimtrie/cmd/cmd/root.go b/claimtrie/cmd/cmd/root.go index be84f5d6..698d0526 100644 --- a/claimtrie/cmd/cmd/root.go +++ b/claimtrie/cmd/cmd/root.go @@ -1,25 +1,55 @@ package cmd import ( + "os" + "github.com/btcsuite/btcd/claimtrie/config" "github.com/btcsuite/btcd/claimtrie/param" "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btclog" "github.com/spf13/cobra" ) -var cfg = config.DefaultConfig +var ( + log btclog.Logger + cfg = config.DefaultConfig + netName string + dataDir string +) -func init() { - param.SetNetwork(wire.MainNet) -} +var rootCmd = NewRootCommand() -var rootCmd = &cobra.Command{ - Use: "claimtrie", - Short: "ClaimTrie Command Line Interface", - SilenceUsage: true, +func NewRootCommand() *cobra.Command { + + cmd := &cobra.Command{ + Use: "claimtrie", + Short: "ClaimTrie Command Line Interface", + SilenceUsage: true, + PersistentPreRun: func(cmd *cobra.Command, args []string) { + switch netName { + case "mainnet": + param.SetNetwork(wire.MainNet) + case "testnet": + param.SetNetwork(wire.TestNet3) + case "regtest": + param.SetNetwork(wire.TestNet) + } + }, + } + + cmd.PersistentFlags().StringVar(&netName, "netname", "mainnet", "Net name") + cmd.PersistentFlags().StringVar(&dataDir, "datadir", cfg.DataDir, "Data dir") + + return cmd } func Execute() { + + backendLogger := btclog.NewBackend(os.Stdout) + defer os.Stdout.Sync() + log = backendLogger.Logger("CMDL") + log.SetLevel(btclog.LevelDebug) + rootCmd.Execute() // nolint : errchk } diff --git a/claimtrie/cmd/cmd/temporal.go b/claimtrie/cmd/cmd/temporal.go index 85f94850..0024deb0 100644 --- a/claimtrie/cmd/cmd/temporal.go +++ b/claimtrie/cmd/cmd/temporal.go @@ -1,62 +1,56 @@ package cmd import ( - "fmt" - "log" - "strconv" + "path/filepath" "github.com/btcsuite/btcd/claimtrie/temporal/temporalrepo" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" ) func init() { - rootCmd.AddCommand(temporalCmd) + rootCmd.AddCommand(NewTemporalCommand()) } -var temporalCmd = &cobra.Command{ - Use: "temporal []]", - Short: "List which nodes are update in a range of heights", - Args: cobra.RangeArgs(1, 2), - RunE: runListNodes, -} - -func runListNodes(cmd *cobra.Command, args []string) error { - - repo, err := temporalrepo.NewPebble(cfg.TemporalRepoPebble.Path) - if err != nil { - log.Fatalf("can't open reported block repo: %s", err) - } - - fromHeight, err := strconv.Atoi(args[0]) - if err != nil { - return fmt.Errorf("invalid args") - } - - toHeight := fromHeight + 1 - if len(args) == 2 { - toHeight, err = strconv.Atoi(args[1]) - if err != nil { - return fmt.Errorf("invalid args") - } - } - - for height := fromHeight; height < toHeight; height++ { - names, err := repo.NodesAt(int32(height)) - if err != nil { - return fmt.Errorf("get node names from temporal") - } - - if len(names) == 0 { - continue - } - - fmt.Printf("%7d: %q", height, names[0]) - for _, name := range names[1:] { - fmt.Printf(", %q ", name) - } - fmt.Printf("\n") - } - - return nil +func NewTemporalCommand() *cobra.Command { + + var fromHeight int32 + var toHeight int32 + + cmd := &cobra.Command{ + Use: "temporal", + Short: "List which nodes are update in a range of heights", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + + dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.TemporalRepoPebble.Path) + log.Debugf("Open temporal repo: %s", dbPath) + repo, err := temporalrepo.NewPebble(dbPath) + if err != nil { + return errors.Wrapf(err, "open temporal repo") + } + + for ht := fromHeight; ht < toHeight; ht++ { + names, err := repo.NodesAt(ht) + if err != nil { + return errors.Wrapf(err, "get node names from temporal") + } + + if len(names) == 0 { + continue + } + + showTemporalNames(ht, names) + } + + return nil + }, + } + + cmd.Flags().Int32Var(&fromHeight, "from", 0, "From height (inclusive)") + cmd.Flags().Int32Var(&toHeight, "to", 0, "To height (inclusive)") + cmd.Flags().SortFlags = false + + return cmd } diff --git a/claimtrie/cmd/cmd/ui.go b/claimtrie/cmd/cmd/ui.go index 28eb6fe8..b7a530a1 100644 --- a/claimtrie/cmd/cmd/ui.go +++ b/claimtrie/cmd/cmd/ui.go @@ -14,8 +14,8 @@ var status = map[node.Status]string{ node.Deactivated: "Deactivated", } -func changeName(c change.ChangeType) string { - switch c { // can't this be done via reflection? +func changeType(c change.ChangeType) string { + switch c { case change.AddClaim: return "AddClaim" case change.SpendClaim: @@ -31,8 +31,8 @@ func changeName(c change.ChangeType) string { } func showChange(chg change.Change) { - fmt.Printf(">>> Height: %6d: %s for %04s, %d, %s\n", - chg.Height, changeName(chg.Type), chg.ClaimID.String(), chg.Amount, chg.OutPoint.String()) + fmt.Printf(">>> Height: %6d: %s for %04s, %15d, %s - %s\n", + chg.Height, changeType(chg.Type), chg.ClaimID, chg.Amount, chg.OutPoint, chg.Name) } func showClaim(c *node.Claim, n *node.Node) { @@ -42,12 +42,12 @@ func showClaim(c *node.Claim, n *node.Node) { } fmt.Printf("%s C ID: %s, TXO: %s\n %5d/%-5d, Status: %9s, Amount: %15d, Support Amount: %15d\n", - mark, c.ClaimID.String(), c.OutPoint.String(), c.AcceptedAt, c.ActiveAt, status[c.Status], c.Amount, n.SupportSums[c.ClaimID.Key()]) + mark, c.ClaimID, c.OutPoint, c.AcceptedAt, c.ActiveAt, status[c.Status], c.Amount, n.SupportSums[c.ClaimID.Key()]) } func showSupport(c *node.Claim) { fmt.Printf(" S id: %s, op: %s, %5d/%-5d, %9s, amt: %15d\n", - c.ClaimID.String(), c.OutPoint.String(), c.AcceptedAt, c.ActiveAt, status[c.Status], c.Amount) + c.ClaimID, c.OutPoint, c.AcceptedAt, c.ActiveAt, status[c.Status], c.Amount) } func showNode(n *node.Node) { @@ -66,3 +66,11 @@ func showNode(n *node.Node) { } fmt.Printf("\n\n") } + +func showTemporalNames(height int32, names [][]byte) { + fmt.Printf("%7d: %q", height, names[0]) + for _, name := range names[1:] { + fmt.Printf(", %q ", name) + } + fmt.Printf("\n") +} diff --git a/go.mod b/go.mod index af85c0db..b91fe5d1 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/btcsuite/goleveldb v1.0.0 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 github.com/btcsuite/winsvc v1.0.0 + github.com/cockroachdb/errors v1.8.1 github.com/cockroachdb/pebble v0.0.0-20210525181856-e45797baeb78 github.com/davecgh/go-spew v1.1.1 github.com/decred/dcrd/lru v1.0.0