[lbry] Rework claimtrie CLIs

1. Ditch in-house block repo, and use btcd blocks database.
2. Commands switch from args to flags.
3. Revive chain recording and replaying, which will be part of CI pipeline.
4. Refactor cleanup the CLI skeletons.
5. Support DataDir, and testnet/regtes (not tested yet).

TODOs:

  1. Remove hardcoded test/development params, and pass them from flags.
  2. Make output more sensible.
  3. Add debug level flag.
  4. Add MerkleTrie implementation switch.
  5. Refactor periodic progess/status reporting for long run-time tasks.
  ...

claimtrie/claimtrie.go

@@ -247,9 +247,11 @@ func (ct *ClaimTrie) AppendBlock() error {
 		updateNames = append(updateNames, newName) // TODO: make sure using the temporalRepo batch is actually faster
 		updateHeights = append(updateHeights, nextUpdate)
 	}
-	err = ct.temporalRepo.SetNodesAt(updateNames, updateHeights)
+	if len(updateNames) != 0 {
-	if err != nil {
+		err = ct.temporalRepo.SetNodesAt(updateNames, updateHeights)
-		return errors.Wrap(err, "temporal repo set")
+		if err != nil {
+			return errors.Wrap(err, "temporal repo set")
+		}
 	}
 
 	hitFork := ct.updateTrieForHashForkIfNecessary()

claimtrie/cmd/cmd/block.go

@@ -2,157 +2,97 @@ package cmd
 
 import (
 	"fmt"
-	"log"
-	"path/filepath"
-	"strconv"
-
-	"github.com/btcsuite/btcd/claimtrie/block/blockrepo"
-	"github.com/btcsuite/btcd/claimtrie/merkletrie"
-	"github.com/btcsuite/btcd/claimtrie/merkletrie/merkletrierepo"
-	"github.com/btcsuite/btcd/claimtrie/temporal/temporalrepo"
 
+	"github.com/cockroachdb/errors"
 	"github.com/spf13/cobra"
 )
 
 func init() {
-	rootCmd.AddCommand(blockCmd)
+	rootCmd.AddCommand(NewBlocCommands())
-
-	blockCmd.AddCommand(blockLastCmd)
-	blockCmd.AddCommand(blockListCmd)
-	blockCmd.AddCommand(blockNameCmd)
 }
 
-var blockCmd = &cobra.Command{
+func NewBlocCommands() *cobra.Command {
-	Use:   "block",
+
-	Short: "Block related commands",
+	cmd := &cobra.Command{
+		Use:   "block",
+		Short: "Block related commands",
+	}
+
+	cmd.AddCommand(NewBlockBestCommand())
+	cmd.AddCommand(NewBlockListCommand())
+
+	return cmd
 }
 
-var blockLastCmd = &cobra.Command{
+func NewBlockBestCommand() *cobra.Command {
-	Use:   "last",
-	Short: "Show the Merkle Hash of the last block",
-	RunE: func(cmd *cobra.Command, args []string) error {
 
-		repo, err := blockrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ReportedBlockRepoPebble.Path))
+	cmd := &cobra.Command{
-		if err != nil {
+		Use:   "best",
-			log.Fatalf("can't open reported block repo: %s", err)
+		Short: "Show the height and hash of the best block",
-		}
+		RunE: func(cmd *cobra.Command, args []string) error {
 
-		last, err := repo.Load()
+			db, err := loadBlocksDB()
-		if err != nil {
-			return fmt.Errorf("load previous height")
-		}
-
-		hash, err := repo.Get(last)
-		if err != nil {
-			return fmt.Errorf("load changes from repo: %w", err)
-		}
-
-		fmt.Printf("blk %-7d: %s\n", last, hash.String())
-
-		return nil
-	},
-}
-
-var blockListCmd = &cobra.Command{
-	Use:   "list <from_height> [<to_height>]",
-	Short: "List the Merkle Hash of block in a range of heights",
-	Args:  cobra.RangeArgs(1, 2),
-	RunE: func(cmd *cobra.Command, args []string) error {
-
-		repo, err := blockrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ReportedBlockRepoPebble.Path))
-		if err != nil {
-			log.Fatalf("can't open reported block repo: %s", err)
-		}
-
-		fromHeight, err := strconv.Atoi(args[0])
-		if err != nil {
-			return fmt.Errorf("invalid args")
-		}
-
-		toHeight := fromHeight + 1
-		if len(args) == 2 {
-			toHeight, err = strconv.Atoi(args[1])
 			if err != nil {
-				return fmt.Errorf("invalid args")
+				return errors.Wrapf(err, "load blocks database")
 			}
-		}
+			defer db.Close()
 
-		last, err := repo.Load()
+			chain, err := loadChain(db)
-		if err != nil {
-			return fmt.Errorf("load previous height")
-		}
-
-		if toHeight >= int(last) {
-			toHeight = int(last)
-		}
-
-		for i := fromHeight; i < toHeight; i++ {
-			hash, err := repo.Get(int32(i))
 			if err != nil {
-				return fmt.Errorf("load changes from repo: %w", err)
+				return errors.Wrapf(err, "load chain")
 			}
-			fmt.Printf("blk %-7d: %s\n", i, hash.String())
-		}
 
-		return nil
+			state := chain.BestSnapshot()
-	},
+			fmt.Printf("Block %7d: %s\n", state.Height, state.Hash.String())
+
+			return nil
+		},
+	}
+
+	return cmd
 }
 
-var blockNameCmd = &cobra.Command{
+func NewBlockListCommand() *cobra.Command {
-	Use:   "vertex <height> <name>",
-	Short: "List the claim and child hashes at vertex name of block at height",
-	Args:  cobra.RangeArgs(2, 2),
-	RunE: func(cmd *cobra.Command, args []string) error {
 
-		repo, err := blockrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.BlockRepoPebble.Path))
+	var fromHeight int32
-		if err != nil {
+	var toHeight int32
-			return fmt.Errorf("can't open reported block repo: %w", err)
-		}
-		defer repo.Close()
 
-		height, err := strconv.Atoi(args[0])
+	cmd := &cobra.Command{
-		if err != nil {
+		Use:   "list",
-			return fmt.Errorf("invalid args")
+		Short: "List merkle hash of blocks between <from_height> <to_height>",
-		}
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
 
-		last, err := repo.Load()
+			db, err := loadBlocksDB()
-		if err != nil {
-			return fmt.Errorf("load previous height: %w", err)
-		}
-
-		if last < int32(height) {
-			return fmt.Errorf("requested height is unavailable")
-		}
-
-		hash, err := repo.Get(int32(height))
-		if err != nil {
-			return fmt.Errorf("load previous height: %w", err)
-		}
-
-		trieRepo, err := merkletrierepo.NewPebble(filepath.Join(cfg.DataDir, cfg.MerkleTrieRepoPebble.Path))
-		if err != nil {
-			return fmt.Errorf("can't open merkle trie repo: %w", err)
-		}
-
-		trie := merkletrie.NewPersistentTrie(nil, trieRepo)
-		defer trie.Close()
-		trie.SetRoot(hash, nil)
-		if len(args) > 1 {
-			trie.Dump(args[1])
-		} else {
-			tmpRepo, err := temporalrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.TemporalRepoPebble.Path))
 			if err != nil {
-				return fmt.Errorf("can't open temporal repo: %w", err)
+				return errors.Wrapf(err, "load blocks database")
 			}
-			nodes, err := tmpRepo.NodesAt(int32(height))
+			defer db.Close()
+
+			chain, err := loadChain(db)
 			if err != nil {
-				return fmt.Errorf("can't read temporal repo at %d: %w", height, err)
+				return errors.Wrapf(err, "load chain")
 			}
-			for _, name := range nodes {
+
-				fmt.Printf("Name: %s, ", string(name))
+			if toHeight > chain.BestSnapshot().Height {
-				trie.Dump(string(name))
+				toHeight = chain.BestSnapshot().Height
 			}
-		}
+
-		return nil
+			for ht := fromHeight; ht <= toHeight; ht++ {
-	},
+				hash, err := chain.BlockHashByHeight(ht)
+				if err != nil {
+					return errors.Wrapf(err, "load hash for %d", ht)
+				}
+				fmt.Printf("Block %7d: %s\n", ht, hash.String())
+			}
+
+			return nil
+		},
+	}
+
+	cmd.Flags().Int32Var(&fromHeight, "from", 0, "From height (inclusive)")
+	cmd.Flags().Int32Var(&toHeight, "to", 0, "To height (inclusive)")
+	cmd.Flags().SortFlags = false
+
+	return cmd
 }

claimtrie/cmd/cmd/chain.go

@@ -2,191 +2,423 @@ package cmd
 
 import (
 	"fmt"
-	"math"
 	"os"
 	"path/filepath"
-	"strconv"
+	"sync"
+	"time"
 
+	"github.com/btcsuite/btcd/blockchain"
 	"github.com/btcsuite/btcd/claimtrie"
-	"github.com/btcsuite/btcd/claimtrie/block"
-	"github.com/btcsuite/btcd/claimtrie/block/blockrepo"
 	"github.com/btcsuite/btcd/claimtrie/chain/chainrepo"
 	"github.com/btcsuite/btcd/claimtrie/change"
 	"github.com/btcsuite/btcd/claimtrie/config"
+	"github.com/btcsuite/btcd/database"
+	_ "github.com/btcsuite/btcd/database/ffldb"
+	"github.com/btcsuite/btcd/txscript"
+	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcutil"
+
+	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble"
 	"github.com/spf13/cobra"
 )
 
 func init() {
-	rootCmd.AddCommand(chainCmd)
+	rootCmd.AddCommand(NewChainCommands())
-
-	chainCmd.AddCommand(chainDumpCmd)
-	chainCmd.AddCommand(chainReplayCmd)
 }
 
-var chainCmd = &cobra.Command{
+func NewChainCommands() *cobra.Command {
-	Use:   "chain",
+
-	Short: "chain related command",
+	cmd := &cobra.Command{
+		Use:   "chain",
+		Short: "chain related command",
+	}
+
+	cmd.AddCommand(NewChainDumpCommand())
+	cmd.AddCommand(NewChainReplayCommand())
+	cmd.AddCommand(NewChainConvertCommand())
+
+	return cmd
 }
 
-var chainDumpCmd = &cobra.Command{
+func NewChainDumpCommand() *cobra.Command {
-	Use:   "dump  <fromHeight> [<toHeight>]",
-	Short: "dump changes from <fromHeight> to [<toHeight>]",
-	Args:  cobra.RangeArgs(1, 2),
-	RunE: func(cmd *cobra.Command, args []string) error {
 
-		fromHeight, err := strconv.Atoi(args[0])
+	var fromHeight int32
-		if err != nil {
+	var toHeight int32
-			return fmt.Errorf("invalid args")
-		}
 
-		toHeight := fromHeight + 1
+	cmd := &cobra.Command{
-		if len(args) == 2 {
+		Use:   "dump",
-			toHeight, err = strconv.Atoi(args[1])
+		Short: "Dump the chain changes between <fromHeight> and <toHeight>",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+
+			dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.ChainRepoPebble.Path)
+			log.Debugf("Open chain repo: %q", dbPath)
+			chainRepo, err := chainrepo.NewPebble(dbPath)
 			if err != nil {
-				return fmt.Errorf("invalid args")
+				return errors.Wrapf(err, "open chain repo")
-			}
-		}
-
-		chainRepo, err := chainrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ChainRepoPebble.Path))
-		if err != nil {
-			return fmt.Errorf("open node repo: %w", err)
-		}
-
-		for height := fromHeight; height < toHeight; height++ {
-			changes, err := chainRepo.Load(int32(height))
-			if err == pebble.ErrNotFound {
-				continue
-			}
-			if err != nil {
-				return fmt.Errorf("load commands: %w", err)
 			}
 
-			for _, chg := range changes {
+			for height := fromHeight; height <= toHeight; height++ {
-				if int(chg.Height) > height {
+				changes, err := chainRepo.Load(height)
-					break
+				if errors.Is(err, pebble.ErrNotFound) {
+					continue
 				}
-				showChange(chg)
-			}
-		}
-
-		return nil
-	},
-}
-
-var chainReplayCmd = &cobra.Command{
-	Use:   "replay <height>",
-	Short: "Replay the chain up to <height>",
-	Args:  cobra.RangeArgs(0, 1),
-	RunE: func(cmd *cobra.Command, args []string) error {
-
-		fmt.Printf("not working until we pass record flag to claimtrie\n")
-
-		fromHeight := 2
-		toHeight := int(math.MaxInt32)
-
-		var err error
-		if len(args) == 1 {
-			toHeight, err = strconv.Atoi(args[0])
-			if err != nil {
-				return fmt.Errorf("invalid args")
-			}
-		}
-
-		err = os.RemoveAll(filepath.Join(cfg.DataDir, cfg.NodeRepoPebble.Path))
-		if err != nil {
-			return fmt.Errorf("delete node repo: %w", err)
-		}
-
-		fmt.Printf("Deleted node repo\n")
-
-		chainRepo, err := chainrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ChainRepoPebble.Path))
-		if err != nil {
-			return fmt.Errorf("open change repo: %w", err)
-		}
-
-		reportedBlockRepo, err := blockrepo.NewPebble(filepath.Join(cfg.DataDir, cfg.ReportedBlockRepoPebble.Path))
-		if err != nil {
-			return fmt.Errorf("open block repo: %w", err)
-		}
-
-		cfg := config.DefaultConfig
-		ct, err := claimtrie.New(cfg)
-		if err != nil {
-			return fmt.Errorf("create claimtrie: %w", err)
-		}
-		defer ct.Close()
-
-		err = ct.ResetHeight(int32(fromHeight - 1))
-		if err != nil {
-			return fmt.Errorf("reset claimtrie height: %w", err)
-		}
-
-		for height := int32(fromHeight); height < int32(toHeight); height++ {
-
-			changes, err := chainRepo.Load(height)
-			if err == pebble.ErrNotFound {
-				// do nothing.
-			} else if err != nil {
-				return fmt.Errorf("load from change repo: %w", err)
-			}
-
-			for _, chg := range changes {
-
-				switch chg.Type {
-				case change.AddClaim:
-					err = ct.AddClaim(chg.Name, chg.OutPoint, chg.ClaimID, chg.Amount)
-
-				case change.UpdateClaim:
-					err = ct.UpdateClaim(chg.Name, chg.OutPoint, chg.Amount, chg.ClaimID)
-
-				case change.SpendClaim:
-					err = ct.SpendClaim(chg.Name, chg.OutPoint, chg.ClaimID)
-
-				case change.AddSupport:
-					err = ct.AddSupport(chg.Name, chg.OutPoint, chg.Amount, chg.ClaimID)
-
-				case change.SpendSupport:
-					err = ct.SpendSupport(chg.Name, chg.OutPoint, chg.ClaimID)
-
-				default:
-					err = fmt.Errorf("invalid change: %v", chg)
-				}
-
 				if err != nil {
-					return fmt.Errorf("execute change %v: %w", chg, err)
+					return errors.Wrapf(err, "load charnges for height: %d")
+				}
+				for _, chg := range changes {
+					showChange(chg)
 				}
 			}
-			err = appendBlock(ct, reportedBlockRepo)
-			if err != nil {
-				return err
-			}
-			if ct.Height()%1000 == 0 {
-				fmt.Printf("block: %d\n", ct.Height())
-			}
-		}
 
-		return nil
+			return nil
-	},
+		},
+	}
+
+	cmd.Flags().Int32Var(&fromHeight, "from", 0, "From height (inclusive)")
+	cmd.Flags().Int32Var(&toHeight, "to", 0, "To height (inclusive)")
+	cmd.Flags().SortFlags = false
+
+	return cmd
 }
 
-func appendBlock(ct *claimtrie.ClaimTrie, blockRepo block.Repo) error {
+func NewChainReplayCommand() *cobra.Command {
+
+	var fromHeight int32
+	var toHeight int32
+
+	cmd := &cobra.Command{
+		Use:   "replay",
+		Short: "Replay the chain changes between <fromHeight> and <toHeight>",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+
+			for _, dbName := range []string{
+				cfg.BlockRepoPebble.Path,
+				cfg.NodeRepoPebble.Path,
+				cfg.MerkleTrieRepoPebble.Path,
+				cfg.TemporalRepoPebble.Path,
+			} {
+				dbPath := filepath.Join(dataDir, netName, "claim_dbs", dbName)
+				log.Debugf("Delete repo: %q", dbPath)
+				err := os.RemoveAll(dbPath)
+				if err != nil {
+					return errors.Wrapf(err, "delete repo: %q", dbPath)
+				}
+			}
+
+			dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.ChainRepoPebble.Path)
+			log.Debugf("Open chain repo: %q", dbPath)
+			chainRepo, err := chainrepo.NewPebble(dbPath)
+			if err != nil {
+				return errors.Wrapf(err, "open chain repo")
+			}
+
+			cfg := config.DefaultConfig
+			cfg.RamTrie = true
+			cfg.DataDir = filepath.Join(dataDir, netName)
+
+			ct, err := claimtrie.New(cfg)
+			if err != nil {
+				return errors.Wrapf(err, "create claimtrie")
+			}
+			defer ct.Close()
+
+			db, err := loadBlocksDB()
+			if err != nil {
+				return errors.Wrapf(err, "load blocks database")
+			}
+
+			chain, err := loadChain(db)
+			if err != nil {
+				return errors.Wrapf(err, "load chain")
+			}
+
+			for ht := fromHeight; ht < toHeight; ht++ {
+
+				changes, err := chainRepo.Load(ht + 1)
+				if errors.Is(err, pebble.ErrNotFound) {
+					// do nothing.
+				} else if err != nil {
+					return errors.Wrapf(err, "load changes for block %d", ht)
+				}
+
+				for _, chg := range changes {
+
+					switch chg.Type {
+					case change.AddClaim:
+						err = ct.AddClaim(chg.Name, chg.OutPoint, chg.ClaimID, chg.Amount)
+					case change.UpdateClaim:
+						err = ct.UpdateClaim(chg.Name, chg.OutPoint, chg.Amount, chg.ClaimID)
+					case change.SpendClaim:
+						err = ct.SpendClaim(chg.Name, chg.OutPoint, chg.ClaimID)
+					case change.AddSupport:
+						err = ct.AddSupport(chg.Name, chg.OutPoint, chg.Amount, chg.ClaimID)
+					case change.SpendSupport:
+						err = ct.SpendSupport(chg.Name, chg.OutPoint, chg.ClaimID)
+					default:
+						err = errors.Errorf("invalid change type: %v", chg)
+					}
+
+					if err != nil {
+						return errors.Wrapf(err, "execute change %v", chg)
+					}
+				}
+				err = appendBlock(ct, chain)
+				if err != nil {
+					return errors.Wrapf(err, "appendBlock")
+				}
+
+				if ct.Height()%1000 == 0 {
+					fmt.Printf("block: %d\n", ct.Height())
+				}
+			}
+
+			return nil
+		},
+	}
+
+	// FIXME
+	cmd.Flags().Int32Var(&fromHeight, "from", 0, "From height")
+	cmd.Flags().Int32Var(&toHeight, "to", 0, "To height")
+	cmd.Flags().SortFlags = false
+
+	return cmd
+}
+
+func appendBlock(ct *claimtrie.ClaimTrie, chain *blockchain.BlockChain) error {
 
 	err := ct.AppendBlock()
 	if err != nil {
-		return fmt.Errorf("append block: %w", err)
+		return errors.Wrapf(err, "append block: %w")
 	}
 
-	height := ct.Height()
+	block, err := chain.BlockByHeight(ct.Height())
-
-	hash, err := blockRepo.Get(height)
 	if err != nil {
-		return fmt.Errorf("load from block repo: %w", err)
+		return errors.Wrapf(err, "load from block repo: %w")
 	}
+	hash := block.MsgBlock().Header.ClaimTrie
 
-	if *ct.MerkleHash() != *hash {
+	if *ct.MerkleHash() != hash {
-		return fmt.Errorf("hash mismatched at height %5d: exp: %s, got: %s", height, hash, ct.MerkleHash())
+		return errors.Errorf("hash mismatched at height %5d: exp: %s, got: %s", ct.Height(), hash, ct.MerkleHash())
 	}
 
 	return nil
 }
+
+func NewChainConvertCommand() *cobra.Command {
+
+	var height int32
+
+	cmd := &cobra.Command{
+		Use:   "convert",
+		Short: "convert changes from to <height>",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+
+			db, err := loadBlocksDB()
+			if err != nil {
+				return errors.Wrapf(err, "load block db")
+			}
+			defer db.Close()
+
+			chain, err := loadChain(db)
+			if err != nil {
+				return errors.Wrapf(err, "load block db")
+			}
+
+			converter := chainConverter{
+				db:          db,
+				chain:       chain,
+				blockChan:   make(chan *btcutil.Block, 1000),
+				changesChan: make(chan []change.Change, 1000),
+				wg:          &sync.WaitGroup{},
+			}
+
+			startTime := time.Now()
+			err = converter.start()
+			if err != nil {
+				return errors.Wrapf(err, "start Converter")
+			}
+
+			converter.wait()
+			log.Infof("Convert chain: took %s", time.Since(startTime))
+
+			return nil
+		},
+	}
+
+	cmd.Flags().Int32Var(&height, "height", 0, "Height")
+
+	return cmd
+}
+
+type chainConverter struct {
+	db    database.DB
+	chain *blockchain.BlockChain
+
+	blockChan   chan *btcutil.Block
+	changesChan chan []change.Change
+
+	wg *sync.WaitGroup
+
+	statBlocksFetched   int
+	statBlocksProcessed int
+	statChangesSaved    int
+}
+
+func (cc *chainConverter) wait() {
+	cc.wg.Wait()
+}
+
+func (cb *chainConverter) start() error {
+
+	go cb.reportStats()
+
+	cb.wg.Add(3)
+	go cb.getBlock()
+	go cb.processBlock()
+	go cb.saveChanges()
+
+	return nil
+}
+
+func (cb *chainConverter) getBlock() {
+	defer cb.wg.Done()
+	defer close(cb.blockChan)
+
+	toHeight := int32(200000)
+	fmt.Printf("blocks: %d\n", cb.chain.BestSnapshot().Height)
+
+	if toHeight > cb.chain.BestSnapshot().Height {
+		toHeight = cb.chain.BestSnapshot().Height
+
+	}
+
+	for ht := int32(0); ht < toHeight; ht++ {
+		block, err := cb.chain.BlockByHeight(ht)
+		if err != nil {
+			log.Errorf("load changes from repo: %w", err)
+			return
+		}
+		cb.statBlocksFetched++
+		cb.blockChan <- block
+	}
+}
+
+func (cb *chainConverter) processBlock() {
+	defer cb.wg.Done()
+	defer close(cb.changesChan)
+
+	view := blockchain.NewUtxoViewpoint()
+	for block := range cb.blockChan {
+		var changes []change.Change
+		for _, tx := range block.Transactions() {
+			view.AddTxOuts(tx, block.Height())
+
+			if blockchain.IsCoinBase(tx) {
+				continue
+			}
+
+			for _, txIn := range tx.MsgTx().TxIn {
+				op := txIn.PreviousOutPoint
+				e := view.LookupEntry(op)
+				if e == nil {
+					log.Criticalf("Missing input in view for %s", op.String())
+				}
+				cs, err := txscript.DecodeClaimScript(e.PkScript())
+				if err == txscript.ErrNotClaimScript {
+					continue
+				}
+				if err != nil {
+					log.Criticalf("Can't parse claim script: %s", err)
+				}
+
+				chg := change.Change{
+					Height:   block.Height(),
+					Name:     cs.Name(),
+					OutPoint: txIn.PreviousOutPoint,
+				}
+
+				switch cs.Opcode() {
+				case txscript.OP_CLAIMNAME:
+					chg.Type = change.SpendClaim
+					chg.ClaimID = change.NewClaimID(chg.OutPoint)
+				case txscript.OP_UPDATECLAIM:
+					chg.Type = change.SpendClaim
+					copy(chg.ClaimID[:], cs.ClaimID())
+				case txscript.OP_SUPPORTCLAIM:
+					chg.Type = change.SpendSupport
+					copy(chg.ClaimID[:], cs.ClaimID())
+				}
+
+				changes = append(changes, chg)
+			}
+
+			op := *wire.NewOutPoint(tx.Hash(), 0)
+			for i, txOut := range tx.MsgTx().TxOut {
+				cs, err := txscript.DecodeClaimScript(txOut.PkScript)
+				if err == txscript.ErrNotClaimScript {
+					continue
+				}
+
+				op.Index = uint32(i)
+				chg := change.Change{
+					Height:   block.Height(),
+					Name:     cs.Name(),
+					OutPoint: op,
+					Amount:   txOut.Value,
+				}
+
+				switch cs.Opcode() {
+				case txscript.OP_CLAIMNAME:
+					chg.Type = change.AddClaim
+					chg.ClaimID = change.NewClaimID(op)
+				case txscript.OP_SUPPORTCLAIM:
+					chg.Type = change.AddSupport
+					copy(chg.ClaimID[:], cs.ClaimID())
+				case txscript.OP_UPDATECLAIM:
+					chg.Type = change.UpdateClaim
+					copy(chg.ClaimID[:], cs.ClaimID())
+				}
+				changes = append(changes, chg)
+			}
+		}
+		cb.statBlocksProcessed++
+
+		if len(changes) != 0 {
+			cb.changesChan <- changes
+		}
+	}
+}
+
+func (cb *chainConverter) saveChanges() {
+	defer cb.wg.Done()
+
+	dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.ChainRepoPebble.Path)
+	chainRepo, err := chainrepo.NewPebble(dbPath)
+	if err != nil {
+		log.Errorf("open chain repo: %s", err)
+		return
+	}
+	defer chainRepo.Close()
+
+	for changes := range cb.changesChan {
+		err = chainRepo.Save(changes[0].Height, changes)
+		if err != nil {
+			log.Errorf("save to chain repo: %s", err)
+			return
+		}
+		cb.statChangesSaved++
+	}
+}
+
+func (cb *chainConverter) reportStats() {
+	tick := time.NewTicker(5 * time.Second)
+	for range tick.C {
+		log.Infof("block : %7d / %7d,  changes saved: %d",
+			cb.statBlocksFetched, cb.statBlocksProcessed, cb.statChangesSaved)
+
+	}
+}

claimtrie/cmd/cmd/helper.go

@@ -0,0 +1,62 @@
+package cmd
+
+import (
+	"path/filepath"
+	"time"
+
+	"github.com/btcsuite/btcd/blockchain"
+	"github.com/btcsuite/btcd/chaincfg"
+	"github.com/btcsuite/btcd/database"
+	"github.com/btcsuite/btcd/txscript"
+
+	"github.com/cockroachdb/errors"
+)
+
+func loadBlocksDB() (database.DB, error) {
+
+	dbPath := filepath.Join(dataDir, netName, "blocks_ffldb")
+	log.Infof("Loading blocks database: %s", dbPath)
+	db, err := database.Open("ffldb", dbPath, chainPramas().Net)
+	if err != nil {
+		return nil, errors.Wrapf(err, "open blocks database")
+	}
+
+	return db, nil
+}
+
+func loadChain(db database.DB) (*blockchain.BlockChain, error) {
+	paramsCopy := chaincfg.MainNetParams
+
+	log.Infof("Loading chain from database")
+
+	startTime := time.Now()
+	chain, err := blockchain.New(&blockchain.Config{
+		DB:          db,
+		ChainParams: &paramsCopy,
+		TimeSource:  blockchain.NewMedianTime(),
+		SigCache:    txscript.NewSigCache(1000),
+	})
+	if err != nil {
+		return nil, errors.Wrapf(err, "create blockchain")
+	}
+
+	log.Infof("Loaded chain from database (%s)", time.Since(startTime))
+
+	return chain, err
+
+}
+
+func chainPramas() chaincfg.Params {
+
+	// Make a copy so the user won't modify the global instance.
+	params := chaincfg.MainNetParams
+	switch netName {
+	case "mainnet":
+		params = chaincfg.MainNetParams
+	case "testnet":
+		params = chaincfg.TestNet3Params
+	case "regtest":
+		params = chaincfg.RegressionNetParams
+	}
+	return params
+}

claimtrie/cmd/cmd/merkletrie.go

@@ -0,0 +1,105 @@
+package cmd
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/btcsuite/btcd/claimtrie/merkletrie"
+	"github.com/btcsuite/btcd/claimtrie/merkletrie/merkletrierepo"
+	"github.com/btcsuite/btcd/claimtrie/temporal/temporalrepo"
+
+	"github.com/cockroachdb/errors"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	rootCmd.AddCommand(NewTrieCommands())
+}
+
+func NewTrieCommands() *cobra.Command {
+
+	cmd := &cobra.Command{
+		Use:   "trie",
+		Short: "MerkleTrie related commands",
+	}
+
+	cmd.AddCommand(NewTrieNameCommand())
+
+	return cmd
+}
+
+func NewTrieNameCommand() *cobra.Command {
+
+	var height int32
+	var name string
+
+	cmd := &cobra.Command{
+		Use:   "name",
+		Short: "List the claim and child hashes at vertex name of block at height",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+
+			db, err := loadBlocksDB()
+			if err != nil {
+				return errors.Wrapf(err, "load blocks database")
+			}
+			defer db.Close()
+
+			chain, err := loadChain(db)
+			if err != nil {
+				return errors.Wrapf(err, "load chain")
+			}
+
+			state := chain.BestSnapshot()
+			fmt.Printf("Block %7d: %s\n", state.Height, state.Hash.String())
+
+			if height > state.Height {
+				return errors.New("requested height is unavailable")
+			}
+
+			hash := state.Hash
+
+			dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.MerkleTrieRepoPebble.Path)
+			log.Debugf("Open merkletrie repo: %q", dbPath)
+			trieRepo, err := merkletrierepo.NewPebble(dbPath)
+			if err != nil {
+				return errors.Wrapf(err, "open merkle trie repo")
+			}
+
+			trie := merkletrie.NewPersistentTrie(nil, trieRepo)
+			defer trie.Close()
+
+			trie.SetRoot(&hash, nil)
+
+			if len(name) > 1 {
+				trie.Dump(name)
+				return nil
+			}
+
+			dbPath = filepath.Join(dataDir, netName, "claim_dbs", cfg.TemporalRepoPebble.Path)
+			log.Debugf("Open temporal repo: %q", dbPath)
+			tmpRepo, err := temporalrepo.NewPebble(dbPath)
+			if err != nil {
+				return errors.Wrapf(err, "open temporal repo")
+			}
+
+			nodes, err := tmpRepo.NodesAt(height)
+			if err != nil {
+				return errors.Wrapf(err, "read temporal repo at %d", height)
+			}
+
+			for _, name := range nodes {
+				fmt.Printf("Name: %s, ", string(name))
+				trie.Dump(string(name))
+			}
+
+			return nil
+		},
+	}
+
+	cmd.Flags().Int32Var(&height, "height", 0, "Height")
+	cmd.Flags().StringVar(&name, "name", "", "Name")
+	cmd.Flags().SortFlags = false
+
+	return cmd
+}

claimtrie/cmd/cmd/node.go

@@ -4,120 +4,151 @@ import (
 	"fmt"
 	"math"
 	"path/filepath"
-	"strconv"
 
 	"github.com/btcsuite/btcd/claimtrie/change"
 	"github.com/btcsuite/btcd/claimtrie/node"
 	"github.com/btcsuite/btcd/claimtrie/node/noderepo"
+	"github.com/cockroachdb/errors"
 
 	"github.com/spf13/cobra"
 )
 
 func init() {
-	rootCmd.AddCommand(nodeCmd)
+	rootCmd.AddCommand(NewNodeCommands())
-
-	nodeCmd.AddCommand(nodeDumpCmd)
-	nodeCmd.AddCommand(nodeReplayCmd)
-	nodeCmd.AddCommand(nodeChildrenCmd)
 }
 
-var nodeCmd = &cobra.Command{
+func NewNodeCommands() *cobra.Command {
-	Use:   "node",
+
-	Short: "Replay the application of changes on a node up to certain height",
+	cmd := &cobra.Command{
+		Use:   "node",
+		Short: "Replay the application of changes on a node up to certain height",
+	}
+
+	cmd.AddCommand(NewNodeDumpCommand())
+	cmd.AddCommand(NewNodeReplayCommand())
+	cmd.AddCommand(NewNodeChildrenCommand())
+
+	return cmd
 }
 
-var nodeDumpCmd = &cobra.Command{
+func NewNodeDumpCommand() *cobra.Command {
-	Use:   "dump <node_name> [<height>]",
-	Short: "Replay the application of changes on a node up to certain height",
-	Args:  cobra.RangeArgs(1, 2),
-	RunE: func(cmd *cobra.Command, args []string) error {
 
-		repo, err := noderepo.NewPebble(filepath.Join(cfg.DataDir, cfg.NodeRepoPebble.Path))
+	var name string
-		if err != nil {
+	var height int32
-			return fmt.Errorf("open node repo: %w", err)
-		}
 
-		name := args[0]
+	cmd := &cobra.Command{
-		height := math.MaxInt32
+		Use:   "dump",
+		Short: "Replay the application of changes on a node up to certain height",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
 
-		if len(args) == 2 {
+			dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.NodeRepoPebble.Path)
-			height, err = strconv.Atoi(args[1])
+			log.Debugf("Open node repo: %q", dbPath)
+			repo, err := noderepo.NewPebble(dbPath)
 			if err != nil {
-				return fmt.Errorf("invalid args")
+				return errors.Wrapf(err, "open node repo")
 			}
-		}
 
-		changes, err := repo.LoadChanges([]byte(name))
+			changes, err := repo.LoadChanges([]byte(name))
-		if err != nil {
-			return fmt.Errorf("load commands: %w", err)
-		}
-
-		for _, chg := range changes {
-			if int(chg.Height) > height {
-				break
-			}
-			showChange(chg)
-		}
-
-		return nil
-	},
-}
-
-var nodeReplayCmd = &cobra.Command{
-	Use:   "replay <node_name> [<height>]",
-	Short: "Replay the application of changes on a node up to certain height",
-	Args:  cobra.RangeArgs(1, 2),
-	RunE: func(cmd *cobra.Command, args []string) error {
-
-		repo, err := noderepo.NewPebble(filepath.Join(cfg.DataDir, cfg.NodeRepoPebble.Path))
-		if err != nil {
-			return fmt.Errorf("open node repo: %w", err)
-		}
-
-		name := []byte(args[0])
-		height := math.MaxInt32
-
-		if len(args) == 2 {
-			height, err = strconv.Atoi(args[1])
 			if err != nil {
-				return fmt.Errorf("invalid args")
+				return errors.Wrapf(err, "load commands")
 			}
-		}
 
-		bm, err := node.NewBaseManager(repo)
+			for _, chg := range changes {
-		if err != nil {
+				if chg.Height > height {
-			return fmt.Errorf("create node manager: %w", err)
+					break
-		}
+				}
-		nm := node.NewNormalizingManager(bm)
+				showChange(chg)
+			}
 
-		n, err := nm.NodeAt(int32(height), name)
+			return nil
-		if err != nil || n == nil {
+		},
-			return fmt.Errorf("get node: %w", err)
+	}
-		}
 
-		showNode(n)
+	cmd.Flags().StringVar(&name, "name", "", "Name")
-		return nil
+	cmd.MarkFlagRequired("name")
-	},
+	cmd.Flags().Int32Var(&height, "height", math.MaxInt32, "Height")
+
+	return cmd
 }
 
-var nodeChildrenCmd = &cobra.Command{
+func NewNodeReplayCommand() *cobra.Command {
-	Use:   "children <node_name>",
-	Short: "Show all the children names of a given node name",
-	Args:  cobra.RangeArgs(1, 1),
-	RunE: func(cmd *cobra.Command, args []string) error {
 
-		repo, err := noderepo.NewPebble(filepath.Join(cfg.DataDir, cfg.NodeRepoPebble.Path))
+	var name string
-		if err != nil {
+	var height int32
-			return fmt.Errorf("open node repo: %w", err)
-		}
 
-		repo.IterateChildren([]byte(args[0]), func(changes []change.Change) bool {
+	cmd := &cobra.Command{
-			// TODO: dump all the changes?
+		Use:   "replay",
-			fmt.Printf("Name: %s, Height: %d, %d\n", changes[0].Name, changes[0].Height,
+		Short: "Replay the changes of <name> up to <height>",
-				changes[len(changes)-1].Height)
+		Args:  cobra.NoArgs,
-			return true
+		RunE: func(cmd *cobra.Command, args []string) error {
-		})
 
-		return nil
+			dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.NodeRepoPebble.Path)
-	},
+			log.Debugf("Open node repo: %q", dbPath)
+			repo, err := noderepo.NewPebble(dbPath)
+			if err != nil {
+				return errors.Wrapf(err, "open node repo")
+			}
+
+			bm, err := node.NewBaseManager(repo)
+			if err != nil {
+				return errors.Wrapf(err, "create node manager")
+			}
+
+			nm := node.NewNormalizingManager(bm)
+
+			n, err := nm.NodeAt(height, []byte(name))
+			if err != nil || n == nil {
+				return errors.Wrapf(err, "get node: %s", name)
+			}
+
+			showNode(n)
+			return nil
+		},
+	}
+
+	cmd.Flags().StringVar(&name, "name", "", "Name")
+	cmd.MarkFlagRequired("name")
+	cmd.Flags().Int32Var(&height, "height", 0, "Height (inclusive)")
+	cmd.Flags().SortFlags = false
+
+	return cmd
+}
+
+func NewNodeChildrenCommand() *cobra.Command {
+
+	var name string
+
+	cmd := &cobra.Command{
+		Use:   "children",
+		Short: "Show all the children names of a given node name",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+
+			dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.NodeRepoPebble.Path)
+			log.Debugf("Open node repo: %q", dbPath)
+			repo, err := noderepo.NewPebble(dbPath)
+			if err != nil {
+				return errors.Wrapf(err, "open node repo")
+			}
+
+			fn := func(changes []change.Change) bool {
+				fmt.Printf("Name: %s, Height: %d, %d\n", changes[0].Name, changes[0].Height,
+					changes[len(changes)-1].Height)
+				return true
+			}
+
+			err = repo.IterateChildren([]byte(name), fn)
+			if err != nil {
+				return errors.Wrapf(err, "iterate children: %s", name)
+			}
+
+			return nil
+		},
+	}
+
+	cmd.Flags().StringVar(&name, "name", "", "Name")
+	cmd.MarkFlagRequired("name")
+
+	return cmd
 }

claimtrie/cmd/cmd/root.go

@@ -1,25 +1,55 @@
 package cmd
 
 import (
+	"os"
+
 	"github.com/btcsuite/btcd/claimtrie/config"
 	"github.com/btcsuite/btcd/claimtrie/param"
 	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btclog"
 
 	"github.com/spf13/cobra"
 )
 
-var cfg = config.DefaultConfig
+var (
+	log     btclog.Logger
+	cfg     = config.DefaultConfig
+	netName string
+	dataDir string
+)
 
-func init() {
+var rootCmd = NewRootCommand()
-	param.SetNetwork(wire.MainNet)
-}
 
-var rootCmd = &cobra.Command{
+func NewRootCommand() *cobra.Command {
-	Use:          "claimtrie",
+
-	Short:        "ClaimTrie Command Line Interface",
+	cmd := &cobra.Command{
-	SilenceUsage: true,
+		Use:          "claimtrie",
+		Short:        "ClaimTrie Command Line Interface",
+		SilenceUsage: true,
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			switch netName {
+			case "mainnet":
+				param.SetNetwork(wire.MainNet)
+			case "testnet":
+				param.SetNetwork(wire.TestNet3)
+			case "regtest":
+				param.SetNetwork(wire.TestNet)
+			}
+		},
+	}
+
+	cmd.PersistentFlags().StringVar(&netName, "netname", "mainnet", "Net name")
+	cmd.PersistentFlags().StringVar(&dataDir, "datadir", cfg.DataDir, "Data dir")
+
+	return cmd
 }
 
 func Execute() {
+
+	backendLogger := btclog.NewBackend(os.Stdout)
+	defer os.Stdout.Sync()
+	log = backendLogger.Logger("CMDL")
+	log.SetLevel(btclog.LevelDebug)
+
 	rootCmd.Execute() // nolint : errchk
 }

claimtrie/cmd/cmd/temporal.go

@@ -1,62 +1,56 @@
 package cmd
 
 import (
-	"fmt"
+	"path/filepath"
-	"log"
-	"strconv"
 
 	"github.com/btcsuite/btcd/claimtrie/temporal/temporalrepo"
 
+	"github.com/cockroachdb/errors"
 	"github.com/spf13/cobra"
 )
 
 func init() {
-	rootCmd.AddCommand(temporalCmd)
+	rootCmd.AddCommand(NewTemporalCommand())
 }
 
-var temporalCmd = &cobra.Command{
+func NewTemporalCommand() *cobra.Command {
-	Use:   "temporal <from_height> [<to_height>]]",
+
-	Short: "List which nodes are update in a range of heights",
+	var fromHeight int32
-	Args:  cobra.RangeArgs(1, 2),
+	var toHeight int32
-	RunE:  runListNodes,
+
-}
+	cmd := &cobra.Command{
-
+		Use:   "temporal",
-func runListNodes(cmd *cobra.Command, args []string) error {
+		Short: "List which nodes are update in a range of heights",
-
+		Args:  cobra.NoArgs,
-	repo, err := temporalrepo.NewPebble(cfg.TemporalRepoPebble.Path)
+		RunE: func(cmd *cobra.Command, args []string) error {
-	if err != nil {
+
-		log.Fatalf("can't open reported block repo: %s", err)
+			dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.TemporalRepoPebble.Path)
-	}
+			log.Debugf("Open temporal repo: %s", dbPath)
-
+			repo, err := temporalrepo.NewPebble(dbPath)
-	fromHeight, err := strconv.Atoi(args[0])
+			if err != nil {
-	if err != nil {
+				return errors.Wrapf(err, "open temporal repo")
-		return fmt.Errorf("invalid args")
+			}
-	}
+
-
+			for ht := fromHeight; ht < toHeight; ht++ {
-	toHeight := fromHeight + 1
+				names, err := repo.NodesAt(ht)
-	if len(args) == 2 {
+				if err != nil {
-		toHeight, err = strconv.Atoi(args[1])
+					return errors.Wrapf(err, "get node names from temporal")
-		if err != nil {
+				}
-			return fmt.Errorf("invalid args")
+
-		}
+				if len(names) == 0 {
-	}
+					continue
-
+				}
-	for height := fromHeight; height < toHeight; height++ {
+
-		names, err := repo.NodesAt(int32(height))
+				showTemporalNames(ht, names)
-		if err != nil {
+			}
-			return fmt.Errorf("get node names from temporal")
+
-		}
+			return nil
-
+		},
-		if len(names) == 0 {
+	}
-			continue
+
-		}
+	cmd.Flags().Int32Var(&fromHeight, "from", 0, "From height (inclusive)")
-
+	cmd.Flags().Int32Var(&toHeight, "to", 0, "To height (inclusive)")
-		fmt.Printf("%7d: %q", height, names[0])
+	cmd.Flags().SortFlags = false
-		for _, name := range names[1:] {
+
-			fmt.Printf(", %q ", name)
+	return cmd
-		}
-		fmt.Printf("\n")
-	}
-
-	return nil
 }

claimtrie/cmd/cmd/ui.go

@@ -14,8 +14,8 @@ var status = map[node.Status]string{
 	node.Deactivated: "Deactivated",
 }
 
-func changeName(c change.ChangeType) string {
+func changeType(c change.ChangeType) string {
-	switch c { // can't this be done via reflection?
+	switch c {
 	case change.AddClaim:
 		return "AddClaim"
 	case change.SpendClaim:
@@ -31,8 +31,8 @@ func changeName(c change.ChangeType) string {
 }
 
 func showChange(chg change.Change) {
-	fmt.Printf(">>> Height: %6d: %s for %04s, %d, %s\n",
+	fmt.Printf(">>> Height: %6d: %s for %04s, %15d, %s - %s\n",
-		chg.Height, changeName(chg.Type), chg.ClaimID.String(), chg.Amount, chg.OutPoint.String())
+		chg.Height, changeType(chg.Type), chg.ClaimID, chg.Amount, chg.OutPoint, chg.Name)
 }
 
 func showClaim(c *node.Claim, n *node.Node) {
@@ -42,12 +42,12 @@ func showClaim(c *node.Claim, n *node.Node) {
 	}
 
 	fmt.Printf("%s  C  ID: %s, TXO: %s\n   %5d/%-5d, Status: %9s, Amount: %15d, Support Amount: %15d\n",
-		mark, c.ClaimID.String(), c.OutPoint.String(), c.AcceptedAt, c.ActiveAt, status[c.Status], c.Amount, n.SupportSums[c.ClaimID.Key()])
+		mark, c.ClaimID, c.OutPoint, c.AcceptedAt, c.ActiveAt, status[c.Status], c.Amount, n.SupportSums[c.ClaimID.Key()])
 }
 
 func showSupport(c *node.Claim) {
 	fmt.Printf("    S id: %s, op: %s, %5d/%-5d, %9s, amt: %15d\n",
-		c.ClaimID.String(), c.OutPoint.String(), c.AcceptedAt, c.ActiveAt, status[c.Status], c.Amount)
+		c.ClaimID, c.OutPoint, c.AcceptedAt, c.ActiveAt, status[c.Status], c.Amount)
 }
 
 func showNode(n *node.Node) {
@@ -66,3 +66,11 @@ func showNode(n *node.Node) {
 	}
 	fmt.Printf("\n\n")
 }
+
+func showTemporalNames(height int32, names [][]byte) {
+	fmt.Printf("%7d: %q", height, names[0])
+	for _, name := range names[1:] {
+		fmt.Printf(", %q ", name)
+	}
+	fmt.Printf("\n")
+}

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/btcsuite/goleveldb v1.0.0
 	github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792
 	github.com/btcsuite/winsvc v1.0.0
+	github.com/cockroachdb/errors v1.8.1
 	github.com/cockroachdb/pebble v0.0.0-20210525181856-e45797baeb78
 	github.com/davecgh/go-spew v1.1.1
 	github.com/decred/dcrd/lru v1.0.0