From 4651558b981b8180def8560de9362fe6ba20370d Mon Sep 17 00:00:00 2001 From: Tzu-Jung Lee Date: Thu, 2 Aug 2018 22:15:08 -0700 Subject: [PATCH] wip: a few updates so far. (the code is not cleaned up yet, especially DB related part) 1. Separate claim nodes from the Trie to NodeMgr (Node Manager). The Trie is mainly responsible for rsolving the MerkleHash. The Node Manager, which manages all the claim nodes implements KeyValue interface. type KeyValue interface{ Get(Key) error Set(Key, Value) error } When the Trie traverses to the Value node, it consults the KV with the prefix to get the value, which is the Hash of Best Claim. 2. Versioined/Snapshot based/Copy-on-Write Merkle Trie. Every resolved trie node is saved to the TrieDB (leveldb) with it's Hash as Key and content as Value. The content has the following format: Char (1B) Hash (32B) {0 to 256 entries } VHash (32B) (0 or 1 entry) The nodes are immutable and content(hash)-addressable. This gives the benefit of de-dup for free. 3. The NodeManager implements Replay, and can construct any past state. After experimentng on Memento vs Replay with the real dataset on the mainnet. I decided to go with Replay (at least for now) for a few reasons: a. Concurrency and usability. In the real world scenario, the ClaimTrie is always working on the Tip of the chain to accept Claim Script, update its own state and generate the Hash. On the other hand, most of the client requests are interested in the past state with minimal number of confirmations required. With Memento, the ClaimTrie has to either: a. Pin down the node, and likely the ClaimTrie itself as well, as it doesn't have the latest state (in terms of the whole Trie) to resolve the Hash. Undo the changes and redo the changes after serving the request. b. Copy the current state of the node and rollback that node to serve the request in the background. With Replay, the ClaimTrie can simply spin a background task without any pause. The history of the nodes is immutable and read-only, so there is contention in reconstructing a node. b. Negligible performance difference. Most of the nodes only have few commands to playback. The time to playback is negligible, and will be dominated by the I/O if the node was flushed to the disk. c. Simplicity. Implementing undo saves more changes of states during the process, and has to pay much more attention to the bidding rules. --- README.md | 2 +- claim/builder.go | 65 ------ claim/claim.go | 183 ++++------------ claim/hash.go | 3 +- claim/hash_test.go | 59 ------ claim/id.go | 18 +- claim/list.go | 42 ++++ claim/memento.go | 74 ------- claim/node.go | 374 +++++++++++++++----------------- claim/node_test.go | 312 --------------------------- claim/replay.go | 135 ++++++++++++ claim/ui.go | 102 ++------- claimtrie.go | 225 +++++++------------- claimtrie_test.go | 51 ----- cmd/claimtrie/README.md | 16 +- cmd/claimtrie/main.go | 436 ++++++++++++++++++-------------------- commit.go | 39 ++++ error.go | 6 - import.go | 134 ++++++++++++ memento/memento.go | 109 ---------- nodemgr/nm.go | 155 ++++++++++++++ trie/README.md | 47 ---- trie/cmd/triesh/README.md | 171 --------------- trie/cmd/triesh/main.go | 243 --------------------- trie/commit.go | 21 -- trie/errors.go | 8 - trie/kv.go | 19 ++ trie/node.go | 100 ++------- trie/node_test.go | 139 ------------ trie/stage.go | 63 ------ trie/stage_test.go | 34 --- trie/test.go | 107 ---------- trie/trie.go | 244 +++++++++++++-------- trie/trie_test.go | 75 ------- 34 files changed, 1237 insertions(+), 2574 deletions(-) delete mode 100644 claim/builder.go delete mode 100644 claim/hash_test.go create mode 100644 claim/list.go delete mode 100644 claim/memento.go delete mode 100644 claim/node_test.go create mode 100644 claim/replay.go delete mode 100644 claimtrie_test.go create mode 100644 commit.go create mode 100644 import.go delete mode 100644 memento/memento.go create mode 100644 nodemgr/nm.go delete mode 100644 trie/README.md delete mode 100644 trie/cmd/triesh/README.md delete mode 100644 trie/cmd/triesh/main.go delete mode 100644 trie/commit.go delete mode 100644 trie/errors.go create mode 100644 trie/kv.go delete mode 100644 trie/node_test.go delete mode 100644 trie/stage.go delete mode 100644 trie/stage_test.go delete mode 100644 trie/test.go delete mode 100644 trie/trie_test.go diff --git a/README.md b/README.md index 3b882ca..06b3c40 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Refer to [claimtrie](https://github.com/lbryio/claimtrie/blob/master/cmd/claimtr ``` quote NAME: - claimtrie - A CLI tool for ClaimTrie + claimtrie - A CLI tool for LBRY ClaimTrie USAGE: main [global options] command [command options] [arguments...] diff --git a/claim/builder.go b/claim/builder.go deleted file mode 100644 index a1d8f9e..0000000 --- a/claim/builder.go +++ /dev/null @@ -1,65 +0,0 @@ -package claim - -import ( - "github.com/lbryio/claimtrie/memento" -) - -type nodeBuildable interface { - build() Node - - setMemento(mem memento.Memento) nodeBuildable - setBestClaims(claims ...Claim) nodeBuildable - setClaims(claims ...Claim) nodeBuildable - setSupports(supports ...Support) nodeBuildable - setHeight(h Height) nodeBuildable - setUpdateNext(b bool) nodeBuildable -} - -func newNodeBuilder() nodeBuildable { - return &nodeBuilder{n: NewNode()} -} - -type nodeBuilder struct{ n *Node } - -func (nb *nodeBuilder) build() Node { - return *nb.n -} - -func (nb *nodeBuilder) setMemento(mem memento.Memento) nodeBuildable { - nb.n.mem = mem - return nb -} - -func (nb *nodeBuilder) setHeight(h Height) nodeBuildable { - nb.n.height = h - return nb -} - -func (nb *nodeBuilder) setUpdateNext(b bool) nodeBuildable { - nb.n.updateNext = b - return nb -} - -func (nb *nodeBuilder) setBestClaims(claims ...Claim) nodeBuildable { - for i := range claims { - c := claims[i] // Copy value, instead of holding reference to the slice. - nb.n.bestClaims[c.ActiveAt] = &c - } - return nb -} - -func (nb *nodeBuilder) setClaims(claims ...Claim) nodeBuildable { - for i := range claims { - c := claims[i] // Copy value, instead of holding reference to the slice. - nb.n.claims = append(nb.n.claims, &c) - } - return nb -} - -func (nb *nodeBuilder) setSupports(supports ...Support) nodeBuildable { - for i := range supports { - s := supports[i] // Copy value, instead of holding reference to the slice. - nb.n.supports = append(nb.n.supports, &s) - } - return nb -} diff --git a/claim/claim.go b/claim/claim.go index d633fc1..0f3c7e7 100644 --- a/claim/claim.go +++ b/claim/claim.go @@ -1,170 +1,81 @@ package claim import ( - "sync/atomic" + "bytes" + "github.com/btcsuite/btcd/chaincfg/chainhash" "github.com/btcsuite/btcd/wire" ) type ( - // Amount ... + // Amount defines the amount in LBC. Amount int64 - // Height ... - Height int64 + // Height defines the height of a block. + Height int32 ) -// seq is a strictly increasing sequence number determine relative order between Claims and Supports. -var seq uint64 - -// New ... -func New(op wire.OutPoint, amt Amount) *Claim { - return &Claim{OutPoint: op, ID: NewID(op), Amt: amt, seq: atomic.AddUint64(&seq, 1)} +// New returns a Claim (or Support) initialized with specified op and amt. +func New(op OutPoint, amt Amount) *Claim { + return &Claim{OutPoint: op, Amt: amt} } -// Claim ... +// Claim defines a structure of a Claim (or Support). type Claim struct { - OutPoint wire.OutPoint + OutPoint OutPoint ID ID Amt Amount + Accepted Height + + // Dynamic values. EffAmt Amount - Accepted Height ActiveAt Height - - seq uint64 } -// SetOutPoint ... -func (c *Claim) SetOutPoint(op wire.OutPoint) *Claim { - c.OutPoint = op - c.ID = NewID(op) - return c -} +func (c *Claim) setOutPoint(op OutPoint) *Claim { c.OutPoint = op; return c } +func (c *Claim) setID(id ID) *Claim { c.ID = id; return c } +func (c *Claim) setAmt(amt Amount) *Claim { c.Amt = amt; return c } +func (c *Claim) setAccepted(h Height) *Claim { c.Accepted = h; return c } +func (c *Claim) setActiveAt(h Height) *Claim { c.ActiveAt = h; return c } +func (c *Claim) String() string { return claimToString(c) } -// SetAmt ... -func (c *Claim) SetAmt(amt Amount) *Claim { - c.Amt = amt - return c -} - -// SetAccepted ... -func (c *Claim) SetAccepted(h Height) *Claim { - c.Accepted = h - return c -} - -// SetActiveAt ... -func (c *Claim) SetActiveAt(h Height) *Claim { - c.ActiveAt = h - return c -} - -// String ... -func (c *Claim) String() string { - return claimToString(c) -} - -// MarshalJSON customizes the representation of JSON. -func (c *Claim) MarshalJSON() ([]byte, error) { return claimToJSON(c) } - -// NewSupport ... -func NewSupport(op wire.OutPoint, amt Amount, claimID ID) *Support { - return &Support{OutPoint: op, Amt: amt, ClaimID: claimID, seq: atomic.AddUint64(&seq, 1)} -} - -// Support ... -type Support struct { - OutPoint wire.OutPoint - ClaimID ID - Amt Amount - Accepted Height - ActiveAt Height - - seq uint64 -} - -// SetOutPoint ... -func (s *Support) SetOutPoint(op wire.OutPoint) *Support { - s.OutPoint = op - return s -} - -// SetAmt ... -func (s *Support) SetAmt(amt Amount) *Support { - s.Amt = amt - return s -} - -// SetClaimID ... -func (s *Support) SetClaimID(id ID) *Support { - s.ClaimID = id - return s -} - -// SetAccepted ... -func (s *Support) SetAccepted(h Height) *Support { - s.Accepted = h - return s -} - -// SetActiveAt ... -func (s *Support) SetActiveAt(h Height) *Support { - s.ActiveAt = h - return s -} - -// String ... -func (s *Support) String() string { - return supportToString(s) -} - -// MarshalJSON customizes the representation of JSON. -func (s *Support) MarshalJSON() ([]byte, error) { - return supportToJSON(s) -} - -type claims []*Claim - -func (cc claims) remove(op wire.OutPoint) claims { - for i, v := range cc { - if v.OutPoint != op { - continue - } - cc[i] = cc[len(cc)-1] - cc[len(cc)-1] = nil - return cc[:len(cc)-1] +func (c *Claim) expireAt() Height { + if c.Accepted >= paramExtendedClaimExpirationForkHeight { + return c.Accepted + paramExtendedClaimExpirationTime } - return cc + return c.Accepted + paramOriginalClaimExpirationTime } -func (cc claims) has(op wire.OutPoint) (*Claim, bool) { - for _, v := range cc { - if v.OutPoint == op { - return v, true - } +func isActiveAt(c *Claim, h Height) bool { + return c != nil && c.ActiveAt <= h && c.expireAt() > h +} + +func equal(a, b *Claim) bool { + if a != nil && b != nil { + return a.OutPoint == b.OutPoint } - return nil, false + return a == nil && b == nil } -type supports []*Support +// OutPoint tracks previous transaction outputs. +type OutPoint struct { + wire.OutPoint +} -func (ss supports) remove(op wire.OutPoint) supports { - for i, v := range ss { - if v.OutPoint != op { - continue - } - ss[i] = ss[len(ss)-1] - ss[len(ss)-1] = nil - return ss[:len(ss)-1] +// NewOutPoint returns a new outpoint with the provided hash and index. +func NewOutPoint(hash *chainhash.Hash, index uint32) *OutPoint { + return &OutPoint{ + *wire.NewOutPoint(hash, index), } - return ss } -func (ss supports) has(op wire.OutPoint) (*Support, bool) { - for _, v := range ss { - if v.OutPoint == op { - return v, true - } +func outPointLess(a, b OutPoint) bool { + switch cmp := bytes.Compare(a.Hash[:], b.Hash[:]); { + case cmp > 0: + return true + case cmp < 0: + return false + default: + return a.Index < b.Index } - return nil, false } diff --git a/claim/hash.go b/claim/hash.go index e4d40ca..042b887 100644 --- a/claim/hash.go +++ b/claim/hash.go @@ -6,10 +6,9 @@ import ( "strconv" "github.com/btcsuite/btcd/chaincfg/chainhash" - "github.com/btcsuite/btcd/wire" ) -func calNodeHash(op wire.OutPoint, tookover Height) *chainhash.Hash { +func calNodeHash(op OutPoint, tookover Height) *chainhash.Hash { txHash := chainhash.DoubleHashH(op.Hash[:]) nOut := []byte(strconv.Itoa(int(op.Index))) diff --git a/claim/hash_test.go b/claim/hash_test.go deleted file mode 100644 index ef017bb..0000000 --- a/claim/hash_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package claim - -import ( - "testing" - - "github.com/btcsuite/btcd/chaincfg/chainhash" - "github.com/btcsuite/btcd/wire" - "github.com/stretchr/testify/assert" -) - -func Test_calNodeHash(t *testing.T) { - type args struct { - op wire.OutPoint - h Height - } - tests := []struct { - name string - args args - want chainhash.Hash - }{ - { - name: "0-1", - args: args{ - op: wire.OutPoint{Hash: newHash("c73232a755bf015f22eaa611b283ff38100f2a23fb6222e86eca363452ba0c51"), Index: 0}, - h: 0, - }, - want: newHash("48a312fc5141ad648cb5dca99eaf221f7b1bc4d2fc559e1cde4664a46d8688a4"), - }, - { - name: "0-2", - args: args{ - op: wire.OutPoint{Hash: newHash("71c7b8d35b9a3d7ad9a1272b68972979bbd18589f1efe6f27b0bf260a6ba78fa"), Index: 1}, - h: 1, - }, - want: newHash("9132cc5ff95ae67bee79281438e7d00c25c9ec8b526174eb267c1b63a55be67c"), - }, - { - name: "0-3", - args: args{ - op: wire.OutPoint{Hash: newHash("c4fc0e2ad56562a636a0a237a96a5f250ef53495c2cb5edd531f087a8de83722"), Index: 0x12345678}, - h: 0x87654321, - }, - want: newHash("023c73b8c9179ffcd75bd0f2ed9784aab2a62647585f4b38e4af1d59cf0665d2"), - }, - { - name: "0-4", - args: args{ - op: wire.OutPoint{Hash: newHash("baf52472bd7da19fe1e35116cfb3bd180d8770ffbe3ae9243df1fb58a14b0975"), Index: 0x11223344}, - h: 0x88776655, - }, - want: newHash("6a2d40f37cb2afea3b38dea24e1532e18cade5d1dc9c2f8bd635aca2bc4ac980"), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, calNodeHash(tt.args.op, tt.args.h)) - }) - } -} diff --git a/claim/id.go b/claim/id.go index 373d360..50010c0 100644 --- a/claim/id.go +++ b/claim/id.go @@ -5,12 +5,11 @@ import ( "encoding/binary" "encoding/hex" - "github.com/btcsuite/btcd/wire" "github.com/btcsuite/btcutil" ) -// NewID ... -func NewID(op wire.OutPoint) ID { +// NewID returns a Claim ID caclculated from Ripemd160(Sha256(OUTPOINT). +func NewID(op OutPoint) ID { w := bytes.NewBuffer(op.Hash[:]) if err := binary.Write(w, binary.BigEndian, op.Index); err != nil { panic(err) @@ -20,17 +19,22 @@ func NewID(op wire.OutPoint) ID { return id } -// NewIDFromString ... +// NewIDFromString returns a Claim ID from a string. func NewIDFromString(s string) (ID, error) { - b, err := hex.DecodeString(s) var id ID - copy(id[:], b) + _, err := hex.Decode(id[:], []byte(s)) + for i, j := 0, len(id)-1; i < j; i, j = i+1, j-1 { + id[i], id[j] = id[j], id[i] + } return id, err } -// ID ... +// ID represents a Claim's ID. type ID [20]byte func (id ID) String() string { + for i, j := 0, len(id)-1; i < j; i, j = i+1, j-1 { + id[i], id[j] = id[j], id[i] + } return hex.EncodeToString(id[:]) } diff --git a/claim/list.go b/claim/list.go new file mode 100644 index 0000000..c9f1728 --- /dev/null +++ b/claim/list.go @@ -0,0 +1,42 @@ +package claim + +type list []*Claim + +type comparator func(c *Claim) bool + +func byOP(op OutPoint) comparator { + return func(c *Claim) bool { + return c.OutPoint == op + } +} + +func byID(id ID) comparator { + return func(c *Claim) bool { + return c.ID == id + } +} + +func remove(l list, cmp comparator) (list, *Claim) { + last := len(l) - 1 + for i, v := range l { + if !cmp(v) { + continue + } + removed := l[i] + l[i] = l[last] + l[last] = nil + return l[:last], removed + } + return l, nil +} + +func find(cmp comparator, lists ...list) *Claim { + for _, l := range lists { + for _, v := range l { + if cmp(v) { + return v + } + } + } + return nil +} diff --git a/claim/memento.go b/claim/memento.go deleted file mode 100644 index 5da713a..0000000 --- a/claim/memento.go +++ /dev/null @@ -1,74 +0,0 @@ -package claim - -type cmdAddClaim struct { - node *Node - claim *Claim -} - -func (c cmdAddClaim) Execute() { c.node.claims = append(c.node.claims, c.claim) } -func (c cmdAddClaim) Undo() { c.node.claims = c.node.claims.remove(c.claim.OutPoint) } - -type cmdRemoveClaim struct { - node *Node - claim *Claim -} - -func (c cmdRemoveClaim) Execute() { c.node.claims = c.node.claims.remove(c.claim.OutPoint) } -func (c cmdRemoveClaim) Undo() { c.node.claims = append(c.node.claims, c.claim) } - -type cmdAddSupport struct { - node *Node - support *Support -} - -func (c cmdAddSupport) Execute() { c.node.supports = append(c.node.supports, c.support) } -func (c cmdAddSupport) Undo() { c.node.supports = c.node.supports.remove(c.support.OutPoint) } - -type cmdRemoveSupport struct { - node *Node - support *Support -} - -func (c cmdRemoveSupport) Execute() { - c.node.supports = c.node.supports.remove(c.support.OutPoint) -} -func (c cmdRemoveSupport) Undo() { c.node.supports = append(c.node.supports, c.support) } - -type cmdUpdateClaimActiveHeight struct { - claim *Claim - old Height - new Height -} - -func (c cmdUpdateClaimActiveHeight) Execute() { c.claim.ActiveAt = c.new } -func (c cmdUpdateClaimActiveHeight) Undo() { c.claim.ActiveAt = c.old } - -type cmdUpdateSupportActiveHeight struct { - support *Support - old Height - new Height -} - -func (c cmdUpdateSupportActiveHeight) Execute() { c.support.ActiveAt = c.new } -func (c cmdUpdateSupportActiveHeight) Undo() { c.support.ActiveAt = c.old } - -type updateNodeBestClaim struct { - node *Node - height Height - old *Claim - new *Claim -} - -func (c updateNodeBestClaim) Execute() { - c.node.bestClaims[c.height] = c.new - if c.node.bestClaims[c.height] == nil { - delete(c.node.bestClaims, c.height) - } -} - -func (c updateNodeBestClaim) Undo() { - c.node.bestClaims[c.height] = c.old - if c.node.bestClaims[c.height] == nil { - delete(c.node.bestClaims, c.height) - } -} diff --git a/claim/node.go b/claim/node.go index c091392..54876ba 100644 --- a/claim/node.go +++ b/claim/node.go @@ -1,34 +1,34 @@ package claim import ( + "fmt" "math" "github.com/btcsuite/btcd/chaincfg/chainhash" - "github.com/btcsuite/btcd/wire" - - "github.com/lbryio/claimtrie/memento" + "github.com/pkg/errors" ) // Node ... type Node struct { - mem memento.Memento - height Height - bestClaims map[Height]*Claim + name string - claims claims - supports supports + height Height - updateNext bool + best *Claim + tookover Height + + claims list + supports list + + // refer to updateClaim. + removed list + + records []*Cmd } // NewNode returns a new Node. -func NewNode() *Node { - return &Node{ - mem: memento.Memento{}, - bestClaims: map[Height]*Claim{0: nil}, - claims: claims{}, - supports: supports{}, - } +func NewNode(name string) *Node { + return &Node{name: name} } // Height returns the current height. @@ -38,175 +38,168 @@ func (n *Node) Height() Height { // BestClaim returns the best claim at the current height. func (n *Node) BestClaim() *Claim { - c, _ := bestClaim(n.height, n.bestClaims) - return c + return n.best } -// Tookover returns the height at which current best claim took over. -func (n *Node) Tookover() Height { - _, since := bestClaim(n.height, n.bestClaims) - return since +// AddClaim adds a claim to the node. +func (n *Node) AddClaim(op OutPoint, amt Amount) error { + return n.execute(n.record(CmdAddClaim, op, amt, ID{})) } -// AdjustTo increments or decrements current height until it reaches the specific height. -func (n *Node) AdjustTo(h Height) error { - for n.height < h { - n.Increment() - } - for n.height > h { - n.Decrement() - } - return nil +// SpendClaim spends a claim in the node. +func (n *Node) SpendClaim(op OutPoint) error { + return n.execute(n.record(CmdSpendClaim, op, 0, ID{})) } -// Increment ... -// Increment also clears out the undone stack if it wasn't empty. -func (n *Node) Increment() error { - n.height++ - n.processBlock() - n.mem.Commit() - return nil +// UpdateClaim updates a claim in the node. +func (n *Node) UpdateClaim(op OutPoint, amt Amount, id ID) error { + return n.execute(n.record(CmdUpdateClaim, op, amt, id)) } -// Decrement ... -func (n *Node) Decrement() error { - n.height-- - n.mem.Undo() - return nil +// AddSupport adds a support in the node. +func (n *Node) AddSupport(op OutPoint, amt Amount, id ID) error { + return n.execute(n.record(CmdAddSupport, op, amt, id)) } -// Redo ... -func (n *Node) Redo() error { - if err := n.mem.Redo(); err != nil { - return err - } - n.height++ - return nil +// SpendSupport spends a spport in the node. +func (n *Node) SpendSupport(op OutPoint) error { + return n.execute(n.record(CmdSpendSupport, op, 0, ID{})) } -// RollbackExecuted ... -func (n *Node) RollbackExecuted() error { - n.mem.RollbackExecuted() - return nil -} - -// AddClaim ... -func (n *Node) AddClaim(c *Claim) error { - if _, ok := n.claims.has(c.OutPoint); ok { +func (n *Node) addClaim(op OutPoint, amt Amount) error { + if find(byOP(op), n.claims, n.supports) != nil { return ErrDuplicate } - next := n.height + 1 - c.SetAccepted(next).SetActiveAt(next) - if n.BestClaim() != nil { - c.SetActiveAt(calActiveHeight(next, next, n.Tookover())) - } - n.mem.Execute(cmdAddClaim{node: n, claim: c}) + accepted := n.height + 1 + c := New(op, amt).setID(NewID(op)).setAccepted(accepted) + c.setActiveAt(accepted + calDelay(accepted, n.tookover)) + if !isActiveAt(n.best, accepted) { + c.setActiveAt(accepted) + n.best, n.tookover = c, accepted + } + n.claims = append(n.claims, c) return nil } -// RemoveClaim ... -func (n *Node) RemoveClaim(op wire.OutPoint) error { - c, ok := n.claims.has(op) - if !ok { +func (n *Node) spendClaim(op OutPoint) error { + var c *Claim + if n.claims, c = remove(n.claims, byOP(op)); c == nil { return ErrNotFound } - n.mem.Execute(cmdRemoveClaim{node: n, claim: c}) - - if *n.BestClaim() != *c { - return nil - } - n.mem.Execute(updateNodeBestClaim{node: n, height: n.Tookover(), old: c, new: nil}) - updateActiveHeights(n.height, n.claims, n.supports, &n.mem) - n.updateNext = true + n.removed = append(n.removed, c) return nil } -// AddSupport ... -func (n *Node) AddSupport(s *Support) error { - next := n.height + 1 - s.SetAccepted(next).SetActiveAt(next) - if n.BestClaim() == nil || n.BestClaim().ID != s.ClaimID { - s.SetActiveAt(calActiveHeight(next, next, n.Tookover())) +// A claim update is composed of two separate commands (2 & 3 below). +// +// (1) blk 500: Add Claim (opA, amtA, NewID(opA) +// ... +// (2) blk 1000: Spend Claim (opA, idA) +// (3) blk 1000: Update Claim (opB, amtB, idA) +// +// For each block, all the spent claims are kept in n.removed until committed. +// The paired (spend, update) commands has to happen in the same trasaction. +func (n *Node) updateClaim(op OutPoint, amt Amount, id ID) error { + if find(byOP(op), n.claims, n.supports) != nil { + return ErrDuplicate + } + var c *Claim + if n.removed, c = remove(n.removed, byID(id)); c == nil { + return errors.Wrapf(ErrNotFound, "remove(n.removed, byID(%s)", id) } - for _, c := range n.claims { - if c.ID != s.ClaimID { - continue - } - n.mem.Execute(cmdAddSupport{node: n, support: s}) + accepted := n.height + 1 + c.setOutPoint(op).setAmt(amt).setAccepted(accepted) + c.setActiveAt(accepted + calDelay(accepted, n.tookover)) + if n.best != nil && n.best.ID == id { + c.setActiveAt(n.tookover) + } + n.claims = append(n.claims, c) + return nil +} + +func (n *Node) addSupport(op OutPoint, amt Amount, id ID) error { + if find(byOP(op), n.claims, n.supports) != nil { + return ErrDuplicate + } + // Accepted by rules. No effects on bidding result though. + // It may be spent later. + if find(byID(id), n.claims, n.removed) == nil { + fmt.Printf("INFO: can't find suooported claim ID: %s\n", id) + } + + accepted := n.height + 1 + s := New(op, amt).setID(id).setAccepted(accepted) + s.setActiveAt(accepted + calDelay(accepted, n.tookover)) + if n.best != nil && n.best.ID == id { + s.setActiveAt(accepted) + } + n.supports = append(n.supports, s) + return nil +} + +func (n *Node) spendSupport(op OutPoint) error { + var s *Claim + if n.supports, s = remove(n.supports, byOP(op)); s != nil { return nil } - - // Is supporting an non-existing Claim aceepted? return ErrNotFound } -// RemoveSupport ... -func (n *Node) RemoveSupport(op wire.OutPoint) error { - s, ok := n.supports.has(op) - if !ok { - return ErrNotFound +// NextUpdate returns the height at which pending updates should happen. +// When no pending updates exist, current height is returned. +func (n *Node) NextUpdate() Height { + next := Height(math.MaxInt32) + min := func(l list) Height { + for _, v := range l { + exp := v.expireAt() + if n.height >= exp { + continue + } + if v.ActiveAt > n.height && v.ActiveAt < next { + next = v.ActiveAt + } + if exp > n.height && exp < next { + next = exp + } + } + return next } - n.supports = n.supports.remove(op) - n.mem.Execute(cmdRemoveSupport{node: n, support: s}) - return nil -} - -// FindNextUpdateHeight returns the smallest height in the future that the the state of the node might change. -// If no such height exists, the current height of the node is returned. -func (n *Node) FindNextUpdateHeight() Height { - if n.updateNext { - n.updateNext = false - return n.height + 1 + min(n.claims) + min(n.supports) + if next == Height(math.MaxInt32) { + next = n.height } - - return findNextUpdateHeight(n.height, n.claims, n.supports) + return next } -// Hash calculates the Hash value based on the OutPoint and at which height it tookover. -func (n *Node) Hash() *chainhash.Hash { - if n.BestClaim() == nil { - return nil - } - return calNodeHash(n.BestClaim().OutPoint, n.Tookover()) -} - -// MarshalJSON customizes JSON marshaling of the Node. -func (n *Node) MarshalJSON() ([]byte, error) { - return nodeToJSON(n) -} - -// String implements Stringer interface. -func (n *Node) String() string { - return nodeToString(n) -} - -func (n *Node) processBlock() { +func (n *Node) bid() { for { - if c := n.BestClaim(); c != nil && !isActive(n.height, c.Accepted, c.ActiveAt) { - n.mem.Execute(updateNodeBestClaim{node: n, height: n.height, old: n.bestClaims[n.height], new: nil}) - updateActiveHeights(n.height, n.claims, n.supports, &n.mem) + if n.best == nil || n.height >= n.best.expireAt() { + n.best, n.tookover = nil, n.height + updateActiveHeights(n, n.claims, n.supports) } updateEffectiveAmounts(n.height, n.claims, n.supports) - candidate := findCandiadte(n.height, n.claims) - if n.BestClaim() == candidate { - return + c := findCandiadte(n.height, n.claims) + if equal(n.best, c) { + break } - n.mem.Execute(updateNodeBestClaim{node: n, height: n.height, old: n.bestClaims[n.height], new: candidate}) - updateActiveHeights(n.height, n.claims, n.supports, &n.mem) + n.best, n.tookover = c, n.height + updateActiveHeights(n, n.claims, n.supports) } + n.removed = nil } -func updateEffectiveAmounts(h Height, claims claims, supports supports) { +func updateEffectiveAmounts(h Height, claims, supports list) { for _, c := range claims { c.EffAmt = 0 - if !isActive(h, c.Accepted, c.ActiveAt) { + if !isActiveAt(c, h) { continue } c.EffAmt = c.Amt for _, s := range supports { - if !isActive(h, s.Accepted, s.ActiveAt) || s.ClaimID != c.ID { + if !isActiveAt(s, h) || s.ID != c.ID { continue } c.EffAmt += s.Amt @@ -214,88 +207,53 @@ func updateEffectiveAmounts(h Height, claims claims, supports supports) { } } -func updateActiveHeights(h Height, claims claims, supports supports, mem *memento.Memento) { - for _, v := range claims { - if old, new := v.ActiveAt, calActiveHeight(v.Accepted, h, h); old != new { - mem.Execute(cmdUpdateClaimActiveHeight{claim: v, old: old, new: new}) - } - } - for _, v := range supports { - if old, new := v.ActiveAt, calActiveHeight(v.Accepted, h, h); old != new { - mem.Execute(cmdUpdateSupportActiveHeight{support: v, old: old, new: new}) +func updateActiveHeights(n *Node, lists ...list) { + for _, l := range lists { + for _, v := range l { + v.ActiveAt = v.Accepted + calDelay(n.height, n.tookover) } } } -// bestClaim returns the best claim at specified height and since when it took over. -func bestClaim(at Height, bestClaims map[Height]*Claim) (*Claim, Height) { - var latest Height - for k := range bestClaims { - if k > at { - continue - } - if k > latest { - latest = k - } - } - return bestClaims[latest], latest -} - -func findNextUpdateHeight(h Height, claims claims, supports supports) Height { - next := Height(math.MaxInt64) - for _, v := range claims { - if v.ActiveAt > h && v.ActiveAt < next { - next = v.ActiveAt - } - } - for _, v := range supports { - if v.ActiveAt > h && v.ActiveAt < next { - next = v.ActiveAt - } - } - if next == Height(math.MaxInt64) { - return h - } - return next -} - -func findCandiadte(h Height, claims claims) *Claim { - var candidate *Claim +func findCandiadte(h Height, claims list) *Claim { + var c *Claim for _, v := range claims { switch { - case v.ActiveAt > h: + case !isActiveAt(v, h): continue - case candidate == nil: - candidate = v - case v.EffAmt > candidate.EffAmt: - candidate = v - case v.EffAmt == candidate.EffAmt && v.seq < candidate.seq: - candidate = v + case c == nil: + c = v + case v.EffAmt > c.EffAmt: + c = v + case v.EffAmt < c.EffAmt: + continue + case v.Accepted < c.Accepted: + c = v + case v.Accepted > c.Accepted: + continue + case outPointLess(c.OutPoint, v.OutPoint): + c = v } } - return candidate + return c } -func isActive(h, accepted, activeAt Height) bool { - if activeAt > h { - // Accepted, but not active yet. - return false - } - if h >= paramExtendedClaimExpirationForkHeight && accepted+paramExtendedClaimExpirationTime <= h { - // Expired on post-HF1807 duration - return false - } - if h < paramExtendedClaimExpirationForkHeight && accepted+paramOriginalClaimExpirationTime <= h { - // Expired on pre-HF1807 duration - return false - } - return true -} - -func calActiveHeight(Accepted, curr, tookover Height) Height { +func calDelay(curr, tookover Height) Height { delay := (curr - tookover) / paramActiveDelayFactor if delay > paramMaxActiveDelay { - delay = paramMaxActiveDelay + return paramMaxActiveDelay } - return Accepted + delay + return delay +} + +// Hash calculates the Hash value based on the OutPoint and when it tookover. +func (n *Node) Hash() *chainhash.Hash { + if n.best == nil { + return nil + } + return calNodeHash(n.best.OutPoint, n.tookover) +} + +func (n *Node) String() string { + return nodeToString(n) } diff --git a/claim/node_test.go b/claim/node_test.go deleted file mode 100644 index 6515601..0000000 --- a/claim/node_test.go +++ /dev/null @@ -1,312 +0,0 @@ -package claim - -import ( - "testing" - - "github.com/lbryio/claimtrie/memento" - - "github.com/btcsuite/btcd/chaincfg/chainhash" - "github.com/btcsuite/btcd/wire" - "github.com/stretchr/testify/assert" -) - -var ( - opA = wire.OutPoint{Hash: newHash("0000000000000000000000000000000011111111111111111111111111111111"), Index: 1} - opB = wire.OutPoint{Hash: newHash("0000000000000000000000000000000022222222222222222222222222222222"), Index: 2} - opC = wire.OutPoint{Hash: newHash("0000000000000000000000000000000033333333333333333333333333333333"), Index: 3} - opD = wire.OutPoint{Hash: newHash("0000000000000000000000000000000044444444444444444444444444444444"), Index: 4} - opE = wire.OutPoint{Hash: newHash("0000000000000000000000000000000555555555555555555555555555555555"), Index: 5} - opF = wire.OutPoint{Hash: newHash("0000000000000000000000000000000666666666666666666666666666666666"), Index: 6} - opX = wire.OutPoint{Hash: newHash("0000000000000000000000000000000777777777777777777777777777777777"), Index: 7} - opY = wire.OutPoint{Hash: newHash("0000000000000000000000000000000888888888888888888888888888888888"), Index: 8} - opZ = wire.OutPoint{Hash: newHash("0000000000000000000000000000000999999999999999999999999999999999"), Index: 9} - - cA = New(opA, 0) - cB = New(opB, 0) - cC = New(opC, 0) - cD = New(opD, 0) - cE = New(opE, 0) - sX = NewSupport(opX, 0, ID{}) - sY = NewSupport(opY, 0, ID{}) - sZ = NewSupport(opZ, 0, ID{}) -) - -func newHash(s string) chainhash.Hash { - h, _ := chainhash.NewHashFromStr(s) - return *h -} - -// The example on the [whitepaper](https://beta.lbry.tech/whitepaper.html) -func Test_BestClaimExample(t *testing.T) { - - SetParams(ActiveDelayFactor(32)) - defer SetParams(ResetParams()) - - n := NewNode() - at := func(h Height) *Node { - if err := n.AdjustTo(h - 1); err != nil { - panic(err) - } - return n - } - bestAt := func(at Height) *Claim { - if len(n.mem.Executed()) != 0 { - n.Increment() - n.Decrement() - } - for n.height < at { - if err := n.Redo(); err == memento.ErrCommandStackEmpty { - n.Increment() - } - } - for n.height > at { - n.Decrement() - } - return n.BestClaim() - } - - sX.SetAmt(14).SetClaimID(cA.ID) - - at(13).AddClaim(cA.SetAmt(10)) // Claim A for 10LBC is accepted. It is the first claim, so it immediately becomes active and controlling. - at(1001).AddClaim(cB.SetAmt(20)) // Claim B for 20LBC is accepted. It’s activation height is 1001+min(4032,floor(1001−1332))=1001+30=1031 - at(1010).AddSupport(sX) // Support X for 14LBC for claim A is accepted. Since it is a support for the controlling claim, it activates immediately. - at(1020).AddClaim(cC.SetAmt(50)) // Claim C for 50LBC is accepted. The activation height is 1020+min(4032,floor(1020−1332))=1020+31=1051 - at(1040).AddClaim(cD.SetAmt(300)) // Claim D for 300LBC is accepted. The activation height is 1040+min(4032,floor(1040−1332))=1040+32=1072 - - assert.Equal(t, cA, bestAt(13)) // A(10) is controlling. - assert.Equal(t, cA, bestAt(1001)) // A(10) is controlling, B(20) is accepted. - assert.Equal(t, cA, bestAt(1010)) // A(10+14) is controlling, B(20) is accepted. - assert.Equal(t, cA, bestAt(1020)) // A(10+14) is controlling, B(20) is accepted, C(50) is accepted. - - // Claim B activates. It has 20LBC, while claim A has 24LBC (10 original + 14 from support X). There is no takeover, and claim A remains controlling. - assert.Equal(t, cA, bestAt(1031)) // A(10+14) is controlling, B(20) is active, C(50) is accepted. - assert.Equal(t, cA, bestAt(1040)) //A(10+14) is controlling, B(20) is active, C(50) is accepted, D(300) is accepted. - - // Claim C activates. It has 50LBC, while claim A has 24LBC, so a takeover is initiated. - // The takeover height for this name is set to 1051, and therefore the activation delay for all the claims becomes min(4032, floor(1051−1051/32)) = 0. - // All the claims become active. - // The totals for each claim are recalculated, and claim D becomes controlling because it has the highest total. - assert.Equal(t, cD, bestAt(1051)) // A(10+14) is active, B(20) is active, C(50) is active, D(300) is controlling. -} - -func Test_BestClaim(t *testing.T) { - - SetParams(ActiveDelayFactor(1)) - defer SetParams(ResetParams()) - - n := NewNode() - at := func(h Height) *Node { - if err := n.AdjustTo(h - 1); err != nil { - panic(err) - } - return n - } - - bestAt := func(at Height) *Claim { - if len(n.mem.Executed()) != 0 { - n.Increment() - n.Decrement() - } - for n.height < at { - if err := n.Redo(); err == memento.ErrCommandStackEmpty { - n.Increment() - } - } - for n.height > at { - n.Decrement() - } - return n.BestClaim() - } - - tests := []func(t *testing.T){ - // No competing bids. - func(t *testing.T) { - at(1).AddClaim(cA.SetAmt(1)) - - assert.Equal(t, cA, bestAt(1)) - assert.Nil(t, bestAt(0)) - }, - // Competing bids inserted at the same height. - func(t *testing.T) { - at(1).AddClaim(cA.SetAmt(1)) - at(1).AddClaim(cB.SetAmt(2)) - - assert.Equal(t, cB, bestAt(1)) - assert.Nil(t, bestAt(0)) - - }, - // Two claims with the same amount. The older one wins. - func(t *testing.T) { - at(1).AddClaim(cA.SetAmt(1)) - at(2).AddClaim(cB.SetAmt(1)) - - assert.Equal(t, cA, bestAt(1)) - assert.Equal(t, cA, bestAt(2)) - assert.Equal(t, cA, bestAt(3)) - assert.Equal(t, cA, bestAt(2)) - assert.Equal(t, cA, bestAt(1)) - assert.Nil(t, bestAt(0)) - }, - // Check claim takeover. - func(t *testing.T) { - at(1).AddClaim(cA.SetAmt(1)) - at(10).AddClaim(cB.SetAmt(2)) - - assert.Equal(t, cA, bestAt(10)) - assert.Equal(t, cA, bestAt(11)) - assert.Equal(t, cB, bestAt(21)) - assert.Equal(t, cA, bestAt(11)) - assert.Equal(t, cA, bestAt(1)) - assert.Nil(t, bestAt(0)) - }, - // Spending winning claim will make losing active claim winner. - func(t *testing.T) { - at(1).AddClaim(cA.SetAmt(2)) - at(1).AddClaim(cB.SetAmt(1)) - at(2).RemoveClaim(cA.OutPoint) - - assert.Equal(t, cA, bestAt(1)) - assert.Equal(t, cB, bestAt(2)) - assert.Equal(t, cA, bestAt(1)) - assert.Nil(t, bestAt(0)) - }, - // spending winning claim will make inactive claim winner - func(t *testing.T) { - at(1).AddClaim(cA.SetAmt(2)) - at(11).AddClaim(cB.SetAmt(1)) - at(12).RemoveClaim(cA.OutPoint) - - assert.Equal(t, cA, bestAt(10)) - assert.Equal(t, cA, bestAt(11)) - assert.Equal(t, cB, bestAt(12)) - assert.Equal(t, cA, bestAt(11)) - assert.Equal(t, cA, bestAt(10)) - assert.Nil(t, bestAt(0)) - }, - // spending winning claim will empty out claim trie - func(t *testing.T) { - at(1).AddClaim(cA.SetAmt(2)) - at(2).RemoveClaim(cA.OutPoint) - - assert.Equal(t, cA, bestAt(1)) - assert.NotEqual(t, cA, bestAt(2)) - assert.Equal(t, cA, bestAt(1)) - assert.Nil(t, bestAt(0)) - }, - // check claim with more support wins - func(t *testing.T) { - at(1).AddClaim(cA.SetAmt(2)) - at(1).AddClaim(cB.SetAmt(1)) - at(1).AddSupport(sX.SetAmt(1).SetClaimID(cA.ID)) - at(1).AddSupport(sY.SetAmt(10).SetClaimID(cB.ID)) - - assert.Equal(t, cB, bestAt(1)) - assert.Equal(t, Amount(11), bestAt(1).EffAmt) - assert.Nil(t, bestAt(0)) - }, - // check support delay - func(t *testing.T) { - at(1).AddClaim(cA.SetAmt(1)) - at(1).AddClaim(cB.SetAmt(2)) - at(11).AddSupport(sX.SetAmt(10).SetClaimID(cA.ID)) - - assert.Equal(t, cB, bestAt(10)) - assert.Equal(t, Amount(2), bestAt(10).EffAmt) - assert.Equal(t, cB, bestAt(20)) - assert.Equal(t, Amount(2), bestAt(20).EffAmt) - assert.Equal(t, cA, bestAt(21)) - assert.Equal(t, Amount(11), bestAt(21).EffAmt) - assert.Equal(t, cB, bestAt(20)) - assert.Equal(t, Amount(2), bestAt(20).EffAmt) - assert.Equal(t, cB, bestAt(10)) - assert.Equal(t, Amount(2), bestAt(10).EffAmt) - assert.Nil(t, bestAt(0)) - }, - // supporting and abandoing on the same block will cause it to crash - func(t *testing.T) { - at(1).AddClaim(cA.SetAmt(2)) - at(2).AddSupport(sX.SetAmt(1).SetClaimID(cA.ID)) - at(2).RemoveClaim(cA.OutPoint) - - assert.NotEqual(t, cA, bestAt(2)) - assert.Equal(t, cA, bestAt(1)) - assert.Nil(t, bestAt(0)) - }, - // support on abandon2 - func(t *testing.T) { - at(1).AddClaim(cA.SetAmt(2)) - at(1).AddSupport(sX.SetAmt(1).SetClaimID(cA.ID)) - - // abandoning a support and abandoing claim on the same block will cause it to crash - at(2).RemoveClaim(cA.OutPoint) - at(2).RemoveSupport(sX.OutPoint) - - assert.Equal(t, cA, bestAt(1)) - assert.Nil(t, bestAt(2)) - assert.Equal(t, cA, bestAt(1)) - assert.Nil(t, bestAt(0)) - }, - // expiration - func(t *testing.T) { - SetParams(OriginalClaimExpirationTime(5)) - defer SetParams(OriginalClaimExpirationTime(DefaultOriginalClaimExpirationTime)) - - at(1).AddClaim(cA.SetAmt(2)) - at(5).AddClaim(cB.SetAmt(1)) - - assert.Equal(t, cA, bestAt(1)) - assert.Equal(t, cA, bestAt(5)) - assert.Equal(t, cB, bestAt(6)) - assert.Equal(t, cB, bestAt(7)) - assert.Equal(t, cB, bestAt(6)) - assert.Equal(t, cA, bestAt(5)) - assert.Equal(t, cA, bestAt(1)) - assert.Nil(t, bestAt(0)) - }, - - // check claims expire and is not updateable (may be changed in future soft fork) - // CMutableTransaction tx3 = fixture.MakeClaim(fixture.GetCoinbase(),"test","one",2); - // fixture.IncrementBlocks(1); - // BOOST_CHECK(is_best_claim("test",tx3)); - // fixture.IncrementBlocks(pclaimTrie->nExpirationTime); - // CMutableTransaction u1 = fixture.MakeUpdate(tx3,"test","two",ClaimIdHash(tx3.GetHash(),0) ,2); - // BOOST_CHECK(!is_best_claim("test",u1)); - - // fixture.DecrementBlocks(pclaimTrie->nExpirationTime); - // BOOST_CHECK(is_best_claim("test",tx3)); - // fixture.DecrementBlocks(1); - - // check supports expire and can cause supported bid to lose claim - // CMutableTransaction tx4 = fixture.MakeClaim(fixture.GetCoinbase(),"test","one",1); - // CMutableTransaction tx5 = fixture.MakeClaim(fixture.GetCoinbase(),"test","one",2); - // CMutableTransaction s1 = fixture.MakeSupport(fixture.GetCoinbase(),tx4,"test",2); - // fixture.IncrementBlocks(1); - // BOOST_CHECK(is_best_claim("test",tx4)); - // CMutableTransaction u2 = fixture.MakeUpdate(tx4,"test","two",ClaimIdHash(tx4.GetHash(),0) ,1); - // CMutableTransaction u3 = fixture.MakeUpdate(tx5,"test","two",ClaimIdHash(tx5.GetHash(),0) ,2); - // fixture.IncrementBlocks(pclaimTrie->nExpirationTime); - // BOOST_CHECK(is_best_claim("test",u3)); - // fixture.DecrementBlocks(pclaimTrie->nExpirationTime); - // BOOST_CHECK(is_best_claim("test",tx4)); - // fixture.DecrementBlocks(1); - - // check updated claims will extend expiration - // CMutableTransaction tx6 = fixture.MakeClaim(fixture.GetCoinbase(),"test","one",2); - // fixture.IncrementBlocks(1); - // BOOST_CHECK(is_best_claim("test",tx6)); - // CMutableTransaction u4 = fixture.MakeUpdate(tx6,"test","two",ClaimIdHash(tx6.GetHash(),0) ,2); - // fixture.IncrementBlocks(1); - // BOOST_CHECK(is_best_claim("test",u4)); - // fixture.IncrementBlocks(pclaimTrie->nExpirationTime-1); - // BOOST_CHECK(is_best_claim("test",u4)); - // fixture.IncrementBlocks(1); - // BOOST_CHECK(!is_best_claim("test",u4)); - // fixture.DecrementBlocks(1); - // BOOST_CHECK(is_best_claim("test",u4)); - // fixture.DecrementBlocks(pclaimTrie->nExpirationTime); - // BOOST_CHECK(is_best_claim("test",tx6)); - } - for _, tt := range tests { - t.Run("BestClaim", tt) - } -} diff --git a/claim/replay.go b/claim/replay.go new file mode 100644 index 0000000..afab06a --- /dev/null +++ b/claim/replay.go @@ -0,0 +1,135 @@ +package claim + +import ( + "fmt" + + "github.com/pkg/errors" +) + +type cmd int + +// ... +const ( + CmdAddClaim cmd = 1 << iota + CmdSpendClaim + CmdUpdateClaim + CmdAddSupport + CmdSpendSupport +) + +var cmdName = map[cmd]string{ + CmdAddClaim: "+C", + CmdSpendClaim: "-C", + CmdUpdateClaim: "+U", + CmdAddSupport: "+S", + CmdSpendSupport: "-S", +} + +// Cmd ... +type Cmd struct { + Height Height + Cmd cmd + Name string + OP OutPoint + Amt Amount + ID ID + Value []byte +} + +func (c Cmd) String() string { + return fmt.Sprintf("%6d %s %s %s %12d [%s]", c.Height, cmdName[c.Cmd], c.OP, c.ID, c.Amt, c.Name) +} + +func (n *Node) record(c cmd, op OutPoint, amt Amount, id ID) *Cmd { + r := &Cmd{Height: n.height + 1, Name: n.name, Cmd: c, OP: op, Amt: amt, ID: id} + n.records = append(n.records, r) + return r +} + +// AdjustTo increments current height until it reaches the specific height. +func (n *Node) AdjustTo(h Height) error { + if h < n.height { + return errors.Wrapf(ErrInvalidHeight, "adjust n.height: %d > %d", n.height, h) + } + if h == n.height { + return nil + } + for n.height < h { + n.height++ + n.bid() + next := n.NextUpdate() + if next > h { + n.height = h + break + } + n.height = next + } + n.bid() + return nil +} + +// Recall ... +func (n *Node) Recall(h Height) error { + if h >= n.height { + return errors.Wrapf(ErrInvalidHeight, "h: %d >= n.height: %d", h, n.height) + } + fmt.Printf("n.Recall from %d to %d\n", n.height, h) + err := n.replay(h, false) + return errors.Wrapf(err, "reply(%d, false)", h) +} + +// Reset rests ... +func (n *Node) Reset(h Height) error { + if h > n.height { + return nil + } + fmt.Printf("n.Reset from %d to %d\n", n.height, h) + err := n.replay(h, true) + return errors.Wrapf(err, "reply(%d, true)", h) +} + +func (n *Node) replay(h Height, truncate bool) error { + fmt.Printf("replay %s from %d to %d:\n", n.name, n.height, h) + backup := n.records + *n = *NewNode(n.name) + n.records = backup + + var i int + var r *Cmd + for i < len(n.records) { + r = n.records[i] + if n.height == r.Height-1 { + if err := n.execute(r); err != nil { + return err + } + i++ + continue + } + n.height++ + n.bid() + if n.height == h { + break + } + } + if truncate { + n.records = n.records[:i] + } + return nil +} + +func (n *Node) execute(c *Cmd) error { + var err error + switch c.Cmd { + case CmdAddClaim: + err = n.addClaim(c.OP, c.Amt) + case CmdSpendClaim: + err = n.spendClaim(c.OP) + case CmdUpdateClaim: + err = n.updateClaim(c.OP, c.Amt, c.ID) + case CmdAddSupport: + err = n.addSupport(c.OP, c.Amt, c.ID) + case CmdSpendSupport: + err = n.spendSupport(c.OP) + } + return errors.Wrapf(err, "cmd %s", c) +} diff --git a/claim/ui.go b/claim/ui.go index d141012..f252501 100644 --- a/claim/ui.go +++ b/claim/ui.go @@ -2,66 +2,44 @@ package claim import ( "bytes" - "encoding/json" "fmt" "html/template" - "sort" ) -func sortedBestClaims(n *Node) []string { - var s []string - for i := Height(0); i <= n.Tookover(); i++ { - v, ok := n.bestClaims[i] - if !ok || v == nil { - continue - } - s = append(s, fmt.Sprintf("{%d, %d}, ", i, v.OutPoint.Index)) - } - return s -} - -func sortedClaims(n *Node) []*Claim { - c := make([]*Claim, len(n.claims)) - copy(c, n.claims) - sort.Slice(c, func(i, j int) bool { return c[i].seq < c[j].seq }) - return c -} - -func sortedSupports(n *Node) []*Support { - s := make([]*Support, len(n.supports)) - copy(s, n.supports) - sort.Slice(s, func(i, j int) bool { return s[i].seq < s[j].seq }) - return s -} - func export(n *Node) interface{} { + hash := "" + if n.Hash() != nil { + hash = n.Hash().String() + } return &struct { Height Height Hash string - BestClaims []string + Tookover Height + NextUpdate Height BestClaim *Claim - Claims []*Claim - Supports []*Support + Claims list + Supports list }{ Height: n.height, - Hash: n.Hash().String(), - BestClaims: sortedBestClaims(n), - BestClaim: n.BestClaim(), - Claims: sortedClaims(n), - Supports: sortedSupports(n), + Hash: hash, + NextUpdate: n.NextUpdate(), + Tookover: n.tookover, + BestClaim: n.best, + Claims: n.claims, + Supports: n.supports, } } func nodeToString(n *Node) string { - ui := ` Height {{.Height}}, {{.Hash}} BestClaims: {{range .BestClaims}}{{.}}{{end}} + ui := ` Height {{.Height}}, {{.Hash}} Tookover: {{.Tookover}} Next: {{.NextUpdate}} {{$best := .BestClaim}} {{- if .Claims}} {{range .Claims -}} - {{.}} {{if (CMP . $best)}} {{end}} + C {{.}} {{if (CMP . $best)}} {{end}} {{end}} {{- end}} {{- if .Supports}} - {{range .Supports}}{{.}} + S {{range .Supports}}{{.}} {{end}} {{- end}}` @@ -75,49 +53,7 @@ func nodeToString(n *Node) string { return w.String() } -func nodeToJSON(n *Node) ([]byte, error) { - return json.Marshal(export(n)) -} - func claimToString(c *Claim) string { - return fmt.Sprintf("C-%-68s amt: %-3d effamt: %-3d accepted: %-3d active: %-3d id: %s", - c.OutPoint, c.Amt, c.EffAmt, c.Accepted, c.ActiveAt, c.ID) -} - -func claimToJSON(c *Claim) ([]byte, error) { - return json.Marshal(&struct { - OutPoint string - ID string - Amount Amount - EffAmount Amount - Accepted Height - ActiveAt Height - }{ - OutPoint: c.OutPoint.String(), - ID: c.ID.String(), - Amount: c.Amt, - EffAmount: c.EffAmt, - Accepted: c.Accepted, - ActiveAt: c.ActiveAt, - }) -} - -func supportToString(s *Support) string { - return fmt.Sprintf("S-%-68s amt: %-3d accepted: %-3d active: %-3d id: %s", - s.OutPoint, s.Amt, s.Accepted, s.ActiveAt, s.ClaimID) -} -func supportToJSON(s *Support) ([]byte, error) { - return json.Marshal(&struct { - OutPoint string - ClaimID string - Amount Amount - Accepted Height - ActiveAt Height - }{ - OutPoint: s.OutPoint.String(), - ClaimID: s.ClaimID.String(), - Amount: s.Amt, - Accepted: s.Accepted, - ActiveAt: s.ActiveAt, - }) + return fmt.Sprintf("%-68s id: %s accepted: %3d active: %3d, amt: %12d effamt: %3d", + c.OutPoint, c.ID, c.Accepted, c.ActiveAt, c.Amt, c.EffAmt) } diff --git a/claimtrie.go b/claimtrie.go index 3e664d9..fa3d0e9 100644 --- a/claimtrie.go +++ b/claimtrie.go @@ -1,50 +1,32 @@ package claimtrie import ( - "github.com/btcsuite/btcd/chaincfg/chainhash" - "github.com/btcsuite/btcd/wire" + "fmt" "github.com/lbryio/claimtrie/claim" - + "github.com/lbryio/claimtrie/nodemgr" "github.com/lbryio/claimtrie/trie" + + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/pkg/errors" + "github.com/syndtr/goleveldb/leveldb" ) // ClaimTrie implements a Merkle Trie supporting linear history of commits. type ClaimTrie struct { - - // The highest block number commited to the ClaimTrie. height claim.Height - - // Immutable linear history. - head *trie.Commit - - // An overlay supporting Copy-on-Write to the current tip commit. - stg *trie.Stage - - // todos tracks pending updates for future block height. - // - // A claim or support has a dynamic active peroid (ActiveAt, ExipresAt). - // This makes the state of each node dynamic as the ClaimTrie increases/decreases its height. - // Instead of polling every node for updates everytime ClaimTrie changes, the node is evaluated - // for the nearest future height it may change the states, and add that height to the todos. - // - // When a ClaimTrie at height h1 is committed with h2, the pending updates from todos (h1, h2] - // will be applied to bring the nodes up to date. - todos map[claim.Height][]string -} - -// CommitMeta implements trie.CommitMeta with commit-specific metadata. -type CommitMeta struct { - Height claim.Height + head *Commit + stg *trie.Trie + nm *nodemgr.NodeMgr } // New returns a ClaimTrie. -func New() *ClaimTrie { - mt := trie.New() +func New(dbTrie, dbNodeMgr *leveldb.DB) *ClaimTrie { + nm := nodemgr.New(dbNodeMgr) return &ClaimTrie{ - head: trie.NewCommit(nil, CommitMeta{0}, mt), - stg: trie.NewStage(mt), - todos: map[claim.Height][]string{}, + head: newCommit(nil, CommitMeta{0}, trie.EmptyTrieHash), + nm: nm, + stg: trie.New(nm, dbTrie), } } @@ -54,167 +36,118 @@ func (ct *ClaimTrie) Height() claim.Height { } // Head returns the tip commit in the commit database. -func (ct *ClaimTrie) Head() *trie.Commit { +func (ct *ClaimTrie) Head() *Commit { return ct.head } -// AddClaim adds a Claim to the Stage of ClaimTrie. -func (ct *ClaimTrie) AddClaim(name string, op wire.OutPoint, amt claim.Amount) error { - modifier := func(n *claim.Node) error { - return n.AddClaim(claim.New(op, amt)) - } - return updateNode(ct, ct.height, name, modifier) +// Trie returns the Stage of the claimtrie . +func (ct *ClaimTrie) Trie() *trie.Trie { + return ct.stg } -// AddSupport adds a Support to the Stage of ClaimTrie. -func (ct *ClaimTrie) AddSupport(name string, op wire.OutPoint, amt claim.Amount, supported claim.ID) error { - modifier := func(n *claim.Node) error { - return n.AddSupport(claim.NewSupport(op, amt, supported)) - } - return updateNode(ct, ct.height, name, modifier) +// NodeMgr returns the Node Manager of the claimtrie . +func (ct *ClaimTrie) NodeMgr() *nodemgr.NodeMgr { + return ct.nm } -// SpendClaim removes a Claim in the Stage. -func (ct *ClaimTrie) SpendClaim(name string, op wire.OutPoint) error { +// AddClaim adds a Claim to the Stage. +func (ct *ClaimTrie) AddClaim(name string, op claim.OutPoint, amt claim.Amount) error { modifier := func(n *claim.Node) error { - return n.RemoveClaim(op) + return n.AddClaim(op, amt) } - return updateNode(ct, ct.height, name, modifier) + return ct.updateNode(name, modifier) } -// SpendSupport removes a Support in the Stage. -func (ct *ClaimTrie) SpendSupport(name string, op wire.OutPoint) error { +// SpendClaim spend a Claim in the Stage. +func (ct *ClaimTrie) SpendClaim(name string, op claim.OutPoint) error { modifier := func(n *claim.Node) error { - return n.RemoveSupport(op) + return n.SpendClaim(op) } - return updateNode(ct, ct.height, name, modifier) + return ct.updateNode(name, modifier) +} + +// UpdateClaim updates a Claim in the Stage. +func (ct *ClaimTrie) UpdateClaim(name string, op claim.OutPoint, amt claim.Amount, id claim.ID) error { + modifier := func(n *claim.Node) error { + return n.UpdateClaim(op, amt, id) + } + return ct.updateNode(name, modifier) +} + +// AddSupport adds a Support to the Stage. +func (ct *ClaimTrie) AddSupport(name string, op claim.OutPoint, amt claim.Amount, id claim.ID) error { + modifier := func(n *claim.Node) error { + return n.AddSupport(op, amt, id) + } + return ct.updateNode(name, modifier) +} + +// SpendSupport spend a support in the Stage. +func (ct *ClaimTrie) SpendSupport(name string, op claim.OutPoint) error { + modifier := func(n *claim.Node) error { + return n.SpendSupport(op) + } + return ct.updateNode(name, modifier) } // Traverse visits Nodes in the Stage. -func (ct *ClaimTrie) Traverse(visit trie.Visit, update, valueOnly bool) error { - return ct.stg.Traverse(visit, update, valueOnly) +func (ct *ClaimTrie) Traverse(visit trie.Visit) error { + return ct.stg.Traverse(visit) } // MerkleHash returns the Merkle Hash of the Stage. -func (ct *ClaimTrie) MerkleHash() chainhash.Hash { +func (ct *ClaimTrie) MerkleHash() (*chainhash.Hash, error) { + // ct.nm.UpdateAll(ct.stg.Update) return ct.stg.MerkleHash() } -// Commit commits the current Stage into commit database. -// If h is lower than the current height, ErrInvalidHeight is returned. -// -// As Stage can be always cleanly reset to a specific commited snapshot, -// any error occurred during the commit would leave the Stage partially updated -// so the caller can inspect the status if interested. -// -// Changes to the ClaimTrie status, such as height or todos, are all or nothing. +// Commit commits the current Stage into database. func (ct *ClaimTrie) Commit(h claim.Height) error { - - // Already caught up. - if h <= ct.height { - return ErrInvalidHeight + if h < ct.height { + return errors.Wrapf(ErrInvalidHeight, "%d < ct.height %d", h, ct.height) } - // Apply pending updates in todos (ct.Height, h]. - // Note that ct.Height is excluded while h is included. for i := ct.height + 1; i <= h; i++ { - for _, name := range ct.todos[i] { - // dummy modifier to have the node brought up to date. - modifier := func(n *claim.Node) error { return nil } - if err := updateNode(ct, i, name, modifier); err != nil { - return err - } + if err := ct.nm.CatchUp(i, ct.stg.Update); err != nil { + return errors.Wrapf(err, "nm.CatchUp(%d, stg.Update)", i) } } - commit, err := ct.stg.Commit(ct.head, CommitMeta{Height: h}) + hash, err := ct.MerkleHash() if err != nil { - return err + return errors.Wrapf(err, "MerkleHash()") } - - // No more errors. Change the ClaimTrie status. + commit := newCommit(ct.head, CommitMeta{Height: h}, hash) ct.head = commit - for i := ct.height + 1; i <= h; i++ { - delete(ct.todos, i) - } ct.height = h + ct.stg.SetRoot(hash) return nil } -// Reset reverts the Stage to a specified commit by height. +// Reset reverts the Stage to the current or previous height specified. func (ct *ClaimTrie) Reset(h claim.Height) error { if h > ct.height { - return ErrInvalidHeight + return errors.Wrapf(ErrInvalidHeight, "%d > ct.height %d", h, ct.height) } - - // Find the most recent commit that is equal or earlier than h. + fmt.Printf("ct.Reset from %d to %d\n", ct.height, h) commit := ct.head - for commit != nil { - if commit.Meta.(CommitMeta).Height <= h { - break - } + for commit.Meta.Height > h { commit = commit.Prev } - - // The commit history is not deep enough. - if commit == nil { - return ErrInvalidHeight + if err := ct.nm.Reset(h); err != nil { + return errors.Wrapf(err, "nm.Reset(%d)", h) } - - // Drop (rollback) any uncommited change, and adjust to the specified height. - rollback := func(prefix trie.Key, value trie.Value) error { - n := value.(*claim.Node) - n.RollbackExecuted() - return n.AdjustTo(h) - } - if err := ct.stg.Traverse(rollback, true, true); err != nil { - // Rollback a node to a known state can't go wrong. - // It's a programming error that can't be recovered. - panic(err) - } - - // Update ClaimTrie status ct.head = commit ct.height = h - for k := range ct.todos { - if k >= h { - delete(ct.todos, k) - } - } - ct.stg = trie.NewStage(commit.MerkleTrie) + ct.stg.SetRoot(commit.MerkleRoot) return nil } -// updateNode implements a get-modify-set sequence to the node associated with name. -// After the modifier is applied, the node is evaluated for how soon in the -// nearest future change. And register it, if any, to the todos for the next updateNode. -func updateNode(ct *ClaimTrie, h claim.Height, name string, modifier func(n *claim.Node) error) error { - - // Get the node from the Stage, or create one if it did not exist yet. - v, err := ct.stg.Get(trie.Key(name)) - if err == trie.ErrKeyNotFound { - v = claim.NewNode() - } else if err != nil { - return err +func (ct *ClaimTrie) updateNode(name string, modifier func(n *claim.Node) error) error { + if err := ct.nm.ModifyNode(name, ct.height, modifier); err != nil { + return errors.Wrapf(err, "nm.ModifyNode(%s, %d)", name, ct.height) } - - n := v.(*claim.Node) - - // Bring the node state up to date. - if err = n.AdjustTo(h); err != nil { - return err + if err := ct.stg.Update(trie.Key(name)); err != nil { + return errors.Wrapf(err, "stg.Update(%s)", name) } - - // Apply the modifier on the node. - if err = modifier(n); err != nil { - return err - } - - // Register pending update, if any, for future height. - next := n.FindNextUpdateHeight() - if next > h { - ct.todos[next] = append(ct.todos[next], name) - } - - // Store the modified value back to the Stage, clearing out all the Merkle Hash on the path. - return ct.stg.Update(trie.Key(name), n) + return nil } diff --git a/claimtrie_test.go b/claimtrie_test.go deleted file mode 100644 index b9f2c25..0000000 --- a/claimtrie_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package claimtrie - -import ( - "testing" - - "github.com/btcsuite/btcd/chaincfg/chainhash" - "github.com/btcsuite/btcd/wire" - - "github.com/lbryio/claimtrie/claim" -) - -// pending ... -func TestClaimTrie_Commit(t *testing.T) { - ct := New() - - tests := []struct { - name string - curr claim.Height - amt claim.Amount - want chainhash.Hash - }{ - {name: "0-0", curr: 5, amt: 11}, - {name: "0-0", curr: 6, amt: 10}, - {name: "0-0", curr: 7, amt: 14}, - {name: "0-0", curr: 8, amt: 18}, - {name: "0-0", curr: 100, amt: 0}, - {name: "0-0", curr: 101, amt: 30}, - {name: "0-0", curr: 102, amt: 00}, - {name: "0-0", curr: 103, amt: 00}, - {name: "0-0", curr: 104, amt: 00}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if tt.amt != 0 { - ct.AddClaim("HELLO", *newOutPoint(0), tt.amt) - } - ct.Commit(tt.curr) - // fmt.Printf("ct.Merkle[%2d]: %s, amt: %d\n", ct.BestBlock(), ct.MerkleHash(), tt.amt) - }) - } -} - -func newOutPoint(idx int) *wire.OutPoint { - // var h chainhash.Hash - // if _, err := rand.Read(h[:]); err != nil { - // return nil - // } - // return wire.NewOutPoint(&h, uint32(idx)) - return wire.NewOutPoint(new(chainhash.Hash), uint32(idx)) -} diff --git a/cmd/claimtrie/README.md b/cmd/claimtrie/README.md index 65942d9..a3a99a7 100644 --- a/cmd/claimtrie/README.md +++ b/cmd/claimtrie/README.md @@ -8,9 +8,9 @@ coming soon ## Usage -``` bash +``` block NAME: - claimtrie - A CLI tool for ClaimTrie + claimtrie - A CLI tool for LBRY ClaimTrie USAGE: main [global options] command [command options] [arguments...] @@ -49,7 +49,7 @@ go run ${GOPATH}/src/github.com/lbryio/claimtrie/cmd/claimtrie/main.go sh Adding claims. -``` bash +``` block claimtrie > add-claim claimtrie > show @@ -90,14 +90,14 @@ claimtrie > commit Commit another claim. -```bash +``` block claimtrie > add-claim --amount 100 claimtrie > commit ``` Show logs -``` bash +``` block claimtrie > log height: 2, commit 9e2a2cf0e7f2a60e195ce46b261d6a953a3cbb68ef6b3274543ec8fdbf8a171b @@ -107,7 +107,7 @@ height: 0, commit 00000000000000000000000000000000000000000000000000000000000000 Show current status. -```bash +``` block claimtrie > show Hello : { @@ -154,7 +154,7 @@ Hello : { Reset the history to height 1. -``` bash +``` block claimtrie > reset --height 1 claimtrie > show @@ -302,4 +302,4 @@ Our PGP key is [here](https://keybase.io/lbry/key.asc) if you need it. ## Contact -The primary contact for this project is [@roylee17](https://github.com/roylee) (roylee@lbry.io) \ No newline at end of file +The primary contact for this project is [@roylee17](https://github.com/roylee17) (roylee@lbry.io) \ No newline at end of file diff --git a/cmd/claimtrie/main.go b/cmd/claimtrie/main.go index a18d323..153183a 100644 --- a/cmd/claimtrie/main.go +++ b/cmd/claimtrie/main.go @@ -3,102 +3,133 @@ package main import ( "bufio" "crypto/rand" - "encoding/json" "errors" "fmt" + "log" "math/big" "os" "os/signal" - "strconv" + "path/filepath" "strings" - "syscall" - - "github.com/btcsuite/btcd/chaincfg/chainhash" - "github.com/btcsuite/btcd/wire" - - "github.com/urfave/cli" "github.com/lbryio/claimtrie" "github.com/lbryio/claimtrie/claim" "github.com/lbryio/claimtrie/trie" + + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcutil" + "github.com/syndtr/goleveldb/leveldb" + "github.com/urfave/cli" ) var ( - flagAll = cli.BoolFlag{Name: "all, a", Usage: "apply to non-value nodes"} - flagAmount = cli.Int64Flag{Name: "amount, a", Usage: "Amount"} + ct *claimtrie.ClaimTrie + + defaultHomeDir = btcutil.AppDataDir("lbrycrd.go", false) + defaultDataDir = filepath.Join(defaultHomeDir, "data") + dbTriePath = filepath.Join(defaultDataDir, "dbTrie") +) + +var ( + all bool + chk bool + name string + height claim.Height + amt claim.Amount + op claim.OutPoint + id claim.ID +) + +var ( + flagAll = cli.BoolFlag{Name: "all, a", Usage: "Show all nodes", Destination: &all} + flagCheck = cli.BoolFlag{Name: "chk, c", Usage: "Check Merkle Hash during importing", Destination: &chk} + flagAmount = cli.Int64Flag{Name: "amount, a", Usage: "Amount", Destination: (*int64)(&amt)} flagHeight = cli.Int64Flag{Name: "height, ht", Usage: "Height"} - flagName = cli.StringFlag{Name: "name, n", Value: "Hello", Usage: "Name"} + flagName = cli.StringFlag{Name: "name, n", Value: "Hello", Usage: "Name", Destination: &name} flagID = cli.StringFlag{Name: "id", Usage: "Claim ID"} flagOutPoint = cli.StringFlag{Name: "outpoint, op", Usage: "Outpoint. (HASH:INDEX)"} - flagJSON = cli.BoolFlag{Name: "json, j", Usage: "Show Claim / Support in JSON format."} ) var ( errNotImplemented = errors.New("not implemented") - errInvalidHeight = errors.New("invalid height") - errCommitNotFound = errors.New("commit not found") + errHeight = errors.New("invalid height") ) func main() { app := cli.NewApp() - app.Name = "claimtrie" - app.Usage = "A CLI tool for ClaimTrie" + app.Usage = "A CLI tool for LBRY ClaimTrie" app.Version = "0.0.1" app.Action = cli.ShowAppHelp app.Commands = []cli.Command{ { Name: "add-claim", Aliases: []string{"ac"}, - Usage: "Claim a name with specified amount. (outPoint is generated randomly, if unspecified)", + Usage: "Claim a name.", + Before: parseArgs, Action: cmdAddClaim, - Flags: []cli.Flag{flagName, flagOutPoint, flagAmount, flagHeight}, - }, - { - Name: "add-support", - Aliases: []string{"as"}, - Usage: "Add support to a specified Claim. (outPoint is generated randomly, if unspecified)", - Action: cmdAddSupport, - Flags: []cli.Flag{flagName, flagOutPoint, flagAmount, flagHeight, flagID}, + Flags: []cli.Flag{flagName, flagOutPoint, flagAmount}, }, { Name: "spend-claim", Aliases: []string{"sc"}, - Usage: "Spend a specified Claim.", + Usage: "Spend a Claim.", + Before: parseArgs, Action: cmdSpendClaim, Flags: []cli.Flag{flagName, flagOutPoint}, }, + { + Name: "update-claim", + Aliases: []string{"uc"}, + Usage: "Update a Claim.", + Before: parseArgs, + Action: cmdUpdateClaim, + Flags: []cli.Flag{flagName, flagOutPoint, flagAmount, flagID}, + }, + { + Name: "add-support", + Aliases: []string{"as"}, + Usage: "Support a Claim.", + Before: parseArgs, + Action: cmdAddSupport, + Flags: []cli.Flag{flagName, flagOutPoint, flagAmount, flagID}, + }, { Name: "spend-support", Aliases: []string{"ss"}, Usage: "Spend a specified Support.", + Before: parseArgs, Action: cmdSpendSupport, Flags: []cli.Flag{flagName, flagOutPoint}, }, { Name: "show", Aliases: []string{"s"}, - Usage: "Show the Key-Value pairs of the Stage or specified commit. (links nodes are showed if -a is also specified)", + Usage: "Show the status of Stage)", + Before: parseArgs, Action: cmdShow, - Flags: []cli.Flag{flagAll, flagJSON, flagHeight}, + Flags: []cli.Flag{flagAll, flagName, flagHeight}, }, { Name: "merkle", Aliases: []string{"m"}, Usage: "Show the Merkle Hash of the Stage.", + Before: parseArgs, Action: cmdMerkle, }, { Name: "commit", Aliases: []string{"c"}, - Usage: "Commit the current Stage to commit database.", + Usage: "Commit the current Stage to database.", + Before: parseArgs, Action: cmdCommit, Flags: []cli.Flag{flagHeight}, }, { Name: "reset", Aliases: []string{"r"}, - Usage: "Reset the Stage to a specified commit.", + Usage: "Reset the Head commit and Stage to a specified commit.", + Before: parseArgs, Action: cmdReset, Flags: []cli.Flag{flagHeight}, }, @@ -106,255 +137,104 @@ func main() { Name: "log", Aliases: []string{"l"}, Usage: "List the commits in the coommit database.", + Before: parseArgs, Action: cmdLog, }, + { + Name: "load", + Aliases: []string{"ld"}, + Usage: "Load prerecorded command from datbase.", + Before: parseArgs, + Action: cmdLoad, + Flags: []cli.Flag{flagHeight, flagCheck}, + }, { Name: "shell", Aliases: []string{"sh"}, Usage: "Enter interactive mode", + Before: parseArgs, Action: func(c *cli.Context) { cmdShell(app) }, }, } + dbTrie, err := leveldb.OpenFile(dbTriePath, nil) + if err != nil { + log.Fatalf("can't open dbTrie at %s, err: %s\n", dbTriePath, err) + } + fmt.Printf("dbTriePath: %q\n", dbTriePath) + ct = claimtrie.New(dbTrie, nil) if err := app.Run(os.Args); err != nil { fmt.Printf("error: %s\n", err) } } -func randInt(min, max int64) int64 { - i, err := rand.Int(rand.Reader, big.NewInt(100)) - if err != nil { - panic(err) - } - return min + i.Int64() -} - -var ct = claimtrie.New() - -// newOutPoint generates random OutPoint for the ease of testing. -func newOutPoint(s string) (*wire.OutPoint, error) { - if len(s) == 0 { - var h chainhash.Hash - if _, err := rand.Read(h[:]); err != nil { - return nil, err - } - return wire.NewOutPoint(&h, uint32(h[0])), nil - } - fields := strings.Split(s, ":") - if len(fields) != 2 { - return nil, fmt.Errorf("invalid outpoint format (HASH:INDEX)") - } - h, err := chainhash.NewHashFromStr(fields[0]) - if err != nil { - return nil, err - } - idx, err := strconv.Atoi(fields[1]) - if err != nil { - return nil, err - } - return wire.NewOutPoint(h, uint32(idx)), nil -} - -type args struct { - *cli.Context - err error -} - -func (a *args) amount() claim.Amount { - if a.err != nil { - return 0 - } - amt := a.Int64("amount") - if !a.IsSet("amount") { - amt = randInt(1, 500) - } - return claim.Amount(amt) -} - -func (a *args) outPoint() wire.OutPoint { - if a.err != nil { - return wire.OutPoint{} - } - op, err := newOutPoint(a.String("outpoint")) - a.err = err - - return *op -} - -func (a *args) name() (name string) { - if a.err != nil { - return - } - return a.String("name") -} - -func (a *args) id() (id claim.ID) { - if a.err != nil { - return - } - if !a.IsSet("id") { - a.err = fmt.Errorf("flag -id is required") - return - } - id, a.err = claim.NewIDFromString(a.String("id")) - return -} - -func (a *args) height() (h claim.Height, ok bool) { - if a.err != nil { - return 0, false - } - return claim.Height(a.Int64("height")), a.IsSet("height") -} - -func (a *args) json() bool { - if a.err != nil { - return false - } - return a.IsSet("json") -} - -func (a *args) all() bool { - if a.err != nil { - return false - } - return a.Bool("all") -} - -var showNode = func(showJSON bool) trie.Visit { - return func(prefix trie.Key, val trie.Value) error { - if val == nil || val.Hash() == nil { - fmt.Printf("%-8s:\n", prefix) - return nil - } - if !showJSON { - fmt.Printf("%-8s: %v\n", prefix, val) - return nil - } - b, err := json.MarshalIndent(val, "", " ") - if err != nil { - return err - } - fmt.Printf("%-8s: %s\n", prefix, b) - return nil - } -} - -var recall = func(h claim.Height, visit trie.Visit) trie.Visit { - return func(prefix trie.Key, val trie.Value) (err error) { - n := val.(*claim.Node) - for err == nil && n.Height() > h { - err = n.Decrement() - } - if err == nil { - err = visit(prefix, val) - } - for err == nil && n.Height() < ct.Height() { - err = n.Redo() - } - return err - } -} - func cmdAddClaim(c *cli.Context) error { - a := args{Context: c} - amt := a.amount() - op := a.outPoint() - name := a.name() - if a.err != nil { - return a.err - } return ct.AddClaim(name, op, amt) } +func cmdSpendClaim(c *cli.Context) error { + return ct.SpendClaim(name, op) +} + +func cmdUpdateClaim(c *cli.Context) error { + if !c.IsSet("id") { + return fmt.Errorf("flag id is required") + } + return ct.UpdateClaim(name, op, amt, id) +} + func cmdAddSupport(c *cli.Context) error { - a := args{Context: c} - name := a.name() - amt := a.amount() - op := a.outPoint() - id := a.id() - if a.err != nil { - return a.err + if !c.IsSet("id") { + return fmt.Errorf("flag id is required") } return ct.AddSupport(name, op, amt, id) } -func cmdSpendClaim(c *cli.Context) error { - a := args{Context: c} - name := a.name() - op := a.outPoint() - if a.err != nil { - return a.err - } - return ct.SpendClaim(name, op) -} - func cmdSpendSupport(c *cli.Context) error { - a := args{Context: c} - name := a.name() - op := a.outPoint() - if a.err != nil { - return a.err - } return ct.SpendSupport(name, op) } func cmdShow(c *cli.Context) error { - a := args{Context: c} - h, setHeight := a.height() - setJSON := a.json() - setAll := a.all() - if a.err != nil { - return a.err - } - if h > ct.Height() { - return errInvalidHeight - } - visit := showNode(setJSON) - if !setHeight { - fmt.Printf("\n\n\n", ct.Height()) - return ct.Traverse(visit, false, !setAll) + fmt.Printf("\n\n\n", ct.Height()) + if all { + name = "" } + return ct.NodeMgr().Show(name) - visit = recall(h, visit) - for commit := ct.Head(); commit != nil; commit = commit.Prev { - meta := commit.Meta.(claimtrie.CommitMeta) - if h == meta.Height { - fmt.Printf("\n\n\n", h) - return commit.MerkleTrie.Traverse(visit, false, !setAll) - } - } - - return errCommitNotFound + // fmt.Printf("\n\n\n", ct.Height()) + // return ct.Traverse(showNode()) } func cmdMerkle(c *cli.Context) error { - fmt.Printf("%s\n", (ct.MerkleHash())) + h, err := ct.MerkleHash() + if err != nil { + return err + } + fmt.Printf("%s at %d\n", h, ct.Height()) return nil } func cmdCommit(c *cli.Context) error { - h := claim.Height(c.Int64("height")) if !c.IsSet("height") { - h = ct.Height() + 1 + height = ct.Height() + 1 } - return ct.Commit(h) + return ct.Commit(height) } func cmdReset(c *cli.Context) error { - h := claim.Height(c.Int64("height")) - return ct.Reset(h) + return ct.Reset(height) } func cmdLog(c *cli.Context) error { - commitVisit := func(c *trie.Commit) { - meta := c.Meta.(claimtrie.CommitMeta) - fmt.Printf("height: %d, commit %s\n", meta.Height, c.MerkleTrie.MerkleHash()) + visit := func(c *claimtrie.Commit) { + meta := c.Meta + fmt.Printf("%s at %d\n", c.MerkleRoot, meta.Height) } + return claimtrie.Log(ct.Head(), visit) +} - fmt.Printf("\n") - trie.Log(ct.Head(), commitVisit) - return nil +func cmdLoad(c *cli.Context) error { + return claimtrie.Load(ct, height, chk) } func cmdShell(app *cli.App) { @@ -368,7 +248,7 @@ func cmdShell(app *cli.App) { } }() defer close(sigs) - signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + // signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) for { fmt.Printf("%s > ", app.Name) text, err := reader.ReadString('\n') @@ -389,3 +269,95 @@ func cmdShell(app *cli.App) { } signal.Stop(sigs) } + +func parseArgs(c *cli.Context) error { + parsers := []func(*cli.Context) error{ + parseOP, + parseOP, + parseAmt, + parseHeight, + parseID, + } + for _, p := range parsers { + if err := p(c); err != nil { + return err + } + } + return nil +} + +func randInt(min, max int64) int64 { + i, err := rand.Int(rand.Reader, big.NewInt(max)) + if err != nil { + panic(err) + } + return min + i.Int64() +} + +func parseHeight(c *cli.Context) error { + height = claim.Height(c.Int("height")) + return nil +} + +// parseOP generates random OutPoint for the ease of testing. +func parseOP(c *cli.Context) error { + var err error + h := &chainhash.Hash{} + idx := randInt(0, 256) + if _, err = rand.Read(h[:]); err != nil { + return err + } + var sh string + if c.IsSet("outpoint") { + if _, err = fmt.Sscanf(c.String("outpoint"), "%64s:%d", &sh, &idx); err != nil { + return err + } + if h, err = chainhash.NewHashFromStr(sh); err != nil { + return err + } + } + op = *claim.NewOutPoint(h, uint32(idx)) + return nil +} + +func parseAmt(c *cli.Context) error { + if !c.IsSet("amount") { + amt = claim.Amount(randInt(1, 500)) + } + return nil +} + +func parseID(c *cli.Context) error { + if !c.IsSet("id") { + return nil + } + var err error + id, err = claim.NewIDFromString(c.String("id")) + return err +} + +var showNode = func() trie.Visit { + return func(prefix trie.Key, val trie.Value) error { + if val == nil || val.Hash() == nil { + fmt.Printf("%-8s:\n", prefix) + return nil + } + fmt.Printf("%-8s: %v\n", prefix, val) + return nil + } +} + +var recall = func(h claim.Height, visit trie.Visit) trie.Visit { + return func(prefix trie.Key, val trie.Value) error { + n := val.(*claim.Node) + old := n.Height() + err := n.Recall(h) + if err == nil { + err = visit(prefix, val) + } + if err == nil { + err = n.AdjustTo(old) + } + return err + } +} diff --git a/commit.go b/commit.go new file mode 100644 index 0000000..9382db3 --- /dev/null +++ b/commit.go @@ -0,0 +1,39 @@ +package claimtrie + +import ( + "github.com/lbryio/claimtrie/claim" + + "github.com/btcsuite/btcd/chaincfg/chainhash" +) + +// CommitMeta represent the meta associated with each commit. +type CommitMeta struct { + Height claim.Height +} + +func newCommit(head *Commit, meta CommitMeta, h *chainhash.Hash) *Commit { + return &Commit{ + Prev: head, + MerkleRoot: h, + Meta: meta, + } +} + +// Commit ... +type Commit struct { + Prev *Commit + MerkleRoot *chainhash.Hash + Meta CommitMeta +} + +// CommitVisit ... +type CommitVisit func(c *Commit) + +// Log ... +func Log(commit *Commit, visit CommitVisit) error { + for commit != nil { + visit(commit) + commit = commit.Prev + } + return nil +} diff --git a/error.go b/error.go index 9a82b64..49b5ce4 100644 --- a/error.go +++ b/error.go @@ -5,10 +5,4 @@ import "fmt" var ( // ErrInvalidHeight is returned when the height is invalid. ErrInvalidHeight = fmt.Errorf("invalid height") - - // ErrNotFound is returned when the Claim or Support is not found. - ErrNotFound = fmt.Errorf("not found") - - // ErrDuplicate is returned when the Claim or Support already exists in the node. - ErrDuplicate = fmt.Errorf("duplicate") ) diff --git a/import.go b/import.go new file mode 100644 index 0000000..d7bf84b --- /dev/null +++ b/import.go @@ -0,0 +1,134 @@ +package claimtrie + +import ( + "bytes" + "encoding/gob" + "fmt" + "log" + "path/filepath" + "strconv" + + "github.com/lbryio/claimtrie/claim" + + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcutil" + "github.com/syndtr/goleveldb/leveldb" +) + +var ( + defaultHomeDir = btcutil.AppDataDir("lbrycrd.go", false) + defaultDataDir = filepath.Join(defaultHomeDir, "data") + dbCmdPath = filepath.Join(defaultDataDir, "dbCmd") +) + +type block struct { + Hash chainhash.Hash + Cmds []claim.Cmd +} + +// Load ... +func Load(ct *ClaimTrie, h claim.Height, chk bool) error { + db := DefaultRecorder() + defer db.Close() + + for i := ct.height + 1; i <= h; i++ { + key := strconv.Itoa(int(i)) + data, err := db.Get([]byte(key), nil) + if err == leveldb.ErrNotFound { + continue + } else if err != nil { + return err + } + var blk block + if err = gob.NewDecoder(bytes.NewBuffer(data)).Decode(&blk); err != nil { + return err + } + + if err = ct.Commit(i - 1); err != nil { + return err + } + for _, cmd := range blk.Cmds { + if err = execute(ct, &cmd); err != nil { + fmt.Printf("execute faile: err %s\n", err) + return err + } + } + if err = ct.Commit(i); err != nil { + return err + } + if !chk { + continue + } + hash, err := ct.MerkleHash() + if err != nil { + return err + } + if *hash != blk.Hash { + return fmt.Errorf("block %d hash: got %s, want %s", i, *hash, blk.Hash) + } + } + return ct.Commit(h) +} + +func execute(ct *ClaimTrie, c *claim.Cmd) error { + // Value []byte + fmt.Printf("%s\n", c) + switch c.Cmd { + case claim.CmdAddClaim: + return ct.AddClaim(c.Name, c.OP, c.Amt) + case claim.CmdSpendClaim: + return ct.SpendClaim(c.Name, c.OP) + case claim.CmdUpdateClaim: + return ct.UpdateClaim(c.Name, c.OP, c.Amt, c.ID) + case claim.CmdAddSupport: + return ct.AddSupport(c.Name, c.OP, c.Amt, c.ID) + case claim.CmdSpendSupport: + return ct.SpendSupport(c.Name, c.OP) + } + return nil +} + +// Recorder .. +type Recorder struct { + db *leveldb.DB +} + +// Put sets the value for the given key. It overwrites any previous value for that key. +func (r *Recorder) Put(key []byte, data interface{}) error { + buf := bytes.NewBuffer(nil) + if err := gob.NewEncoder(buf).Encode(data); err != nil { + return fmt.Errorf("can't encode cmds, err: %s", err) + } + if err := r.db.Put(key, buf.Bytes(), nil); err != nil { + return fmt.Errorf("can't put to db, err: %s", err) + + } + return nil +} + +// Get ... +func (r *Recorder) Get(key []byte, data interface{}) ([]byte, error) { + return r.db.Get(key, nil) +} + +// Close ... +func (r *Recorder) Close() error { + err := r.db.Close() + r.db = nil + return err +} + +var recorder Recorder + +// DefaultRecorder ... +func DefaultRecorder() *Recorder { + if recorder.db == nil { + db, err := leveldb.OpenFile(dbCmdPath, nil) + if err != nil { + log.Fatalf("can't open :%s, err: %s\n", dbCmdPath, err) + } + fmt.Printf("dbCmds %s opened\n", dbCmdPath) + recorder.db = db + } + return &recorder +} diff --git a/memento/memento.go b/memento/memento.go deleted file mode 100644 index d11a75a..0000000 --- a/memento/memento.go +++ /dev/null @@ -1,109 +0,0 @@ -package memento - -import ( - "errors" -) - -var ( - // ErrCommandStackEmpty is returned when the command stack is empty. - ErrCommandStackEmpty = errors.New("command stack empty") -) - -// Command specifies the interface of a command for the Memento. -type Command interface { - Execute() - Undo() -} - -// CommandList is a list of command. -type CommandList []Command - -func (l CommandList) empty() bool { return len(l) == 0 } -func (l CommandList) top() Command { return l[len(l)-1] } -func (l CommandList) pop() CommandList { return l[:len(l)-1] } -func (l CommandList) push(c Command) CommandList { return append(l, c) } - -// CommandStack is stack of command list. -type CommandStack []CommandList - -func (s CommandStack) empty() bool { return len(s) == 0 } -func (s CommandStack) top() CommandList { return s[len(s)-1] } -func (s CommandStack) pop() CommandStack { return s[:len(s)-1] } -func (s CommandStack) push(l CommandList) CommandStack { return append(s, l) } - -// Memento implements functionality for state to be undo and redo. -type Memento struct { - // Executed but not yet commited command list. - executed CommandList - - // A stack of command list that have been commited. - commited CommandStack - - // A stack of commited command list that have been undone. - undone CommandStack -} - -// Executed returns a list of executed command that have not been commited. -func (m *Memento) Executed() CommandList { return m.executed } - -// Commited returns the stack of command liist that have been commited. -func (m *Memento) Commited() CommandStack { return m.commited } - -// Undone returns the stack of command list that have been undone. -func (m *Memento) Undone() CommandStack { return m.undone } - -// Execute executes a command and appends it to the Executed command list. -// Any command list on the Undone will discarded, and can no longer be redone. -func (m *Memento) Execute(c Command) error { - m.executed = m.executed.push(c) - c.Execute() - m.undone = nil - return nil -} - -// Commit commits the Executed command list to the Commited Stack, and empty the Executed List. -func (m *Memento) Commit() { - m.commited = m.commited.push(m.executed) - m.executed = nil - m.undone = nil -} - -// Undo undos the most recent command list on the Commited stack, and moves it to the Undone Stack. -func (m *Memento) Undo() error { - if m.commited.empty() { - return ErrCommandStackEmpty - } - m.commited, m.undone = process(m.commited, m.undone, true) - return nil -} - -// Redo redos the most recent command list on the Undone Stack, and moves it back to the Commited Stack. -func (m *Memento) Redo() error { - if m.undone.empty() { - return ErrCommandStackEmpty - } - m.undone, m.commited = process(m.undone, m.commited, false) - return nil -} - -// RollbackExecuted undos commands on the Executed list, and empty the list. -func (m *Memento) RollbackExecuted() { - for !m.executed.empty() { - m.executed.top().Undo() - m.executed = m.executed.pop() - } - m.executed = nil -} - -func process(a, b CommandStack, undo bool) (CommandStack, CommandStack) { - processed := CommandList{} - for cmds := a.top(); !cmds.empty(); cmds = cmds.pop() { - if undo { - cmds.top().Undo() - } else { - cmds.top().Execute() - } - processed = processed.push(cmds.top()) - } - return a.pop(), b.push(processed) -} diff --git a/nodemgr/nm.go b/nodemgr/nm.go new file mode 100644 index 0000000..e14e57f --- /dev/null +++ b/nodemgr/nm.go @@ -0,0 +1,155 @@ +package nodemgr + +import ( + "fmt" + "sort" + "sync" + + "github.com/lbryio/claimtrie/claim" + "github.com/lbryio/claimtrie/trie" + "github.com/syndtr/goleveldb/leveldb" +) + +// NodeMgr ... +type NodeMgr struct { + sync.RWMutex + + db *leveldb.DB + nodes map[string]*claim.Node + dirty map[string]bool + nextUpdates todos +} + +// New ... +func New(db *leveldb.DB) *NodeMgr { + nm := &NodeMgr{ + db: db, + nodes: map[string]*claim.Node{}, + dirty: map[string]bool{}, + nextUpdates: todos{}, + } + return nm +} + +// Get ... +func (nm *NodeMgr) Get(key trie.Key) (trie.Value, error) { + nm.Lock() + defer nm.Unlock() + + if n, ok := nm.nodes[string(key)]; ok { + return n, nil + } + if nm.db != nil { + b, err := nm.db.Get(key, nil) + if err == nil { + _ = b // TODO: Loaded. Deserialize it. + } else if err != leveldb.ErrNotFound { + // DB error. Propagated. + return nil, err + } + } + // New node. + n := claim.NewNode(string(key)) + nm.nodes[string(key)] = n + return n, nil +} + +// Set ... +func (nm *NodeMgr) Set(key trie.Key, val trie.Value) { + n := val.(*claim.Node) + + nm.Lock() + defer nm.Unlock() + + nm.nodes[string(key)] = n + nm.dirty[string(key)] = true + + // TODO: flush to disk. +} + +// Reset resets all nodes to specified height. +func (nm *NodeMgr) Reset(h claim.Height) error { + for _, n := range nm.nodes { + if err := n.Reset(h); err != nil { + return err + } + } + return nil +} + +// NodeAt returns the node adjusted to specified height. +func (nm *NodeMgr) NodeAt(name string, h claim.Height) (*claim.Node, error) { + v, err := nm.Get(trie.Key(name)) + if err != nil { + return nil, err + } + n := v.(*claim.Node) + if err = n.AdjustTo(h); err != nil { + return nil, err + } + return n, nil +} + +// ModifyNode returns the node adjusted to specified height. +func (nm *NodeMgr) ModifyNode(name string, h claim.Height, modifier func(*claim.Node) error) error { + n, err := nm.NodeAt(name, h) + if err != nil { + return err + } + if err = modifier(n); err != nil { + return err + } + nm.nextUpdates.set(name, h+1) + return nil +} + +// CatchUp ... +func (nm *NodeMgr) CatchUp(h claim.Height, notifier func(key trie.Key) error) error { + for name := range nm.nextUpdates[h] { + n, err := nm.NodeAt(name, h) + if err != nil { + return err + } + if err = notifier(trie.Key(name)); err != nil { + return err + } + if next := n.NextUpdate(); next > h { + nm.nextUpdates.set(name, next) + } + } + return nil +} + +// Show ... +func (nm *NodeMgr) Show(name string) error { + if len(name) != 0 { + fmt.Printf("[%s] %s\n", name, nm.nodes[name]) + return nil + } + names := []string{} + for name := range nm.nodes { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + fmt.Printf("[%s] %s\n", name, nm.nodes[name]) + } + return nil +} + +// UpdateAll ... +func (nm *NodeMgr) UpdateAll(m func(key trie.Key) error) error { + for name := range nm.nodes { + m(trie.Key(name)) + } + return nil +} + +type todos map[claim.Height]map[string]bool + +func (t todos) set(name string, h claim.Height) { + if t[h] == nil { + t[h] = map[string]bool{} + } + t[h][name] = true +} diff --git a/trie/README.md b/trie/README.md deleted file mode 100644 index 710fb50..0000000 --- a/trie/README.md +++ /dev/null @@ -1,47 +0,0 @@ -# MerkleTrie - -coming soon - -## Installation - -coming soon - -## Usage - -coming soon - -## Running from Source - -This project requires [Go v1.10](https://golang.org/doc/install) or higher. - -``` bash -go get -v github.com/lbryio/trie -``` - -## Examples - -Refer to [triesh](https://github.com/lbryio/trie/blob/master/cmd/triesh) - -## Testing - -``` bash -go test -v github.com/lbryio/trie -gocov test -v github.com/lbryio/trie 1>/dev/null -``` - -## Contributing - -coming soon - -## License - -This project is MIT licensed. - -## Security - -We take security seriously. Please contact security@lbry.io regarding any security issues. -Our PGP key is [here](https://keybase.io/lbry/key.asc) if you need it. - -## Contact - -The primary contact for this project is [@lyoshenka](https://github.com/lyoshenka) (grin@lbry.io) \ No newline at end of file diff --git a/trie/cmd/triesh/README.md b/trie/cmd/triesh/README.md deleted file mode 100644 index 3354d7b..0000000 --- a/trie/cmd/triesh/README.md +++ /dev/null @@ -1,171 +0,0 @@ -# Triesh - -An example Key-Value store to excercise the merkletree package - -Currently, it's only in-memory. - -## Installation - -This project requires [Go v1.10](https://golang.org/doc/install) or higher. - -``` bash -go get -v github.com/lbryio/trie -``` - -## Usage - -Adding values. - -``` bloocks -triesh > u -k alex -v lion -alex=lion -triesh > u -k al -v tiger -al=tiger -triesh > u -k tess -v dolphin -tess=dolphin -triesh > u -k bob -v pig -bob=pig -triesh > u -k ted -v do -ted=do -triesh > u -k ted -v dog -ted=dog -``` - -Showing Merkle Hash. - -``` blocks -triesh > merkle -bfa2927b147161146411b7f6187e1ed0c08c3dc19b200550c3458d44c0032285 - -triesh > u -k teddy -v bear -teddy=bear - -triesh > merkle -94831650b8bf76d579ca4eda1cb35861c6f5c88eb4f5b089f60fe687defe8f3d -``` - -Showing all values. - -``` blocks -triesh > s -[al ] tiger -[alex ] lion -[bob ] pig -[ted ] dog -[teddy ] bear -[tess ] dolphin -``` - -Showing all values and link nodes. - -``` bloocks -triesh > s -a -[a ] -[al ] tiger -[ale ] -[alex ] lion -[b ] -[bo ] -[bob ] pig -[t ] -[te ] -[ted ] dog -[tedd ] -[teddy ] bear -[tes ] -[tess ] dolphin -``` - -Deleting values (setting key to nil / ""). - -``` blocks -triesh > u -k al -al= -triesh > u -k alex -alex= -``` - -Updating Values. - -``` blocks -triesh > u -k bob -v cat -bob=cat -``` - -Showing all nodes, include non-pruned link nodes" - -``` blocks -triesh > s -a -[a ] -[al ] -[ale ] -[alex ] -[b ] -[bo ] -[bob ] cat -[t ] -[te ] -[ted ] dog -[tedd ] -[teddy ] bear -[tes ] -[tess ] dolphin - -``` - -Calculate Merkle Hash. - -``` blocks -triesh > merkle -c2fdce68a30e3cabf6efb3b7ebfd32afdaf09f9ebd062743fe91e181f682252b -``` - -Prune link nodes that do not reach to any values. - -``` blocks -triesh > p -pruned -``` - -Show pruned Trie and caculate the Merkle Hash again. - -``` blocks -triesh > s -a -[b ] -[bo ] -[bob ] cat -[t ] -[te ] -[ted ] dog -[tedd ] -[teddy ] bear -[tes ] -[tess ] dolphin - -triesh > merkle -c2fdce68a30e3cabf6efb3b7ebfd32afdaf09f9ebd062743fe91e181f682252b -``` - -## Running from Source - -``` bash -cd $(go env GOPATH)/src/github.com/lbryio/trie -go run cmd/triesh/*.go sh -``` - -## Contributing - -coming soon - -## License - -This project is MIT licensed. - -## Security - -We take security seriously. Please contact security@lbry.io regarding any security issues. -Our PGP key is [here](https://keybase.io/lbry/key.asc) if you need it. - -## Contact - -The primary contact for this project is [@lyoshenka](https://github.com/lyoshenka) (grin@lbry.io) \ No newline at end of file diff --git a/trie/cmd/triesh/main.go b/trie/cmd/triesh/main.go deleted file mode 100644 index c0c914b..0000000 --- a/trie/cmd/triesh/main.go +++ /dev/null @@ -1,243 +0,0 @@ -package main - -import ( - "bufio" - "fmt" - "os" - "os/signal" - "strings" - "syscall" - - "github.com/btcsuite/btcd/chaincfg/chainhash" - "github.com/lbryio/claimtrie/trie" - "github.com/urfave/cli" -) - -var ( - flgKey = cli.StringFlag{Name: "key, k", Usage: "Key"} - flgValue = cli.StringFlag{Name: "value, v", Usage: "Value"} - flgAll = cli.BoolFlag{Name: "all, a", Usage: "Apply to non-value nodes"} - flgMessage = cli.StringFlag{Name: "message, m", Usage: "Commit Message"} - flgID = cli.StringFlag{Name: "id", Usage: "Commit ID"} -) - -var ( - // ErrNotImplemented is returned when a function is not implemented yet. - ErrNotImplemented = fmt.Errorf("not implemented") -) - -func main() { - app := cli.NewApp() - - app.Name = "triesh" - app.Usage = "A CLI tool for Merkle MerkleTrie" - app.Version = "0.0.1" - app.Action = cli.ShowAppHelp - - app.Commands = []cli.Command{ - { - Name: "update", - Aliases: []string{"u"}, - Usage: "Update Value for Key", - Action: cmdUpdate, - Flags: []cli.Flag{flgKey, flgValue}, - }, - { - Name: "get", - Aliases: []string{"g"}, - Usage: "Get Value for specified Key", - Action: cmdGet, - Flags: []cli.Flag{flgKey, flgID}, - }, - { - Name: "show", - Aliases: []string{"s"}, - Usage: "Show Key-Value pairs of specified commit", - Action: cmdShow, - Flags: []cli.Flag{flgAll, flgID}, - }, - { - Name: "merkle", - Aliases: []string{"m"}, - Usage: "Show Merkle Hash of stage", - Action: cmdMerkle, - }, - { - Name: "prune", - Aliases: []string{"p"}, - Usage: "Prune link nodes that doesn't reach to any value", - Action: cmdPrune, - }, - { - Name: "commit", - Aliases: []string{"c"}, - Usage: "Commit current stage to database", - Action: cmdCommit, - Flags: []cli.Flag{flgMessage}, - }, - { - Name: "reset", - Aliases: []string{"r"}, - Usage: "Reset HEAD & Stage to specified commit", - Action: cmdReset, - Flags: []cli.Flag{flgAll, flgID}, - }, - { - Name: "log", - Aliases: []string{"l"}, - Usage: "Show commit logs", - Action: cmdLog, - }, - { - Name: "shell", - Aliases: []string{"sh"}, - Usage: "Enter interactive mode", - Action: func(c *cli.Context) { cmdShell(app) }, - }, - } - - if err := app.Run(os.Args); err != nil { - fmt.Printf("error: %s\n", err) - } -} - -type strValue string - -func (s strValue) Hash() *chainhash.Hash { - h := chainhash.DoubleHashH([]byte(s)) - return &h -} - -var ( - mt = trie.New() - head = trie.NewCommit(nil, "initial", mt) - stg = trie.NewStage(mt) -) - -func commitVisit(c *trie.Commit) { - fmt.Printf("commit %s\n\n", c.MerkleTrie.MerkleHash()) - fmt.Printf("\t%s\n\n", c.Meta.(string)) -} - -func cmdUpdate(c *cli.Context) error { - key, value := c.String("key"), c.String("value") - fmt.Printf("%s=%s\n", key, value) - if len(value) == 0 { - return stg.Update(trie.Key(key), nil) - } - return stg.Update(trie.Key(key), strValue(value)) -} - -func cmdGet(c *cli.Context) error { - key := c.String("key") - value, err := stg.Get(trie.Key(key)) - if err != nil { - return err - } - if str, ok := value.(strValue); ok { - fmt.Printf("[%s]\n", str) - } - return nil -} - -func cmdShow(c *cli.Context) error { - dump := func(prefix trie.Key, val trie.Value) error { - if val == nil { - fmt.Printf("[%-8s]\n", prefix) - return nil - } - fmt.Printf("[%-8s] %v\n", prefix, val) - return nil - } - id := c.String("id") - if len(id) == 0 { - return stg.Traverse(dump, false, !c.Bool("all")) - } - for commit := head; commit != nil; commit = commit.Prev { - if commit.MerkleTrie.MerkleHash().String() == id { - return commit.MerkleTrie.Traverse(dump, false, true) - } - - } - return fmt.Errorf("commit noot found") -} - -func cmdMerkle(c *cli.Context) error { - fmt.Printf("%s\n", stg.MerkleHash()) - return nil -} - -func cmdPrune(c *cli.Context) error { - stg.Prune() - fmt.Printf("pruned\n") - return nil -} - -func cmdCommit(c *cli.Context) error { - msg := c.String("message") - if len(msg) == 0 { - return fmt.Errorf("no message specified") - } - h, err := stg.Commit(head, msg) - if err != nil { - return err - } - head = h - return nil -} - -func cmdReset(c *cli.Context) error { - id := c.String("id") - for commit := head; commit != nil; commit = commit.Prev { - if commit.MerkleTrie.MerkleHash().String() != id { - continue - } - head = commit - stg = trie.NewStage(head.MerkleTrie) - return nil - } - return fmt.Errorf("commit noot found") -} - -func cmdLog(c *cli.Context) error { - commitVisit := func(c *trie.Commit) { - fmt.Printf("commit %s\n\n", c.MerkleTrie.MerkleHash()) - fmt.Printf("\t%s\n\n", c.Meta.(string)) - } - - trie.Log(head, commitVisit) - return nil -} - -func cmdShell(app *cli.App) { - cli.OsExiter = func(c int) {} - reader := bufio.NewReader(os.Stdin) - sigs := make(chan os.Signal, 1) - go func() { - for range sigs { - fmt.Printf("\n(type quit or q to exit)\n\n") - fmt.Printf("%s > ", app.Name) - } - }() - defer close(sigs) - signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) - for { - fmt.Printf("%s > ", app.Name) - text, err := reader.ReadString('\n') - if err != nil { - fmt.Printf("error: %s\n", err) - } - text = strings.TrimSpace(text) - if text == "" { - continue - } - if text == "quit" || text == "q" { - break - } - if err := app.Run(append(os.Args[1:], strings.Split(text, " ")...)); err != nil { - fmt.Printf("errot: %s\n", err) - } - - } - signal.Stop(sigs) -} diff --git a/trie/commit.go b/trie/commit.go deleted file mode 100644 index 64bfa2d..0000000 --- a/trie/commit.go +++ /dev/null @@ -1,21 +0,0 @@ -package trie - -// CommitMeta ... -type CommitMeta interface{} - -// NewCommit ... -func NewCommit(head *Commit, meta CommitMeta, mt *MerkleTrie) *Commit { - commit := &Commit{ - Prev: head, - MerkleTrie: mt, - Meta: meta, - } - return commit -} - -// Commit ... -type Commit struct { - Prev *Commit - MerkleTrie *MerkleTrie - Meta CommitMeta -} diff --git a/trie/errors.go b/trie/errors.go deleted file mode 100644 index d25d49a..0000000 --- a/trie/errors.go +++ /dev/null @@ -1,8 +0,0 @@ -package trie - -import "errors" - -var ( - // ErrKeyNotFound is returned when the key doesn't exist in the MerkleTrie. - ErrKeyNotFound = errors.New("key not found") -) diff --git a/trie/kv.go b/trie/kv.go new file mode 100644 index 0000000..839a685 --- /dev/null +++ b/trie/kv.go @@ -0,0 +1,19 @@ +package trie + +import ( + "github.com/btcsuite/btcd/chaincfg/chainhash" +) + +// Key defines the key type of the MerkleTrie. +type Key []byte + +// Value defines value for the MerkleTrie. +type Value interface { + Hash() *chainhash.Hash +} + +// KeyValue ... +type KeyValue interface { + Get(k Key) (Value, error) + Set(k Key, v Value) +} diff --git a/trie/node.go b/trie/node.go index 1c0fbd5..dec2272 100644 --- a/trie/node.go +++ b/trie/node.go @@ -5,94 +5,32 @@ import ( ) type node struct { - hash *chainhash.Hash - links [256]*node - value Value + hash *chainhash.Hash + links [256]*node + hasValue bool } -func newNode(val Value) *node { - return &node{links: [256]*node{}, value: val} +func newNode() *node { + return &node{} } -// We clear the Merkle Hash for every node along the path, including the root. -// Calculation of the hash happens much less frequently then updating to the MerkleTrie. -func update(n *node, key Key, val Value) { - n.hash = nil - // Follow the path to reach the node. - for _, k := range key { - if n.links[k] == nil { - // The path didn't exist yet. Build it. - n.links[k] = newNode(nil) - } - n.hash = nil - n = n.links[k] - } +// nbuf decodes the on-disk format of a node, which has the following form: +// ch(1B) hash(32B) +// ... +// ch(1B) hash(32B) +// vhash(32B) +type nbuf []byte - n.value = val - n.hash = nil +func (nb nbuf) entries() int { + return len(nb) / 33 } -func prune(n *node) *node { - if n == nil { - return nil - } - var ret *node - for i, v := range n.links { - if n.links[i] = prune(v); n.links[i] != nil { - ret = n - } - } - if n.value != nil { - ret = n - } - return ret +func (nb nbuf) entry(i int) (byte, *chainhash.Hash) { + h := chainhash.Hash{} + copy(h[:], nb[33*i+1:]) + return nb[33*i], &h } -func traverse(n *node, prefix Key, visit Visit) error { - if n == nil { - return nil - } - for i, v := range n.links { - if v == nil { - continue - } - p := append(prefix, byte(i)) - if err := visit(p, v.value); err != nil { - return err - } - if err := traverse(v, p, visit); err != nil { - return err - } - } - return nil -} - -// merkle recursively caculates the Merkle Hash of a given node -// It works with both pruned or unpruned nodes. -func merkle(n *node) *chainhash.Hash { - if n.hash != nil { - return n.hash - } - buf := Key{} - for i, v := range n.links { - if v == nil { - continue - } - if h := merkle(v); h != nil { - buf = append(buf, byte(i)) - buf = append(buf, h[:]...) - } - } - if n.value != nil { - if h := n.value.Hash(); h != nil { - buf = append(buf, h[:]...) - } - } - - if len(buf) != 0 { - // At least one of the sub nodes has contributed a value hash. - h := chainhash.DoubleHashH(buf) - n.hash = &h - } - return n.hash +func (nb nbuf) hasValue() bool { + return len(nb)%33 == 32 } diff --git a/trie/node_test.go b/trie/node_test.go deleted file mode 100644 index abc58d0..0000000 --- a/trie/node_test.go +++ /dev/null @@ -1,139 +0,0 @@ -package trie - -import ( - "fmt" - "reflect" - "testing" - - "github.com/btcsuite/btcd/chaincfg/chainhash" -) - -func Test_update(t *testing.T) { - res1 := buildNode(newNode(nil), pairs1()) - tests := []struct { - name string - res *node - exp *node - }{ - {"test1", res1, unprunedNode()}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if !reflect.DeepEqual(tt.res, tt.exp) { - traverse(tt.res, Key{}, dump) - fmt.Println("") - traverse(tt.exp, Key{}, dump) - t.Errorf("update() = %v, want %v", tt.res, tt.exp) - } - }) - } -} - -func Test_nullify(t *testing.T) { - tests := []struct { - name string - res *node - exp *node - }{ - {"test1", prune(unprunedNode()), prunedNode()}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if !reflect.DeepEqual(tt.res, tt.exp) { - t.Errorf("traverse() = %v, want %v", tt.res, tt.exp) - } - }) - } -} - -func Test_traverse(t *testing.T) { - res1 := []pair{} - fn := func(prefix Key, value Value) error { - res1 = append(res1, pair{string(prefix), value}) - return nil - } - traverse(unprunedNode(), Key{}, fn) - exp1 := []pair{ - {"a", nil}, - {"al", nil}, - {"ale", nil}, - {"alex", nil}, - {"b", nil}, - {"bo", nil}, - {"bob", strValue("cat")}, - {"t", nil}, - {"te", nil}, - {"ted", strValue("dog")}, - {"tedd", nil}, - {"teddy", strValue("bear")}, - {"tes", nil}, - {"tess", strValue("dolphin")}, - } - - res2 := []pair{} - fn2 := func(prefix Key, value Value) error { - res2 = append(res2, pair{string(prefix), value}) - return nil - } - traverse(prunedNode(), Key{}, fn2) - exp2 := []pair{ - {"b", nil}, - {"bo", nil}, - {"bob", strValue("cat")}, - {"t", nil}, - {"te", nil}, - {"ted", strValue("dog")}, - {"tedd", nil}, - {"teddy", strValue("bear")}, - {"tes", nil}, - {"tess", strValue("dolphin")}, - } - - tests := []struct { - name string - res []pair - exp []pair - }{ - {"test1", res1, exp1}, - {"test2", res2, exp2}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if !reflect.DeepEqual(tt.res, tt.exp) { - t.Errorf("traverse() = %v, want %v", tt.res, tt.exp) - } - }) - } -} - -func Test_merkle(t *testing.T) { - n1 := buildNode(newNode(nil), pairs1()) - // n2 := func() *node { - // p1 := wire.OutPoint{Hash: *newHashFromStr("627ecfee2110b28fbc4b012944cadf66a72f394ad9fa9bb18fec30789e26c9ac"), Index: 0} - // p2 := wire.OutPoint{Hash: *newHashFromStr("c31bd469112abf04930879c6b6007d2b23224e042785d404bbeff1932dd94880"), Index: 0} - - // n1 := claim.NewNode(&claim.Claim{OutPoint: p1, ClaimID: nil, Amount: 50, Height: 100, ValidAtHeight: 200}) - // n2 := claim.NewNode(&claim.Claim{OutPoint: p2, ClaimID: nil, Amount: 50, Height: 100, ValidAtHeight: 200}) - - // pairs := []pair{ - // {"test", n1}, - // {"test2", n2}, - // } - // return buildNode(newNode(nil), pairs) - // }() - tests := []struct { - name string - n *node - want *chainhash.Hash - }{ - {"test1", n1, newHashFromStr("c2fdce68a30e3cabf6efb3b7ebfd32afdaf09f9ebd062743fe91e181f682252b")}, - // {"test2", n2, newHashFromStr("71c7b8d35b9a3d7ad9a1272b68972979bbd18589f1efe6f27b0bf260a6ba78fa")}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := merkle(tt.n); !reflect.DeepEqual(got, tt.want) { - t.Errorf("merkle() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/trie/stage.go b/trie/stage.go deleted file mode 100644 index d6191cd..0000000 --- a/trie/stage.go +++ /dev/null @@ -1,63 +0,0 @@ -package trie - -// Stage implements Copy-on-Write staging area on top of a MerkleTrie. -type Stage struct { - *MerkleTrie -} - -// NewStage returns a Stage initialized with a specified MerkleTrie. -func NewStage(t *MerkleTrie) *Stage { - s := &Stage{ - MerkleTrie: New(), - } - s.mu = t.mu - s.root = newNode(nil) - *s.root = *t.root - return s -} - -// Update updates the internal MerkleTrie in a Copy-on-Write manner. -func (s *Stage) Update(key Key, val Value) error { - s.mu.Lock() - defer s.mu.Unlock() - - n := s.root - n.hash = nil - for _, k := range key { - org := n.links[k] - n.links[k] = newNode(nil) - if org != nil { - *n.links[k] = *org - } - n.hash = nil - n = n.links[k] - } - n.value = val - n.hash = nil - return nil -} - -// Commit ... -func (s *Stage) Commit(head *Commit, meta CommitMeta) (*Commit, error) { - // Update Merkle Hash. - s.MerkleHash() - - c := NewCommit(head, meta, s.MerkleTrie) - - s.MerkleTrie = New() - s.mu = c.MerkleTrie.mu - s.root = newNode(nil) - *s.root = *c.MerkleTrie.root - return c, nil -} - -// CommitVisit ... -type CommitVisit func(c *Commit) - -// Log ... -func Log(commit *Commit, visit CommitVisit) { - for commit != nil { - visit(commit) - commit = commit.Prev - } -} diff --git a/trie/stage_test.go b/trie/stage_test.go deleted file mode 100644 index 2f768d0..0000000 --- a/trie/stage_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package trie - -import ( - "fmt" - "reflect" - "testing" -) - -func TestStage_Update(t *testing.T) { - tr1 := buildTrie(New(), pairs1()) - - s1 := NewStage(tr1) - s1.Update(Key("cook"), strValue("hello")) - s1.Update(Key("ted"), nil) - - tr1Exp := buildTrie(New(), pairs1()) - - s1Exp := buildTrie(New(), pairs1()) - s1Exp.Update(Key("cook"), strValue("hello")) - s1Exp.Update(Key("ted"), nil) - - if !reflect.DeepEqual(tr1, tr1Exp) { - t.Errorf("Stage.Update() tr1 != tr1Exp") - traverse(tr1.root, Key{}, dump) - fmt.Println("") - traverse(tr1Exp.root, Key{}, dump) - } - if !reflect.DeepEqual(s1.MerkleTrie, s1Exp) { - t.Errorf("Stage.Update() s1 != s1Exp") - traverse(s1.root, Key{}, dump) - fmt.Println("") - traverse(s1Exp.root, Key{}, dump) - } -} diff --git a/trie/test.go b/trie/test.go deleted file mode 100644 index b92e01d..0000000 --- a/trie/test.go +++ /dev/null @@ -1,107 +0,0 @@ -package trie - -import ( - "fmt" - - "github.com/btcsuite/btcd/chaincfg/chainhash" -) - -// Internal utility functions to facilitate the tests. - -type strValue string - -func (s strValue) Hash() *chainhash.Hash { - h := chainhash.DoubleHashH([]byte(s)) - return &h -} - -func dump(prefix Key, value Value) error { - if value == nil { - fmt.Printf("[%-8s]\n", prefix) - return nil - } - fmt.Printf("[%-8s] %v\n", prefix, value) - return nil -} - -func buildNode(n *node, pairs []pair) *node { - for _, val := range pairs { - update(n, Key(val.k), val.v) - } - return n -} - -func buildTrie(mt *MerkleTrie, pairs []pair) *MerkleTrie { - for _, val := range pairs { - mt.Update(Key(val.k), val.v) - } - return mt -} - -func buildMap(m map[string]Value, pairs []pair) map[string]Value { - for _, p := range pairs { - if p.v == nil { - delete(m, p.k) - } else { - m[p.k] = p.v - } - } - return m -} - -func newMap() map[string]Value { - return map[string]Value{} -} - -type pair struct { - k string - v Value -} - -func pairs1() []pair { - return []pair{ - {"alex", strValue("lion")}, - {"al", strValue("tiger")}, - {"tess", strValue("dolphin")}, - {"bob", strValue("pig")}, - {"ted", strValue("dog")}, - {"teddy", strValue("bear")}, - {"al", nil}, - {"alex", nil}, - {"bob", strValue("cat")}, - } -} - -func prunedNode() *node { - n := newNode(nil) - n.links['b'] = newNode(nil) - n.links['b'].links['o'] = newNode(nil) - n.links['b'].links['o'].links['b'] = newNode(strValue("cat")) - n.links['t'] = newNode(nil) - n.links['t'].links['e'] = newNode(nil) - n.links['t'].links['e'].links['d'] = newNode(strValue("dog")) - n.links['t'].links['e'].links['d'].links['d'] = newNode(nil) - n.links['t'].links['e'].links['d'].links['d'].links['y'] = newNode(strValue("bear")) - n.links['t'].links['e'].links['s'] = newNode(nil) - n.links['t'].links['e'].links['s'].links['s'] = newNode(strValue("dolphin")) - return n -} - -func unprunedNode() *node { - n := newNode(nil) - n.links['a'] = newNode(nil) - n.links['a'].links['l'] = newNode(nil) - n.links['a'].links['l'].links['e'] = newNode(nil) - n.links['a'].links['l'].links['e'].links['x'] = newNode(nil) - n.links['b'] = newNode(nil) - n.links['b'].links['o'] = newNode(nil) - n.links['b'].links['o'].links['b'] = newNode(strValue("cat")) - n.links['t'] = newNode(nil) - n.links['t'].links['e'] = newNode(nil) - n.links['t'].links['e'].links['d'] = newNode(strValue("dog")) - n.links['t'].links['e'].links['d'].links['d'] = newNode(nil) - n.links['t'].links['e'].links['d'].links['d'].links['y'] = newNode(strValue("bear")) - n.links['t'].links['e'].links['s'] = newNode(nil) - n.links['t'].links['e'].links['s'].links['s'] = newNode(strValue("dolphin")) - return n -} diff --git a/trie/trie.go b/trie/trie.go index 21a5410..7c7eece 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -1,128 +1,200 @@ package trie import ( + "bytes" + "fmt" "sync" "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/pkg/errors" + "github.com/syndtr/goleveldb/leveldb" ) var ( - // EmptyTrieHash represent the Merkle Hash of an empty MerkleTrie. - EmptyTrieHash = *newHashFromStr("0000000000000000000000000000000000000000000000000000000000000001") + // ErrResolve is returned when an error occured during resolve. + ErrResolve = fmt.Errorf("can't resolve") +) +var ( + // EmptyTrieHash represents the Merkle Hash of an empty Trie. + // "0000000000000000000000000000000000000000000000000000000000000001" + EmptyTrieHash = &chainhash.Hash{1} ) -// Key defines the key type of the MerkleTrie. -type Key []byte +// Trie implements a 256-way prefix tree. +type Trie struct { + kv KeyValue + db *leveldb.DB -// Value implements value for the MerkleTrie. -type Value interface { - Hash() *chainhash.Hash + root *node + bufs *sync.Pool + batch *leveldb.Batch } -// MerkleTrie implements a 256-way prefix tree, which takes Key as key and any value that implements the Value interface. -type MerkleTrie struct { - mu *sync.RWMutex - root *node -} - -// New returns a MerkleTrie. -func New() *MerkleTrie { - return &MerkleTrie{ - mu: &sync.RWMutex{}, - root: newNode(nil), +// New returns a Trie. +func New(kv KeyValue, db *leveldb.DB) *Trie { + return &Trie{ + kv: kv, + db: db, + root: newNode(), + bufs: &sync.Pool{ + New: func() interface{} { + return new(bytes.Buffer) + }, + }, } } -// Get returns the Value associated with the key, or nil with error. -// Most common error is ErrMissing, which indicates no Value is associated with the key. -// However, there could be other errors propagated from I/O layer (TBD). -func (t *MerkleTrie) Get(key Key) (Value, error) { - t.mu.RLock() - defer t.mu.RUnlock() +// SetRoot drops all resolved nodes in the Trie, and set the root with specified hash. +func (t *Trie) SetRoot(h *chainhash.Hash) { + t.root = newNode() + t.root.hash = h +} +// Update updates the nodes along the path to the key. +// Each node is resolved or created with their Hash cleared. +func (t *Trie) Update(key Key) error { n := t.root - for _, k := range key { - if n.links[k] == nil { - // Path does not exist. - return nil, ErrKeyNotFound + for _, ch := range key { + if err := t.resolve(n); err != nil { + return ErrResolve } - n = n.links[k] + if n.links[ch] == nil { + n.links[ch] = newNode() + } + n.hash = nil + n = n.links[ch] } - if n.value == nil { - // Path exists, but no Value is associated. - // This happens when the key had been deleted, but the MerkleTrie has not nullified yet. - return nil, ErrKeyNotFound + if err := t.resolve(n); err != nil { + return ErrResolve } - return n.value, nil -} - -// Update updates the MerkleTrie with specified key-value pair. -// Setting Value to nil deletes the Value, if exists, associated to the key. -func (t *MerkleTrie) Update(key Key, val Value) error { - t.mu.Lock() - defer t.mu.Unlock() - - update(t.root, key, val) + n.hasValue = true + n.hash = nil return nil } -// Prune removes nodes that do not reach to any value node. -func (t *MerkleTrie) Prune() { - t.mu.Lock() - defer t.mu.Unlock() - - prune(t.root) -} - -// Size returns the number of values. -func (t *MerkleTrie) Size() int { - t.mu.RLock() - defer t.mu.RUnlock() - - size := 0 // captured in the closure. - fn := func(prefix Key, v Value) error { - if v != nil { - size++ - } +func (t *Trie) resolve(n *node) error { + if n.hash == nil { return nil } - traverse(t.root, Key{}, fn) - return size + b, err := t.db.Get(n.hash[:], nil) + if err == leveldb.ErrNotFound { + return nil + } else if err != nil { + return errors.Wrapf(err, "db.Get(%s)", n.hash) + } + + nb := nbuf(b) + n.hasValue = nb.hasValue() + for i := 0; i < nb.entries(); i++ { + p, h := nb.entry(i) + n.links[p] = newNode() + n.links[p].hash = h + } + return nil } // Visit implements callback function invoked when the Value is visited. // During the traversal, if a non-nil error is returned, the traversal ends early. type Visit func(prefix Key, val Value) error -// Traverse visits every Value in the MerkleTrie and returns error defined by specified Visit function. -// update indicates if the visit function modify the state of MerkleTrie. -func (t *MerkleTrie) Traverse(visit Visit, update, valueOnly bool) error { - if update { - t.mu.Lock() - defer t.mu.Unlock() - } else { - t.mu.RLock() - defer t.mu.RUnlock() - } - fn := func(prefix Key, value Value) error { - if !valueOnly || value != nil { - return visit(prefix, value) +// Traverse implements preorder traversal visiting each Value node. +func (t *Trie) Traverse(visit Visit) error { + var traverse func(prefix Key, n *node) error + traverse = func(prefix Key, n *node) error { + if n == nil { + return nil + } + for ch, n := range n.links { + if n == nil || !n.hasValue { + continue + } + + p := append(prefix, byte(ch)) + val, err := t.kv.Get(p) + if err != nil { + return errors.Wrapf(err, "kv.Get(%s)", p) + } + if err := visit(p, val); err != nil { + return err + } + + if err := traverse(p, n); err != nil { + return err + } } return nil } - return traverse(t.root, Key{}, fn) + buf := make([]byte, 0, 4096) + return traverse(buf, t.root) } -// MerkleHash calculates the Merkle Hash of the MerkleTrie. -// If the MerkleTrie is empty, EmptyTrieHash is returned. -func (t *MerkleTrie) MerkleHash() chainhash.Hash { - if merkle(t.root) == nil { - return EmptyTrieHash +// MerkleHash returns the Merkle Hash of the Trie. +// All nodes must have been resolved before calling this function. +func (t *Trie) MerkleHash() (*chainhash.Hash, error) { + t.batch = &leveldb.Batch{} + buf := make([]byte, 0, 4096) + if err := t.merkle(buf, t.root); err != nil { + return nil, err } - return *t.root.hash + if t.root.hash == nil { + return EmptyTrieHash, nil + } + if t.db != nil && t.batch.Len() != 0 { + if err := t.db.Write(t.batch, nil); err != nil { + return nil, errors.Wrapf(err, "db.Write(t.batch, nil)") + } + } + return t.root.hash, nil } -func newHashFromStr(s string) *chainhash.Hash { - h, _ := chainhash.NewHashFromStr(s) - return h +// merkle recursively resolves the hashes of the node. +// All nodes must have been resolved before calling this function. +func (t *Trie) merkle(prefix Key, n *node) error { + if n.hash != nil { + return nil + } + b := t.bufs.Get().(*bytes.Buffer) + defer t.bufs.Put(b) + b.Reset() + + for ch, n := range n.links { + if n == nil { + continue + } + p := append(prefix, byte(ch)) + if err := t.merkle(p, n); err != nil { + return err + } + if n.hash == nil { + continue + } + if err := b.WriteByte(byte(ch)); err != nil { + panic(err) // Can't happen. Kepp linter happy. + } + if _, err := b.Write(n.hash[:]); err != nil { + panic(err) // Can't happen. Kepp linter happy. + } + } + + if n.hasValue { + val, err := t.kv.Get(prefix) + if err != nil { + return errors.Wrapf(err, "t.kv.get(%s)", prefix) + } + if h := val.Hash(); h != nil { + if _, err = b.Write(h[:]); err != nil { + panic(err) // Can't happen. Kepp linter happy. + } + } + } + + if b.Len() == 0 { + return nil + } + h := chainhash.DoubleHashH(b.Bytes()) + n.hash = &h + if t.db != nil { + t.batch.Put(n.hash[:], b.Bytes()) + } + return nil } diff --git a/trie/trie_test.go b/trie/trie_test.go deleted file mode 100644 index 0caf285..0000000 --- a/trie/trie_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package trie - -import ( - "reflect" - "testing" - - "github.com/btcsuite/btcd/chaincfg/chainhash" -) - -func TestTrie_Update(t *testing.T) { - mt := buildTrie(New(), pairs1()) - m := buildMap(newMap(), pairs1()) - - for k := range m { - v, _ := mt.Get(Key(k)) - if m[k] != v { - t.Errorf("exp %s got %s", m[k], v) - } - } -} - -func TestTrie_Hash(t *testing.T) { - tr1 := buildTrie(New(), pairs1()) - // tr2 := func() *MerkleTrie { - // p1 := wire.OutPoint{Hash: *newHashFromStr("627ecfee2110b28fbc4b012944cadf66a72f394ad9fa9bb18fec30789e26c9ac"), Index: 0} - // p2 := wire.OutPoint{Hash: *newHashFromStr("c31bd469112abf04930879c6b6007d2b23224e042785d404bbeff1932dd94880"), Index: 0} - - // n1 := claim.NewNode(&claim.Claim{OutPoint: p1, ClaimID: nil, Amount: 50, Height: 100, ValidAtHeight: 200}) - // n2 := claim.NewNode(&claim.Claim{OutPoint: p2, ClaimID: nil, Amount: 50, Height: 100, ValidAtHeight: 200}) - - // pairs := []pair{ - // {"test", n1}, - // {"test2", n2}, - // } - // return buildTrie(New(), pairs) - // }() - tests := []struct { - name string - mt *MerkleTrie - want chainhash.Hash - }{ - {"empty", New(), *newHashFromStr("0000000000000000000000000000000000000000000000000000000000000001")}, - {"test1", tr1, *newHashFromStr("c2fdce68a30e3cabf6efb3b7ebfd32afdaf09f9ebd062743fe91e181f682252b")}, - // {"test2", tr2, *newHashFromStr("71c7b8d35b9a3d7ad9a1272b68972979bbd18589f1efe6f27b0bf260a6ba78fa")}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - mt := tt.mt - if got := mt.MerkleHash(); !reflect.DeepEqual(got, tt.want) { - t.Errorf("trie.MerkleHash() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestTrie_Size(t *testing.T) { - mt1 := buildTrie(New(), pairs1()) - map1 := buildMap(newMap(), pairs1()) - - tests := []struct { - name string - mt *MerkleTrie - want int - }{ - {"test1", mt1, len(map1)}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - mt := tt.mt - if got := mt.Size(); got != tt.want { - t.Errorf("trie.Size() = %v, want %v", got, tt.want) - } - }) - } -}