From 0518180508549f6ca60dbedf2b73b4b54d339064 Mon Sep 17 00:00:00 2001 From: Brannon King Date: Wed, 14 Jul 2021 14:42:00 -0400 Subject: [PATCH] in progress on final delay workaround --- claimtrie/claimtrie.go | 3 +- claimtrie/cmd/cmd/node.go | 23 ++++++++ claimtrie/merkletrie/merkletrie.go | 2 + claimtrie/merkletrie/ramtrie.go | 3 + claimtrie/node/manager.go | 93 +++++++++++++++++++----------- claimtrie/node/node.go | 6 +- claimtrie/node/noderepo/pebble.go | 15 +++-- 7 files changed, 102 insertions(+), 43 deletions(-) diff --git a/claimtrie/claimtrie.go b/claimtrie/claimtrie.go index 9fa86817..8de2b44c 100644 --- a/claimtrie/claimtrie.go +++ b/claimtrie/claimtrie.go @@ -23,7 +23,6 @@ import ( "github.com/btcsuite/btcd/chaincfg/chainhash" "github.com/btcsuite/btcd/wire" - "runtime" "sort" ) @@ -150,6 +149,7 @@ func New(cfg config.Config) (*ClaimTrie, error) { ct.Close() return nil, fmt.Errorf("node manager init: %w", err) } + // TODO: pass in the interrupt signal here: trie.SetRoot(hash, nil) // keep this after IncrementHeightTo if !ct.MerkleHash().IsEqual(hash) { @@ -289,7 +289,6 @@ func (ct *ClaimTrie) AppendBlock() error { if hitFork { ct.merkleTrie.SetRoot(h, names) // for clearing the memory entirely - runtime.GC() } return nil diff --git a/claimtrie/cmd/cmd/node.go b/claimtrie/cmd/cmd/node.go index a943ee41..b5eca50e 100644 --- a/claimtrie/cmd/cmd/node.go +++ b/claimtrie/cmd/cmd/node.go @@ -17,6 +17,7 @@ func init() { nodeCmd.AddCommand(nodeDumpCmd) nodeCmd.AddCommand(nodeReplayCmd) + nodeCmd.AddCommand(nodeChildrenCmd) } var nodeCmd = &cobra.Command{ @@ -102,3 +103,25 @@ var nodeReplayCmd = &cobra.Command{ return nil }, } + +var nodeChildrenCmd = &cobra.Command{ + Use: "children ", + Short: "Show all the children names of a given node name", + Args: cobra.RangeArgs(1, 1), + RunE: func(cmd *cobra.Command, args []string) error { + + repo, err := noderepo.NewPebble(localConfig.NodeRepoPebble.Path) + if err != nil { + return fmt.Errorf("open node repo: %w", err) + } + + repo.IterateChildren([]byte(args[0]), func(changes []change.Change) bool { + // TODO: dump all the changes? + fmt.Printf("Name: %s, Height: %d, %d\n", changes[0].Name, changes[0].Height, + changes[len(changes)-1].Height) + return true + }) + + return nil + }, +} diff --git a/claimtrie/merkletrie/merkletrie.go b/claimtrie/merkletrie/merkletrie.go index 3c279652..5f14107c 100644 --- a/claimtrie/merkletrie/merkletrie.go +++ b/claimtrie/merkletrie/merkletrie.go @@ -3,6 +3,7 @@ package merkletrie import ( "bytes" "fmt" + "runtime" "sort" "sync" @@ -55,6 +56,7 @@ func NewPersistentTrie(store ValueStore, repo Repo) *PersistentTrie { // SetRoot drops all resolved nodes in the PersistentTrie, and set the Root with specified hash. func (t *PersistentTrie) SetRoot(h *chainhash.Hash, names [][]byte) { t.root = newVertex(h) + runtime.GC() } // Update updates the nodes along the path to the key. diff --git a/claimtrie/merkletrie/ramtrie.go b/claimtrie/merkletrie/ramtrie.go index 77620f06..35190675 100644 --- a/claimtrie/merkletrie/ramtrie.go +++ b/claimtrie/merkletrie/ramtrie.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/btcsuite/btcd/chaincfg/chainhash" "github.com/btcsuite/btcd/claimtrie/node" + "runtime" "sync" ) @@ -35,6 +36,7 @@ func NewRamTrie(s ValueStore) *RamTrie { func (rt *RamTrie) SetRoot(h *chainhash.Hash, names [][]byte) { if rt.Root.merkleHash.IsEqual(h) { + runtime.GC() return } @@ -51,6 +53,7 @@ func (rt *RamTrie) SetRoot(h *chainhash.Hash, names [][]byte) { rt.Update(name, false) } } + runtime.GC() } func (rt *RamTrie) Update(name []byte, _ bool) { diff --git a/claimtrie/node/manager.go b/claimtrie/node/manager.go index acbf4bf7..ac60ec14 100644 --- a/claimtrie/node/manager.go +++ b/claimtrie/node/manager.go @@ -66,6 +66,9 @@ func (nm *BaseManager) Node(name []byte) (*Node, error) { return nil, nil } + if len(nm.cache) > param.MaxNodeManagerCacheSize { + nm.cache = map[string]*Node{} // TODO: let's get a real LRU cache in here + } nm.cache[nameStr] = n return n, nil } @@ -90,6 +93,7 @@ func (nm *BaseManager) newNodeFromChanges(changes []change.Change, height int32) count = i break } + if previous < chg.Height { n.AdjustTo(previous, chg.Height-1, chg.Name) // update bids and activation previous = chg.Height @@ -111,16 +115,6 @@ func (nm *BaseManager) newNodeFromChanges(changes []change.Change, height int32) func (nm *BaseManager) AppendChange(chg change.Change) error { - if len(nm.changes) <= 0 { - // this little code block is acting as a "block complete" method - // that could be called after the merkle hash is complete - if len(nm.cache) > param.MaxNodeManagerCacheSize { - // TODO: use a better cache model? - fmt.Printf("Clearing manager cache at height %d\n", nm.height) - nm.cache = map[string]*Node{} - } - } - delete(nm.cache, string(chg.Name)) nm.changes = append(nm.changes, chg) @@ -171,7 +165,12 @@ func (nm *BaseManager) DecrementHeightTo(affectedNames [][]byte, height int32) e } func (nm *BaseManager) getDelayForName(n *Node, chg change.Change) int32 { - hasBest := n.BestClaim != nil // && n.BestClaim.Status == Activated + // Note: we don't consider the active status of BestClaim here on purpose. + // That's because we deactivate and reactivate as part of claim updates. + // However, the final status will be accounted for when we compute the takeover heights; + // claims may get activated early at that point. + + hasBest := n.BestClaim != nil if hasBest && n.BestClaim.ClaimID == chg.ClaimID { return 0 } @@ -182,10 +181,8 @@ func (nm *BaseManager) getDelayForName(n *Node, chg change.Change) int32 { return 0 } - needsWorkaround := nm.decideIfWorkaroundNeeded(n, chg) - delay := calculateDelay(chg.Height, n.TakenOverAt) - if delay > 0 && needsWorkaround { + if delay > 0 && nm.aWorkaroundIsNeeded(n, chg) { // TODO: log this (but only once per name-height combo) //fmt.Printf("Delay workaround applies to %s at %d\n", chg.Name, chg.Height) return 0 @@ -193,30 +190,55 @@ func (nm *BaseManager) getDelayForName(n *Node, chg change.Change) int32 { return delay } -// decideIfWorkaroundNeeded handles bugs that existed in previous versions -func (nm *BaseManager) decideIfWorkaroundNeeded(n *Node, chg change.Change) bool { +func isInDelayPart2(chg change.Change) bool { + heights, ok := param.DelayWorkaroundsPart2[string(chg.Name)] + if ok { + for _, h := range heights { + if h == chg.Height { + return true + } + } + } + return false +} + +func hasZeroActiveClaims(n *Node) bool { + // this isn't quite the same as having an active best (since that is only updated after all changes are processed) + for _, c := range n.Claims { + if c.Status == Activated { + return false + } + } + return true +} + +// aWorkaroundIsNeeded handles bugs that existed in previous versions +func (nm *BaseManager) aWorkaroundIsNeeded(n *Node, chg change.Change) bool { + + if chg.Type == change.SpendClaim || chg.Type == change.SpendSupport { + return false + } if chg.Height >= param.MaxRemovalWorkaroundHeight { // TODO: hard fork this out; it's a bug from previous versions: + // old 17.3 C++ code we're trying to mimic (where empty means no active claims): + // auto it = nodesToAddOrUpdate.find(name); // nodesToAddOrUpdate is the working changes, base is previous block + // auto answer = (it || (it = base->find(name))) && !it->empty() ? nNextHeight - it->nHeightOfLastTakeover : 0; + + needed := hasZeroActiveClaims(n) && nm.hasChildren(chg.Name, chg.Height, 2) if chg.Height <= 933294 { - heights, ok := param.DelayWorkaroundsPart2[string(chg.Name)] - if ok { - for _, h := range heights { - if h == chg.Height { - //hc := nm.hasChildrenButNoSelf(chg.Name, chg.Height, 2) - hc := true - fmt.Printf("HC: %s: %t\n", chg.Name, hc) - return true - } + w := isInDelayPart2(chg) + if w { + if !needed { + fmt.Printf("FALSE NEGATIVE! %d: %s: %t\n", chg.Height, chg.Name, needed) } + } else if needed { + fmt.Printf("FALSE POSITIVE! %d: %s: %t\n", chg.Height, chg.Name, needed) } - } else { - // Known hits: - if nm.hasChildrenButNoSelf(chg.Name, chg.Height, 2) { - return true - } + // return w // if you want to sync to 933294+ } + return needed } else if len(n.Claims) > 0 { // NOTE: old code had a bug in it where nodes with no claims but with children would get left in the cache after removal. // This would cause the getNumBlocksOfContinuousOwnership to return zero (causing incorrect takeover height calc). @@ -266,7 +288,7 @@ func (nm *BaseManager) Close() error { return nil } -func (nm *BaseManager) hasChildrenButNoSelf(name []byte, height int32, required int) bool { +func (nm *BaseManager) hasChildren(name []byte, height int32, required int) bool { c := map[byte]bool{} nm.repo.IterateChildren(name, func(changes []change.Change) bool { @@ -275,11 +297,11 @@ func (nm *BaseManager) hasChildrenButNoSelf(name []byte, height int32, required if len(changes) == 0 { return true } + if c[changes[0].Name[len(name)]] { // assuming all names here are longer than starter name + return true // we already checked a similar name + } n, _ := nm.newNodeFromChanges(changes, height) - if n != nil && n.BestClaim != nil && n.BestClaim.Status == Activated { - if len(name) >= len(changes[0].Name) { - return false // hit self - } + if n != nil && n.HasActiveBestClaim() { c[changes[0].Name[len(name)]] = true if len(c) >= required { return false @@ -300,6 +322,7 @@ func (nm *BaseManager) claimHashes(name []byte) *chainhash.Hash { if err != nil || n == nil { return nil } + n.SortClaims() claimHashes := make([]*chainhash.Hash, 0, len(n.Claims)) for _, c := range n.Claims { diff --git a/claimtrie/node/node.go b/claimtrie/node/node.go index 66dfa0db..a650ee13 100644 --- a/claimtrie/node/node.go +++ b/claimtrie/node/node.go @@ -25,6 +25,10 @@ func New() *Node { return &Node{SupportSums: map[string]int64{}} } +func (n *Node) HasActiveBestClaim() bool { + return n.BestClaim != nil && n.BestClaim.Status == Activated +} + func (n *Node) ApplyChange(chg change.Change, delay int32) error { out := NewOutPointFromString(chg.OutPoint) @@ -132,7 +136,7 @@ func (n *Node) updateTakeoverHeight(height int32, name []byte, refindBest bool) } hasCandidate := candidate != nil - hasCurrentWinner := n.BestClaim != nil && n.BestClaim.Status == Activated + hasCurrentWinner := n.HasActiveBestClaim() takeoverHappening := !hasCandidate || !hasCurrentWinner || candidate.ClaimID != n.BestClaim.ClaimID diff --git a/claimtrie/node/noderepo/pebble.go b/claimtrie/node/noderepo/pebble.go index 3da46cd9..06dc326d 100644 --- a/claimtrie/node/noderepo/pebble.go +++ b/claimtrie/node/noderepo/pebble.go @@ -107,13 +107,18 @@ func (repo *Pebble) DropChanges(name []byte, finalHeight int32) error { } func (repo *Pebble) IterateChildren(name []byte, f func(changes []change.Change) bool) { - end := bytes.NewBuffer(nil) - end.Write(name) - end.Write(bytes.Repeat([]byte{255, 255, 255, 255}, 64)) + start := make([]byte, len(name)+1) // zeros that last byte; need a constant len for stack alloc? + copy(start, name) + + end := make([]byte, 256) // max name length is 255 + copy(end, name) + for i := len(name); i < 256; i++ { + end[i] = 255 + } prefixIterOptions := &pebble.IterOptions{ - LowerBound: name, - UpperBound: end.Bytes(), + LowerBound: start, + UpperBound: end, } iter := repo.db.NewIter(prefixIterOptions)