LBRYCommunity/lbcd
1
0
Fork 0
Code Issues 17 Pull requests 4 Projects Releases 28 Packages Wiki Activity

[lbry] cli: switch from utxoview to native map.

Browse source 
The utxo is too big to hold in the memory. (~58 GB at 860K blocks)
Since we're extracting claim scripts from an already validated database,
a dumb native map saves us memory as well as overhead maintaining a
uxto.

This takes 8.5 minutes to extract claim scripts in 1M blocks, but
still takes ~56 GB of memory.
This commit is contained in:
Roy Lee 2021-08-03 12:01:01 -07:00
parent 0377a3e7ac
commit 88dbf2267c
1 changed files with 6 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
claimtrie/cmd/cmd/chain.go
View file

@ -331,23 +331,19 @@ func (cb *chainConverter) processBlock() {
defer cb.wg.Done() defer cb.wg.Done()
defer close(cb.changesChan) defer close(cb.changesChan)
view := blockchain.NewUtxoViewpoint() utxoPubScripts := map[wire.OutPoint][]byte{}
for block := range cb.blockChan { for block := range cb.blockChan {
var changes []change.Change var changes []change.Change
for _, tx := range block.Transactions() { for _, tx := range block.Transactions() {
view.AddTxOuts(tx, block.Height())
if blockchain.IsCoinBase(tx) { if blockchain.IsCoinBase(tx) {
continue continue
} }
for _, txIn := range tx.MsgTx().TxIn { for _, txIn := range tx.MsgTx().TxIn {
op := txIn.PreviousOutPoint prevOutpoint := txIn.PreviousOutPoint
e := view.LookupEntry(op) pkScript := utxoPubScripts[prevOutpoint]
if e == nil { cs, err := txscript.DecodeClaimScript(pkScript)
log.Criticalf("Missing input in view for %s", op.String())
}
cs, err := txscript.DecodeClaimScript(e.PkScript())
if err == txscript.ErrNotClaimScript { if err == txscript.ErrNotClaimScript {
continue continue
} }
@ -360,6 +356,7 @@ func (cb *chainConverter) processBlock() {
Name: cs.Name(), Name: cs.Name(),
OutPoint: txIn.PreviousOutPoint, OutPoint: txIn.PreviousOutPoint,
} }
delete(utxoPubScripts, prevOutpoint)
switch cs.Opcode() { switch cs.Opcode() {
case txscript.OP_CLAIMNAME: case txscript.OP_CLAIMNAME:
@ -390,6 +387,7 @@ func (cb *chainConverter) processBlock() {
OutPoint: op, OutPoint: op,
Amount: txOut.Value, Amount: txOut.Value,
} }
utxoPubScripts[op] = txOut.PkScript
switch cs.Opcode() { switch cs.Opcode() {
case txscript.OP_CLAIMNAME: case txscript.OP_CLAIMNAME: