rocksdb (#29)
* Initial rocksdb commit
Basic reading from rocksdb works
* Try github action thing
* try local dockerfile
* asdf
* qwer
* asdf
* Try adding test db with git-lfs
* update action
* cleanup
* Don't hardcode stop on read
* Progress of reading rocksdb
* fixes and arg test
* asdf
* Fix rocksdb iterator and tests
* update script
* asdf
* Better iterator. Need to implement a lot of keys next, and tests, maybe
tests needed.
* asdf
* asdf
* asdf
* Implementation, testing, and cleanup.
Implemented more prefixes. Figured out a good test that should work for
all prefixes. Removed binary databases so we can just store human
readable csv files.
* more tests, prefixes and small refactor
* Another prefix
* EffectiveAmount
* ActiveAmount
* ActivatedClaimAndSupport
* PendingActivation
* ClaimTakeover
* ClaimExpiration
* SupportToClaim
* ClaimToSupport
* Fix bug with variable length keys
* ChannelToClaim
* ClaimToChannel
* ClaimShortID
* TXOToClaim
* ClaimToTXO
* BlockHeader
* BlockHash
* Undo
* HashXHistory
* Tx and big refactor
* rest the the keys
* Refactor and starting to add resolve
* asdf
* Refactor tests and add column families
* changes
* more work on implementing resolve
* code cleanup, function tests
* small code refactoring
* start building pieces of the test data set for full resolve.
* Export constant, add test
* another test
* TestGetTxHash
* more tests
* more tests
* More tests
* Refactor db functions into three files
* added slice backed stack, need to fix tests
* fix some issues with test suite
* some cleanup and adding arguments and db load / refresh to server command
* fix some bugs, start using logrus for leveled logging, upgrade to go 1.17, run go mod tidy
* logrus, protobuf updates, resolve grpc endpoint
* don't run integration test with unit tests
* signal handling and cleanup functions
* signal handling code files
* Unit tests for db stack
* reorganize bisect function so we lock it properly
* fix txcounts loading
* cleanup some logic around iterators and fix a bug where I was running two detect changes threads
* add some metrics
* cleanup
* blocking and filtering implemented
* add params for blocking and filtering channels and streams
* updates and fixes for integration tests
* use newer version of lbry.go when possible
* Add height endpoint and move string functions internal
* remove gitattributes, unused
* some cleanup
* more cleanup / refactor. almost ready for another review
* More cleanup
* use chainhash.Hash types from lbcd where appropriate
* update github action to go-1.17.8
* update go version needed
* trying to fix these builds
* cleanup
* trying to fix memory leak
* fix memory leak (iterator never finished so cleanup didn't run)
* changes per code review
* remove lbry.go v2
* rename sort.go search.go
* fix test
2022-04-29 17:04:01 +02:00
|
|
|
package stack
|
|
|
|
|
|
|
|
// The db_stack package contains the implementation of a generic slice backed stack
|
|
|
|
// used for tracking various states in the hub, i.e. headers and txcounts
|
|
|
|
|
|
|
|
import (
|
|
|
|
"sync"
|
|
|
|
|
2022-08-09 13:43:01 +02:00
|
|
|
"github.com/lbryio/herald.go/internal"
|
rocksdb (#29)
* Initial rocksdb commit
Basic reading from rocksdb works
* Try github action thing
* try local dockerfile
* asdf
* qwer
* asdf
* Try adding test db with git-lfs
* update action
* cleanup
* Don't hardcode stop on read
* Progress of reading rocksdb
* fixes and arg test
* asdf
* Fix rocksdb iterator and tests
* update script
* asdf
* Better iterator. Need to implement a lot of keys next, and tests, maybe
tests needed.
* asdf
* asdf
* asdf
* Implementation, testing, and cleanup.
Implemented more prefixes. Figured out a good test that should work for
all prefixes. Removed binary databases so we can just store human
readable csv files.
* more tests, prefixes and small refactor
* Another prefix
* EffectiveAmount
* ActiveAmount
* ActivatedClaimAndSupport
* PendingActivation
* ClaimTakeover
* ClaimExpiration
* SupportToClaim
* ClaimToSupport
* Fix bug with variable length keys
* ChannelToClaim
* ClaimToChannel
* ClaimShortID
* TXOToClaim
* ClaimToTXO
* BlockHeader
* BlockHash
* Undo
* HashXHistory
* Tx and big refactor
* rest the the keys
* Refactor and starting to add resolve
* asdf
* Refactor tests and add column families
* changes
* more work on implementing resolve
* code cleanup, function tests
* small code refactoring
* start building pieces of the test data set for full resolve.
* Export constant, add test
* another test
* TestGetTxHash
* more tests
* more tests
* More tests
* Refactor db functions into three files
* added slice backed stack, need to fix tests
* fix some issues with test suite
* some cleanup and adding arguments and db load / refresh to server command
* fix some bugs, start using logrus for leveled logging, upgrade to go 1.17, run go mod tidy
* logrus, protobuf updates, resolve grpc endpoint
* don't run integration test with unit tests
* signal handling and cleanup functions
* signal handling code files
* Unit tests for db stack
* reorganize bisect function so we lock it properly
* fix txcounts loading
* cleanup some logic around iterators and fix a bug where I was running two detect changes threads
* add some metrics
* cleanup
* blocking and filtering implemented
* add params for blocking and filtering channels and streams
* updates and fixes for integration tests
* use newer version of lbry.go when possible
* Add height endpoint and move string functions internal
* remove gitattributes, unused
* some cleanup
* more cleanup / refactor. almost ready for another review
* More cleanup
* use chainhash.Hash types from lbcd where appropriate
* update github action to go-1.17.8
* update go version needed
* trying to fix these builds
* cleanup
* trying to fix memory leak
* fix memory leak (iterator never finished so cleanup didn't run)
* changes per code review
* remove lbry.go v2
* rename sort.go search.go
* fix test
2022-04-29 17:04:01 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
type SliceBacked struct {
|
|
|
|
slice []interface{}
|
|
|
|
len uint32
|
|
|
|
mut sync.RWMutex
|
|
|
|
}
|
|
|
|
|
|
|
|
func NewSliceBacked(size int) *SliceBacked {
|
|
|
|
return &SliceBacked{
|
|
|
|
slice: make([]interface{}, size),
|
|
|
|
len: 0,
|
|
|
|
mut: sync.RWMutex{},
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *SliceBacked) Push(v interface{}) {
|
|
|
|
s.mut.Lock()
|
|
|
|
defer s.mut.Unlock()
|
|
|
|
|
|
|
|
if s.len == uint32(len(s.slice)) {
|
|
|
|
s.slice = append(s.slice, v)
|
|
|
|
} else {
|
|
|
|
s.slice[s.len] = v
|
|
|
|
}
|
|
|
|
s.len++
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *SliceBacked) Pop() interface{} {
|
|
|
|
s.mut.Lock()
|
|
|
|
defer s.mut.Unlock()
|
|
|
|
|
|
|
|
if s.len == 0 {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
s.len--
|
|
|
|
return s.slice[s.len]
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *SliceBacked) Get(i uint32) interface{} {
|
|
|
|
s.mut.RLock()
|
|
|
|
defer s.mut.RUnlock()
|
|
|
|
|
|
|
|
if i >= s.len {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
return s.slice[i]
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *SliceBacked) GetTip() interface{} {
|
|
|
|
s.mut.RLock()
|
|
|
|
defer s.mut.RUnlock()
|
|
|
|
|
|
|
|
if s.len == 0 {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
return s.slice[s.len-1]
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *SliceBacked) Len() uint32 {
|
|
|
|
s.mut.RLock()
|
|
|
|
defer s.mut.RUnlock()
|
|
|
|
|
|
|
|
return s.len
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *SliceBacked) Cap() int {
|
|
|
|
s.mut.RLock()
|
|
|
|
defer s.mut.RUnlock()
|
|
|
|
|
|
|
|
return cap(s.slice)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *SliceBacked) GetSlice() []interface{} {
|
|
|
|
// This is not thread safe so I won't bother with locking
|
|
|
|
return s.slice
|
|
|
|
}
|
|
|
|
|
|
|
|
// This function is dangerous because it assumes underlying types
|
|
|
|
func (s *SliceBacked) TxCountsBisectRight(txNum, rootTxNum uint32) (uint32, uint32) {
|
|
|
|
s.mut.RLock()
|
|
|
|
defer s.mut.RUnlock()
|
|
|
|
|
|
|
|
txCounts := s.slice[:s.Len()]
|
|
|
|
height := internal.BisectRight(txCounts, txNum)
|
|
|
|
createdHeight := internal.BisectRight(txCounts, rootTxNum)
|
|
|
|
|
|
|
|
return height, createdHeight
|
|
|
|
}
|