LBRYCommunity/lbcd
1
0
Fork 0
Code Issues17 Pull requests4 Projects Releases28 Packages Wiki Activity

improve addblock

Browse source
This commit is contained in:
David Hill 2013-10-10 14:05:54 -04:00
parent 8a2299c1e1
commit 55b300b6c2
1 changed files with 65 additions and 63 deletions

128
util/addblock/addblock.go
View file

@ -6,13 +6,13 @@ package main
import ( import (
"encoding/binary" "encoding/binary"
"flag"
"fmt" "fmt"
"github.com/conformal/btcdb" "github.com/conformal/btcdb"
_ "github.com/conformal/btcdb/ldb" _ "github.com/conformal/btcdb/ldb"
_ "github.com/conformal/btcdb/sqlite3" _ "github.com/conformal/btcdb/sqlite3"
"github.com/conformal/btcutil" "github.com/conformal/btcutil"
"github.com/conformal/btcwire" "github.com/conformal/btcwire"
"github.com/conformal/go-flags"
"github.com/conformal/seelog" "github.com/conformal/seelog"
"io" "io"
"os" "os"
@ -23,6 +23,14 @@ import (
type ShaHash btcwire.ShaHash type ShaHash btcwire.ShaHash
type config struct {
DataDir string `short:"b" long:"datadir" description:"Directory to store data"`
DbType string `long:"dbtype" description:"Database backend"`
TestNet3 bool `long:"testnet" description:"Use the test network"`
Progress bool `short:"p" description:"show progress"`
InFile string `short:"i" long:"infile" description:"File containing the block(s)" required:"true"`
}
var log seelog.LoggerInterface var log seelog.LoggerInterface
const ( const (
@ -42,25 +50,21 @@ type blkQueue struct {
} }
func main() { func main() {
var err error cfg := config{
var dbType string DbType: "leveldb",
var datadir string DataDir: filepath.Join(btcdHomeDir(), "data"),
var infile string }
var progress int parser := flags.NewParser(&cfg, flags.Default)
flag.StringVar(&dbType, "dbtype", "", "Database backend to use for the Block Chain") _, err := parser.Parse()
flag.StringVar(&datadir, "datadir", "", "Directory to store data") if err != nil {
flag.StringVar(&infile, "i", "", "infile") if e, ok := err.(*flags.Error); !ok || e.Type != flags.ErrHelp {
flag.IntVar(&progress, "p", 0, "show progress") parser.WriteHelp(os.Stderr)
}
flag.Parse()
runtime.GOMAXPROCS(runtime.NumCPU())
if len (infile) == 0 {
fmt.Printf("Must specify inputfile")
return return
} }
runtime.GOMAXPROCS(runtime.NumCPU())
log, err = seelog.LoggerFromWriterWithMinLevel(os.Stdout, log, err = seelog.LoggerFromWriterWithMinLevel(os.Stdout,
seelog.InfoLvl) seelog.InfoLvl)
if err != nil { if err != nil {
@ -70,30 +74,29 @@ func main() {
defer log.Flush() defer log.Flush()
btcdb.UseLogger(log) btcdb.UseLogger(log)
if len(dbType) == 0 { var testnet string
dbType = "sqlite" if cfg.TestNet3 {
testnet = "testnet"
} else {
testnet = "mainnet"
} }
if len(datadir) == 0 { cfg.DataDir = filepath.Join(cfg.DataDir, testnet)
datadir = filepath.Join(btcdHomeDir(), "data")
}
datadir = filepath.Join(datadir, "mainnet")
err = os.MkdirAll(datadir, 0700) err = os.MkdirAll(cfg.DataDir, 0700)
if err != nil { if err != nil {
fmt.Printf("unable to create db repo area %v, %v", datadir, err) fmt.Printf("unable to create db repo area %v, %v", cfg.DataDir, err)
} }
blockDbNamePrefix := "blocks" blockDbNamePrefix := "blocks"
dbName := blockDbNamePrefix + "_" + dbType dbName := blockDbNamePrefix + "_" + cfg.DbType
if dbType == "sqlite" { if cfg.DbType == "sqlite" {
dbName = dbName + ".db" dbName = dbName + ".db"
} }
dbPath := filepath.Join(datadir, dbName) dbPath := filepath.Join(cfg.DataDir, dbName)
log.Infof("loading db") log.Infof("loading db")
db, err := btcdb.CreateDB(dbType, dbPath) db, err := btcdb.CreateDB(cfg.DbType, dbPath)
if err != nil { if err != nil {
log.Warnf("db open failed: %v", err) log.Warnf("db open failed: %v", err)
return return
@ -101,16 +104,15 @@ func main() {
defer db.Close() defer db.Close()
log.Infof("db created") log.Infof("db created")
var fi io.ReadCloser var fi io.ReadCloser
fi, err = os.Open(infile) fi, err = os.Open(cfg.InFile)
if err != nil { if err != nil {
log.Warnf("failed to open file %v, err %v", infile, err) log.Warnf("failed to open file %v, err %v", cfg.InFile, err)
} }
defer func() { defer func() {
if err := fi.Close(); err != nil { if err := fi.Close(); err != nil {
log.Warn("failed to close file %v %v", infile, err) log.Warn("failed to close file %v %v", cfg.InFile, err)
} }
}() }()
@ -125,11 +127,11 @@ func main() {
go readBlocks(fi, bufqueue) go readBlocks(fi, bufqueue)
var eheight int64 var eheight int64
doneMap := map [int64] *blkQueue {} doneMap := map[int64]*blkQueue{}
for { for {
select { select {
case blkM := <- blkqueue: case blkM := <-blkqueue:
doneMap[blkM.height] = blkM doneMap[blkM.height] = blkM
for { for {
@ -138,12 +140,12 @@ func main() {
blkP.complete <- true blkP.complete <- true
db.InsertBlock(blkP.blk) db.InsertBlock(blkP.blk)
if progress != 0 && eheight%int64(progress) == 0 { if cfg.Progress && eheight%int64(1) == 0 {
log.Infof("Processing block %v", eheight) log.Infof("Processing block %v", eheight)
} }
eheight++ eheight++
if eheight % 2000 == 0 { if eheight%2000 == 0 {
f, err := os.Create(fmt.Sprintf("profile.%d", eheight)) f, err := os.Create(fmt.Sprintf("profile.%d", eheight))
if err == nil { if err == nil {
pprof.WriteHeapProfile(f) pprof.WriteHeapProfile(f)
@ -161,10 +163,10 @@ func main() {
} }
func processBuf(idx int, bufqueue chan *bufQueue, blkqueue chan *blkQueue) { func processBuf(idx int, bufqueue chan *bufQueue, blkqueue chan *blkQueue) {
complete := make (chan bool) complete := make(chan bool)
for { for {
select { select {
case bq := <- bufqueue: case bq := <-bufqueue:
var blkmsg blkQueue var blkmsg blkQueue
blkmsg.height = bq.height blkmsg.height = bq.height
@ -174,7 +176,7 @@ func processBuf(idx int, bufqueue chan *bufQueue, blkqueue chan *blkQueue) {
blkqueue <- &blkmsg blkqueue <- &blkmsg
} }
blk, err := btcutil.NewBlockFromBytes(bq.blkbuf) blk, err := btcutil.NewBlockFromBytes(bq.blkbuf)
if err != nil { if err != nil {
fmt.Printf("failed to parse block %v", bq.height) fmt.Printf("failed to parse block %v", bq.height)
return return
@ -183,7 +185,7 @@ func processBuf(idx int, bufqueue chan *bufQueue, blkqueue chan *blkQueue) {
blkmsg.complete = complete blkmsg.complete = complete
blkqueue <- &blkmsg blkqueue <- &blkmsg
select { select {
case <- complete: case <-complete:
} }
} }
} }
@ -204,7 +206,7 @@ func readBlocks(fi io.Reader, bufqueue chan *bufQueue) {
bufqueue <- &bufM bufqueue <- &bufM
} }
if net != uint32(btcwire.MainNet) { if net != uint32(btcwire.MainNet) {
fmt.Printf("network mismatch %v %v", fmt.Printf("network mismatch %v %v",
net, uint32(btcwire.MainNet)) net, uint32(btcwire.MainNet))
bufqueue <- &bufM bufqueue <- &bufM
@ -224,7 +226,7 @@ func readBlocks(fi io.Reader, bufqueue chan *bufQueue) {
// newLogger creates a new seelog logger using the provided logging level and // newLogger creates a new seelog logger using the provided logging level and
// log message prefix. // log message prefix.
func newLogger(level string, prefix string) seelog.LoggerInterface { func newLogger(level string, prefix string) seelog.LoggerInterface {
fmtstring := ` fmtstring := `
<seelog type="adaptive" mininterval="2000000" maxinterval="100000000" <seelog type="adaptive" mininterval="2000000" maxinterval="100000000"
critmsgcount="500" minlevel="%s"> critmsgcount="500" minlevel="%s">
<outputs formatid="all"> <outputs formatid="all">
@ -234,31 +236,31 @@ func newLogger(level string, prefix string) seelog.LoggerInterface {
<format id="all" format="[%%Time %%Date] [%%LEV] [%s] %%Msg%%n" /> <format id="all" format="[%%Time %%Date] [%%LEV] [%s] %%Msg%%n" />
</formats> </formats>
</seelog>` </seelog>`
config := fmt.Sprintf(fmtstring, level, prefix) config := fmt.Sprintf(fmtstring, level, prefix)
logger, err := seelog.LoggerFromConfigAsString(config) logger, err := seelog.LoggerFromConfigAsString(config)
if err != nil { if err != nil {
fmt.Fprintf(os.Stderr, "failed to create logger: %v", err) fmt.Fprintf(os.Stderr, "failed to create logger: %v", err)
os.Exit(1) os.Exit(1)
} }
return logger return logger
} }
// btcdHomeDir returns an OS appropriate home directory for btcd. // btcdHomeDir returns an OS appropriate home directory for btcd.
func btcdHomeDir() string { func btcdHomeDir() string {
// Search for Windows APPDATA first. This won't exist on POSIX OSes. // Search for Windows APPDATA first. This won't exist on POSIX OSes.
appData := os.Getenv("APPDATA") appData := os.Getenv("APPDATA")
if appData != "" { if appData != "" {
return filepath.Join(appData, "btcd") return filepath.Join(appData, "btcd")
} }
// Fall back to standard HOME directory that works for most POSIX OSes. // Fall back to standard HOME directory that works for most POSIX OSes.
home := os.Getenv("HOME") home := os.Getenv("HOME")
if home != "" { if home != "" {
return filepath.Join(home, ".btcd") return filepath.Join(home, ".btcd")
} }
// In the worst case, use the current directory. // In the worst case, use the current directory.
return "." return "."
} }