From 7659a134c93b823a33eab4e6a8b4c0ee353c6428 Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Tue, 20 Aug 2013 12:11:59 -0400 Subject: [PATCH] Add new tool to rewind the database to a specific height or sha Initial code for a block show/dump tool. Initial code for a simple block inserter. --- util/addblock/addblock.go | 264 +++++++++++++++++++++++++++++++++ util/dropafter/dropafter.go | 197 +++++++++++++++++++++++++ util/showblock/showblock.go | 283 ++++++++++++++++++++++++++++++++++++ 3 files changed, 744 insertions(+) create mode 100644 util/addblock/addblock.go create mode 100644 util/dropafter/dropafter.go create mode 100644 util/showblock/showblock.go diff --git a/util/addblock/addblock.go b/util/addblock/addblock.go new file mode 100644 index 00000000..8c63ab89 --- /dev/null +++ b/util/addblock/addblock.go @@ -0,0 +1,264 @@ +/* + * Copyright (c) 2013 Conformal Systems LLC. + */ + +package main + +import ( + "encoding/binary" + "flag" + "fmt" + "github.com/conformal/btcdb" + "github.com/conformal/btcutil" + _ "github.com/conformal/btcdb/sqlite3" + _ "github.com/conformal/btcdb/ldb" + "github.com/conformal/btcwire" + "github.com/conformal/seelog" + "io" + "os" + "path/filepath" + "runtime" + "runtime/pprof" +) + +type ShaHash btcwire.ShaHash + +var log seelog.LoggerInterface + +const ( + ArgSha = iota + ArgHeight +) + +type bufQueue struct { + height int64 + blkbuf []byte +} + +type blkQueue struct { + complete chan bool + height int64 + blk *btcutil.Block +} + +func main() { + var err error + var dbType string + var datadir string + var infile string + var progress int + flag.StringVar(&dbType, "dbtype", "", "Database backend to use for the Block Chain") + flag.StringVar(&datadir, "datadir", "", "Directory to store data") + flag.StringVar(&infile, "i", "", "infile") + flag.IntVar(&progress, "p", 0, "show progress") + + flag.Parse() + + runtime.GOMAXPROCS(runtime.NumCPU()) + + if len (infile) == 0 { + fmt.Printf("Must specify inputfile") + return + } + + log, err = seelog.LoggerFromWriterWithMinLevel(os.Stdout, + seelog.InfoLvl) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to create logger: %v", err) + return + } + defer log.Flush() + btcdb.UseLogger(log) + + if len(dbType) == 0 { + dbType = "sqlite" + } + + if len(datadir) == 0 { + datadir = filepath.Join(btcdHomeDir(), "data") + } + datadir = filepath.Join(datadir, "mainnet") + + err = os.MkdirAll(datadir, 0700) + if err != nil { + fmt.Printf("unable to create db repo area %v, %v", datadir, err) + } + + + blockDbNamePrefix := "blocks" + dbName := blockDbNamePrefix + "_" + dbType + if dbType == "sqlite" { + dbName = dbName + ".db" + } + dbPath := filepath.Join(datadir, dbName) + + log.Infof("loading db") + db, err := btcdb.CreateDB(dbType, dbPath) + if err != nil { + log.Warnf("db open failed: %v", err) + return + } + defer db.Close() + log.Infof("db created") + + + var fi io.ReadCloser + + fi, err = os.Open(infile) + if err != nil { + log.Warnf("failed to open file %v, err %v", infile, err) + } + defer func() { + if err := fi.Close(); err != nil { + log.Warn("failed to close file %v %v", infile, err) + } + }() + + bufqueue := make(chan *bufQueue, 2) + blkqueue := make(chan *blkQueue, 2) + + for i := 0; i < runtime.NumCPU(); i++ { + go processBuf(i, bufqueue, blkqueue) + } + go processBuf(0, bufqueue, blkqueue) + + go readBlocks(fi, bufqueue) + + var eheight int64 + doneMap := map [int64] *blkQueue {} + for { + + select { + case blkM := <- blkqueue: + doneMap[blkM.height] = blkM + + for { + if blkP, ok := doneMap[eheight]; ok { + delete(doneMap, eheight) + blkP.complete <- true + db.InsertBlock(blkP.blk) + + if progress != 0 && eheight%int64(progress) == 0 { + log.Infof("Processing block %v", eheight) + } + eheight++ + + if eheight % 2000 == 0 { + f, err := os.Create(fmt.Sprintf("profile.%d", eheight)) + if err == nil { + pprof.WriteHeapProfile(f) + f.Close() + } else { + log.Warnf("profile failed %v", err) + } + } + } else { + break + } + } + } + } +} + +func processBuf(idx int, bufqueue chan *bufQueue, blkqueue chan *blkQueue) { + complete := make (chan bool) + for { + select { + case bq := <- bufqueue: + var blkmsg blkQueue + + blkmsg.height = bq.height + + if len(bq.blkbuf) == 0 { + // we are done + blkqueue <- &blkmsg + } + + blk, err := btcutil.NewBlockFromBytes(bq.blkbuf) + if err != nil { + fmt.Printf("failed to parse block %v", bq.height) + return + } + blkmsg.blk = blk + blkmsg.complete = complete + blkqueue <- &blkmsg + select { + case <- complete: + } + } + } +} + +func readBlocks(fi io.Reader, bufqueue chan *bufQueue) { + var height int64 + for { + var net, blen uint32 + + var bufM bufQueue + bufM.height = height + + // generate and write header values + err := binary.Read(fi, binary.LittleEndian, &net) + if err != nil { + break + bufqueue <- &bufM + } + if net != uint32(btcwire.MainNet) { + fmt.Printf("network mismatch %v %v", + net, uint32(btcwire.MainNet)) + + bufqueue <- &bufM + } + err = binary.Read(fi, binary.LittleEndian, &blen) + if err != nil { + bufqueue <- &bufM + } + blkbuf := make([]byte, blen) + err = binary.Read(fi, binary.LittleEndian, blkbuf) + bufM.blkbuf = blkbuf + bufqueue <- &bufM + height++ + } +} + +// newLogger creates a new seelog logger using the provided logging level and +// log message prefix. +func newLogger(level string, prefix string) seelog.LoggerInterface { + fmtstring := ` + + + + + + + + ` + config := fmt.Sprintf(fmtstring, level, prefix) + + logger, err := seelog.LoggerFromConfigAsString(config) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to create logger: %v", err) + os.Exit(1) + } + + return logger +} + +// btcdHomeDir returns an OS appropriate home directory for btcd. +func btcdHomeDir() string { + // Search for Windows APPDATA first. This won't exist on POSIX OSes. + appData := os.Getenv("APPDATA") + if appData != "" { + return filepath.Join(appData, "btcd") + } + + // Fall back to standard HOME directory that works for most POSIX OSes. + home := os.Getenv("HOME") + if home != "" { + return filepath.Join(home, ".btcd") + } + + // In the worst case, use the current directory. + return "." +} diff --git a/util/dropafter/dropafter.go b/util/dropafter/dropafter.go new file mode 100644 index 00000000..1d29f995 --- /dev/null +++ b/util/dropafter/dropafter.go @@ -0,0 +1,197 @@ +/* +t f + * Copyright (c) 2013 Conformal Systems LLC. + */ + +package main + +import ( + "errors" + "flag" + "fmt" + "github.com/conformal/btcdb" + _ "github.com/conformal/btcdb/ldb" + "github.com/conformal/btcwire" + "github.com/conformal/seelog" + "os" + "path/filepath" + "strconv" +) + +type ShaHash btcwire.ShaHash + +var log seelog.LoggerInterface + +const ( + ArgSha = iota + ArgHeight +) + +func main() { + var err error + var dbType string + var datadir string + var shastring string + flag.StringVar(&dbType, "dbtype", "", "Database backend to use for the Block Chain") + flag.StringVar(&datadir, "datadir", "", "Directory to store data") + flag.StringVar(&shastring, "s", "", "Block sha to process") + + flag.Parse() + + log, err = seelog.LoggerFromWriterWithMinLevel(os.Stdout, + seelog.TraceLvl) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to create logger: %v", err) + return + } + defer log.Flush() + btcdb.UseLogger(log) + + if len(dbType) == 0 { + dbType = "sqlite" + } + + if len(datadir) == 0 { + datadir = filepath.Join(btcdHomeDir(), "data") + } + datadir = filepath.Join(datadir, "mainnet") + + blockDbNamePrefix := "blocks" + dbName := blockDbNamePrefix + "_" + dbType + if dbType == "sqlite" { + dbName = dbName + ".db" + } + dbPath := filepath.Join(datadir, dbName) + + log.Infof("loading db") + db, err := btcdb.OpenDB(dbType, dbPath) + if err != nil { + log.Warnf("db open failed: %v", err) + return + } + defer db.Close() + log.Infof("db load complete") + + _, height, err := db.NewestSha() + log.Infof("loaded block height %v", height) + + sha, err := getSha(db, shastring) + if err != nil { + log.Infof("Invalid block %v", shastring) + return + } + + err = db.DropAfterBlockBySha(&sha) + if err != nil { + log.Warnf("failed %v", err) + } + +} + +func getSha(db btcdb.Db, str string) (btcwire.ShaHash, error) { + argtype, idx, sha, err := parsesha(str) + if err != nil { + log.Warnf("unable to decode [%v] %v", str, err) + return btcwire.ShaHash{}, err + } + + switch argtype { + case ArgSha: + // nothing to do + case ArgHeight: + sha, err = db.FetchBlockShaByHeight(idx) + if err != nil { + return btcwire.ShaHash{}, err + } + } + if sha == nil { + fmt.Printf("wtf sha is nil but err is %v", err) + } + return *sha, nil +} + + +var ntxcnt int64 +var txspendcnt int64 +var txgivecnt int64 + +var ErrBadShaPrefix = errors.New("invalid prefix") +var ErrBadShaLen = errors.New("invalid len") +var ErrBadShaChar = errors.New("invalid character") + +func parsesha(argstr string) (argtype int, height int64, psha *btcwire.ShaHash, err error) { + var sha btcwire.ShaHash + + var hashbuf string + + switch len(argstr) { + case 64: + hashbuf = argstr + case 66: + if argstr[0:2] != "0x" { + log.Infof("prefix is %v", argstr[0:2]) + err = ErrBadShaPrefix + return + } + hashbuf = argstr[2:] + default: + if len(argstr) <= 16 { + // assume value is height + argtype = ArgHeight + var h int + h, err = strconv.Atoi(argstr) + if err == nil { + height = int64(h) + return + } + log.Infof("Unable to parse height %v, err %v", height, err) + } + err = ErrBadShaLen + return + } + + var buf [32]byte + for idx, ch := range hashbuf { + var val rune + + switch { + case ch >= '0' && ch <= '9': + val = ch - '0' + case ch >= 'a' && ch <= 'f': + val = ch - 'a' + rune(10) + case ch >= 'A' && ch <= 'F': + val = ch - 'A' + rune(10) + default: + err = ErrBadShaChar + return + } + b := buf[31-idx/2] + if idx&1 == 1 { + b |= byte(val) + } else { + b |= (byte(val) << 4) + } + buf[31-idx/2] = b + } + sha.SetBytes(buf[0:32]) + psha = &sha + return +} + +// btcdHomeDir returns an OS appropriate home directory for btcd. +func btcdHomeDir() string { + // Search for Windows APPDATA first. This won't exist on POSIX OSes. + appData := os.Getenv("APPDATA") + if appData != "" { + return filepath.Join(appData, "btcd") + } + + // Fall back to standard HOME directory that works for most POSIX OSes. + home := os.Getenv("HOME") + if home != "" { + return filepath.Join(home, ".btcd") + } + + // In the worst case, use the current directory. + return "." +} diff --git a/util/showblock/showblock.go b/util/showblock/showblock.go new file mode 100644 index 00000000..82e2dc86 --- /dev/null +++ b/util/showblock/showblock.go @@ -0,0 +1,283 @@ +/* + * Copyright (c) 2013 Conformal Systems LLC. + */ + +package main + +import ( + "encoding/binary" + "errors" + "flag" + "fmt" + "github.com/conformal/btcdb" + _ "github.com/conformal/btcdb/sqlite3" + _ "github.com/conformal/btcdb/ldb" + "github.com/conformal/btcwire" + "github.com/conformal/seelog" + "github.com/davecgh/go-spew/spew" + "io" + "os" + "path/filepath" + "strconv" +) + +type ShaHash btcwire.ShaHash + +var log seelog.LoggerInterface + +const ( + ArgSha = iota + ArgHeight +) + +func main() { + var err error + var dbType string + var datadir string + var shastring, eshastring, outfile string + var rflag, fflag, tflag bool + var progress int + end := int64(-1) + flag.StringVar(&dbType, "dbtype", "", "Database backend to use for the Block Chain") + flag.StringVar(&datadir, "datadir", ".", "Directory to store data") + + flag.StringVar(&shastring, "s", "", "Block sha to process") + flag.StringVar(&eshastring, "e", "", "Block sha to process") + flag.StringVar(&outfile, "o", "", "outfile") + flag.BoolVar(&rflag, "r", false, "raw block") + flag.BoolVar(&fflag, "f", false, "fmt block") + flag.BoolVar(&tflag, "t", false, "show transactions") + flag.IntVar(&progress, "p", 0, "show progress") + + flag.Parse() + + log, err = seelog.LoggerFromWriterWithMinLevel(os.Stdout, + seelog.InfoLvl) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to create logger: %v", err) + return + } + defer log.Flush() + btcdb.UseLogger(log) + + if len(dbType) == 0 { + dbType = "sqlite" + } + + if len(datadir) == 0 { + datadir = filepath.Join(btcdHomeDir(), "data") + } + datadir = filepath.Join(datadir, "mainnet") + + blockDbNamePrefix := "blocks" + dbName := blockDbNamePrefix + "_" + dbType + if dbType == "sqlite" { + dbName = dbName + ".db" + } + dbPath := filepath.Join(datadir, dbName) + + + log.Infof("loading db %v", dbType) + db, err := btcdb.OpenDB(dbType, dbPath) + if err != nil { + log.Warnf("db open failed: %v", err) + return + } + defer db.Close() + log.Infof("db load complete") + + height, err := getHeight(db, shastring) + if err != nil { + log.Infof("Invalid block %v", shastring) + return + } + if eshastring != "" { + end, err = getHeight(db, eshastring) + if err != nil { + log.Infof("Invalid end block %v", eshastring) + return + } + } else { + end = height + 1 + } + + log.Infof("height %v end %v", height, end) + + var fo io.WriteCloser + if outfile != "" { + fo, err = os.Create(outfile) + if err != nil { + log.Warnf("failed to open file %v, err %v", outfile, err) + } + defer func() { + if err := fo.Close(); err != nil { + log.Warn("failed to close file %v %v", outfile, err) + } + }() + } + + for ; height < end; height++ { + if progress != 0 && height%int64(progress) == 0 { + log.Infof("Processing block %v", height) + } + err = DumpBlock(db, height, fo, rflag, fflag, tflag) + if err != nil { + break + } + } + if progress != 0 { + height-- + log.Infof("Processing block %v", height) + } +} + +func getHeight(db btcdb.Db, str string) (int64, error) { + argtype, idx, sha, err := parsesha(str) + if err != nil { + log.Warnf("unable to decode [%v] %v", str, err) + return 0, err + } + + switch argtype { + case ArgSha: + // nothing to do + blk, err := db.FetchBlockBySha(sha) + if err != nil { + log.Warnf("unable to locate block sha %v err %v", + sha, err) + return 0, err + } + idx = blk.Height() + case ArgHeight: + } + return idx, nil +} + +func DumpBlock(db btcdb.Db, height int64, fo io.Writer, rflag bool, fflag bool, tflag bool) error { + sha, err := db.FetchBlockShaByHeight(height) + + if err != nil { + return err + } + blk, err := db.FetchBlockBySha(sha) + if err != nil { + log.Warnf("Failed to fetch block %v, err %v", sha, err) + return err + } + rblk, err := blk.Bytes() + blkid := blk.Height() + + if rflag { + log.Infof("Block %v depth %v %v", sha, blkid, spew.Sdump(rblk)) + } + + mblk := blk.MsgBlock() + if fflag { + log.Infof("Block %v depth %v %v", sha, blkid, spew.Sdump(mblk)) + } + if tflag { + log.Infof("Num transactions %v", len(mblk.Transactions)) + for i, tx := range mblk.Transactions { + txsha, err := tx.TxSha() + if err != nil { + continue + } + log.Infof("tx %v: %v", i, &txsha) + + } + } + if fo != nil { + // generate and write header values + binary.Write(fo, binary.LittleEndian, uint32(btcwire.MainNet)) + binary.Write(fo, binary.LittleEndian, uint32(len(rblk))) + + // write block + fo.Write(rblk) + } + return nil +} + +var ntxcnt int64 +var txspendcnt int64 +var txgivecnt int64 + +var ErrBadShaPrefix = errors.New("invalid prefix") +var ErrBadShaLen = errors.New("invalid len") +var ErrBadShaChar = errors.New("invalid character") + +func parsesha(argstr string) (argtype int, height int64, psha *btcwire.ShaHash, err error) { + var sha btcwire.ShaHash + + var hashbuf string + + switch len(argstr) { + case 64: + hashbuf = argstr + case 66: + if argstr[0:2] != "0x" { + log.Infof("prefix is %v", argstr[0:2]) + err = ErrBadShaPrefix + return + } + hashbuf = argstr[2:] + default: + if len(argstr) <= 16 { + // assume value is height + argtype = ArgHeight + var h int + h, err = strconv.Atoi(argstr) + if err == nil { + height = int64(h) + return + } + log.Infof("Unable to parse height %v, err %v", height, err) + } + err = ErrBadShaLen + return + } + + var buf [32]byte + for idx, ch := range hashbuf { + var val rune + + switch { + case ch >= '0' && ch <= '9': + val = ch - '0' + case ch >= 'a' && ch <= 'f': + val = ch - 'a' + rune(10) + case ch >= 'A' && ch <= 'F': + val = ch - 'A' + rune(10) + default: + err = ErrBadShaChar + return + } + b := buf[31-idx/2] + if idx&1 == 1 { + b |= byte(val) + } else { + b |= (byte(val) << 4) + } + buf[31-idx/2] = b + } + sha.SetBytes(buf[0:32]) + psha = &sha + return +} + +// btcdHomeDir returns an OS appropriate home directory for btcd. +func btcdHomeDir() string { + // Search for Windows APPDATA first. This won't exist on POSIX OSes. + appData := os.Getenv("APPDATA") + if appData != "" { + return filepath.Join(appData, "btcd") + } + + // Fall back to standard HOME directory that works for most POSIX OSes. + home := os.Getenv("HOME") + if home != "" { + return filepath.Join(home, ".btcd") + } + + // In the worst case, use the current directory. + return "." +}