diff --git a/fees/README.md b/fees/README.md new file mode 100644 index 00000000..03b85696 --- /dev/null +++ b/fees/README.md @@ -0,0 +1,23 @@ +fees +==== + + +[![Build Status](https://github.com/decred/dcrd/workflows/Build%20and%20Test/badge.svg)](https://github.com/decred/dcrd/actions) +[![ISC License](https://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org) +[![Doc](https://img.shields.io/badge/doc-reference-blue.svg)](https://pkg.go.dev/github.com/decred/dcrd/internal/fees) + +Package fees provides decred-specific methods for tracking and estimating fee +rates for new transactions to be mined into the network. Fee rate estimation has +two main goals: + +- Ensuring transactions are mined within a target _confirmation range_ + (expressed in blocks); +- Attempting to minimize fees while maintaining be above restriction. + +This package was started in order to resolve issue decred/dcrd#1412 and related. +See that issue for discussion of the selected approach. + +## License + +Package dcrutil is licensed under the [copyfree](http://copyfree.org) ISC +License. diff --git a/fees/cmd/dumpfeedb/dumpfeedb.go b/fees/cmd/dumpfeedb/dumpfeedb.go new file mode 100644 index 00000000..38e2492a --- /dev/null +++ b/fees/cmd/dumpfeedb/dumpfeedb.go @@ -0,0 +1,52 @@ +// Copyright (c) 2018-2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +// Tool dumpfeedb can be used to dump the internal state of the buckets of an +// estimator's feedb so that it can be externally analyzed. +package main + +import ( + "errors" + "fmt" + "os" + "path" + + "github.com/decred/dcrd/dcrutil/v4" + "github.com/decred/dcrd/internal/fees" + flags "github.com/jessevdk/go-flags" +) + +type config struct { + DB string `short:"b" long:"db" description:"Path to fee database"` +} + +func main() { + cfg := config{ + DB: path.Join(dcrutil.AppDataDir("dcrd", false), "data", "mainnet", "feesdb"), + } + + parser := flags.NewParser(&cfg, flags.Default) + _, err := parser.Parse() + if err != nil { + var e *flags.Error + if !errors.As(err, &e) || e.Type != flags.ErrHelp { + parser.WriteHelp(os.Stderr) + } + return + } + + ecfg := fees.EstimatorConfig{ + DatabaseFile: cfg.DB, + ReplaceBucketsOnLoad: true, + MinBucketFee: 1, + MaxBucketFee: 2, + FeeRateStep: fees.DefaultFeeRateStep, + } + est, err := fees.NewEstimator(&ecfg) + if err != nil { + panic(err) + } + + fmt.Println(est.DumpBuckets()) +} diff --git a/fees/doc.go b/fees/doc.go new file mode 100644 index 00000000..77419317 --- /dev/null +++ b/fees/doc.go @@ -0,0 +1,107 @@ +// Copyright (c) 2018-2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +/* +Package fees provides decred-specific methods for tracking and estimating fee +rates for new transactions to be mined into the network. Fee rate estimation has +two main goals: + +- Ensuring transactions are mined within a target _confirmation range_ + (expressed in blocks); +- Attempting to minimize fees while maintaining be above restriction. + +Preliminaries + +There are two main regimes against which fee estimation needs to be evaluated +according to how full blocks being mined are (and consequently how important fee +rates are): _low contention_ and _high contention_: + +In a low contention regime, the mempool sits mostly empty, transactions are +usually mined very soon after being published and transaction fees are mostly +sent using the minimum relay fee. + +In a high contention regime, the mempool is usually filled with unmined +transactions, there is active dispute for space in a block (by transactions +using higher fees) and blocks are usually full. + +The exact point of where these two regimes intersect is arbitrary, but it should +be clear in the examples and simulations which of these is being discussed. + +Note: a very high contention scenario (> 90% of blocks being full and +transactions remaining in the mempool indefinitely) is one in which stakeholders +should be discussing alternative solutions (increase block size, provide other +second layer alternatives, etc). Also, the current fill rate of blocks in decred +is low, so while we try to account for this regime, I personally expect that the +implementation will need more tweaks as it approaches this. + +The current approach to implement this estimation is based on bitcoin core's +algorithm. References [1] and [2] provide a high level description of how it +works there. Actual code is linked in references [3] and [4]. + +Outline of the Algorithm + +The algorithm is currently based in fee estimation as used in v0.14 of bitcoin +core (which is also the basis for the v0.15+ method). A more comprehensive +overview is available in reference [1]. + +This particular version was chosen because it's simpler to implement and should +be sufficient for low contention regimes. It probably overestimates fees in +higher contention regimes and longer target confirmation windows, but as pointed +out earlier should be sufficient for current fill rate of decred's network. + +The basic algorithm is as follows (as executed by a single full node): + +Stats building stage: + +- For each transaction observed entering mempool, record the block at which it + was first seen +- For each mined transaction which was previously observed to enter the mempool, + record how long (in blocks) it took to be mined and its fee rate +- Group mined transactions into fee rate _buckets_ and _confirmation ranges_, + creating a table of how many transactions were mined at each confirmation + range and fee rate bucket and their total committed fee +- Whenever a new block is mined, decay older transactions to account for a + dynamic fee environment + +Estimation stage: + +- Input a target confirmation range (how many blocks to wait for the tx to be + mined) +- Starting at the highest fee bucket, look for buckets where the chance of + confirmation within the desired confirmation window is > 95% +- Average all such buckets to get the estimated fee rate + +Simulation + +Development of the estimator was originally performed and simulated using the +code in [5]. Simulation of the current code can be performed by using the +dcrfeesim tool available in [6]. + +Acknowledgements + +Thanks to @davecgh for providing the initial review of the results and the +original developers of the bitcoin core code (the brunt of which seems to have +been made by @morcos). + +## References + +[1] Introduction to Bitcoin Core Estimation: +https://bitcointechtalk.com/an-introduction-to-bitcoin-core-fee-estimation-27920880ad0 + +[2] Proposed Changes to Fee Estimation in version 0.15: +https://gist.github.com/morcos/d3637f015bc4e607e1fd10d8351e9f41 + +[3] Source for fee estimation in v0.14: +https://github.com/bitcoin/bitcoin/blob/v0.14.2/src/policy/fees.cpp + +[4] Source for fee estimation in version 0.16.2: +https://github.com/bitcoin/bitcoin/blob/v0.16.2/src/policy/fees.cpp + +[5] Source for the original dcrfeesim and estimator work: +https://github.com/matheusd/dcrfeesim_dev + +[6] Source for the current dcrfeesim, using this module: +https://github.com/matheusd/dcrfeesim +*/ +package fees diff --git a/fees/estimator.go b/fees/estimator.go new file mode 100644 index 00000000..d98f037f --- /dev/null +++ b/fees/estimator.go @@ -0,0 +1,920 @@ +// Copyright (c) 2018-2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package fees + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "math" + "sort" + "sync" + + "github.com/decred/dcrd/blockchain/stake/v4" + "github.com/decred/dcrd/chaincfg/chainhash" + "github.com/decred/dcrd/dcrutil/v4" + "github.com/syndtr/goleveldb/leveldb" + ldbutil "github.com/syndtr/goleveldb/leveldb/util" +) + +const ( + // DefaultMaxBucketFeeMultiplier is the default multiplier used to find the + // largest fee bucket, starting at the minimum fee. + DefaultMaxBucketFeeMultiplier int = 100 + + // DefaultMaxConfirmations is the default number of confirmation ranges to + // track in the estimator. + DefaultMaxConfirmations uint32 = 32 + + // DefaultFeeRateStep is the default multiplier between two consecutive fee + // rate buckets. + DefaultFeeRateStep float64 = 1.1 + + // defaultDecay is the default value used to decay old transactions from the + // estimator. + defaultDecay float64 = 0.998 + + // maxAllowedBucketFees is an upper bound of how many bucket fees can be + // used in the estimator. This is verified during estimator initialization + // and database loading. + maxAllowedBucketFees = 2000 + + // maxAllowedConfirms is an upper bound of how many confirmation ranges can + // be used in the estimator. This is verified during estimator + // initialization and database loading. + maxAllowedConfirms = 788 +) + +var ( + // ErrNoSuccessPctBucketFound is the error returned when no bucket has been + // found with the minimum required percentage success. + ErrNoSuccessPctBucketFound = errors.New("no bucket with the minimum " + + "required success percentage found") + + // ErrNotEnoughTxsForEstimate is the error returned when not enough + // transactions have been seen by the fee generator to give an estimate. + ErrNotEnoughTxsForEstimate = errors.New("not enough transactions seen for " + + "estimation") + + dbByteOrder = binary.BigEndian + + dbKeyVersion = []byte("version") + dbKeyBucketFees = []byte("bucketFeeBounds") + dbKeyMaxConfirms = []byte("maxConfirms") + dbKeyBestHeight = []byte("bestHeight") + dbKeyBucketPrefix = []byte{0x01, 0x70, 0x1d, 0x00} +) + +// ErrTargetConfTooLarge is the type of error returned when an user of the +// estimator requested a confirmation range higher than tracked by the estimator. +type ErrTargetConfTooLarge struct { + MaxConfirms int32 + ReqConfirms int32 +} + +func (e ErrTargetConfTooLarge) Error() string { + return fmt.Sprintf("target confirmation requested (%d) higher than "+ + "maximum confirmation range tracked by estimator (%d)", e.ReqConfirms, + e.MaxConfirms) +} + +type feeRate float64 + +type txConfirmStatBucketCount struct { + txCount float64 + feeSum float64 +} + +type txConfirmStatBucket struct { + confirmed []txConfirmStatBucketCount + confirmCount float64 + feeSum float64 +} + +// EstimatorConfig stores the configuration parameters for a given fee +// estimator. It is used to initialize an empty fee estimator. +type EstimatorConfig struct { + // MaxConfirms is the maximum number of confirmation ranges to check. + MaxConfirms uint32 + + // MinBucketFee is the value of the fee rate of the lowest bucket for which + // estimation is tracked. + MinBucketFee dcrutil.Amount + + // MaxBucketFee is the value of the fee for the highest bucket for which + // estimation is tracked. + // + // It MUST be higher than MinBucketFee. + MaxBucketFee dcrutil.Amount + + // ExtraBucketFee is an additional bucket fee rate to include in the + // database for tracking transactions. Specifying this can be useful when + // the default relay fee of the network is undergoing change (due to a new + // release of the software for example), so that the older fee can be + // tracked exactly. + // + // It MUST have a value between MinBucketFee and MaxBucketFee, otherwise + // it's ignored. + ExtraBucketFee dcrutil.Amount + + // FeeRateStep is the multiplier to generate the fee rate buckets (each + // bucket is higher than the previous one by this factor). + // + // It MUST have a value > 1.0. + FeeRateStep float64 + + // DatabaseFile is the location of the estimator database file. If empty, + // updates to the estimator state are not backed by the filesystem. + DatabaseFile string + + // ReplaceBucketsOnLoad indicates whether to replace the buckets in the + // current estimator by those stored in the feesdb file instead of + // validating that they are both using the same set of fees. + ReplaceBucketsOnLoad bool +} + +// memPoolTxDesc is an aux structure used to track the local estimator mempool. +type memPoolTxDesc struct { + addedHeight int64 + bucketIndex int32 + fees feeRate +} + +// Estimator tracks historical data for published and mined transactions in +// order to estimate fees to be used in new transactions for confirmation +// within a target block window. +type Estimator struct { + // bucketFeeBounds are the upper bounds for each individual fee bucket. + bucketFeeBounds []feeRate + + // buckets are the confirmed tx count and fee sum by bucket fee. + buckets []txConfirmStatBucket + + // memPool are the mempool transaction count and fee sum by bucket fee. + memPool []txConfirmStatBucket + + // memPoolTxs is the map of transaction hashes and data of known mempool txs. + memPoolTxs map[chainhash.Hash]memPoolTxDesc + + maxConfirms int32 + decay float64 + bestHeight int64 + db *leveldb.DB + lock sync.RWMutex +} + +// NewEstimator returns an empty estimator given a config. This estimator +// then needs to be fed data for published and mined transactions before it can +// be used to estimate fees for new transactions. +func NewEstimator(cfg *EstimatorConfig) (*Estimator, error) { + // Sanity check the config. + if cfg.MaxBucketFee <= cfg.MinBucketFee { + return nil, errors.New("maximum bucket fee should not be lower than " + + "minimum bucket fee") + } + if cfg.FeeRateStep <= 1.0 { + return nil, errors.New("fee rate step should not be <= 1.0") + } + if cfg.MinBucketFee <= 0 { + return nil, errors.New("minimum bucket fee rate cannot be <= 0") + } + if cfg.MaxConfirms > maxAllowedConfirms { + return nil, fmt.Errorf("confirmation count requested (%d) larger than "+ + "maximum allowed (%d)", cfg.MaxConfirms, maxAllowedConfirms) + } + + decay := defaultDecay + maxConfirms := cfg.MaxConfirms + max := float64(cfg.MaxBucketFee) + var bucketFees []feeRate + prevF := 0.0 + extraBucketFee := float64(cfg.ExtraBucketFee) + for f := float64(cfg.MinBucketFee); f < max; f *= cfg.FeeRateStep { + if (f > extraBucketFee) && (prevF < extraBucketFee) { + // Add the extra bucket fee for tracking. + bucketFees = append(bucketFees, feeRate(extraBucketFee)) + } + bucketFees = append(bucketFees, feeRate(f)) + prevF = f + } + + // The last bucket catches everything else, so it uses an upper bound of + // +inf which any rate must be lower than. + bucketFees = append(bucketFees, feeRate(math.Inf(1))) + + nbBuckets := len(bucketFees) + res := &Estimator{ + bucketFeeBounds: bucketFees, + buckets: make([]txConfirmStatBucket, nbBuckets), + memPool: make([]txConfirmStatBucket, nbBuckets), + maxConfirms: int32(maxConfirms), + decay: decay, + memPoolTxs: make(map[chainhash.Hash]memPoolTxDesc), + bestHeight: -1, + } + + for i := range bucketFees { + res.buckets[i] = txConfirmStatBucket{ + confirmed: make([]txConfirmStatBucketCount, maxConfirms), + } + res.memPool[i] = txConfirmStatBucket{ + confirmed: make([]txConfirmStatBucketCount, maxConfirms), + } + } + + if cfg.DatabaseFile != "" { + db, err := leveldb.OpenFile(cfg.DatabaseFile, nil) + if err != nil { + return nil, fmt.Errorf("error opening estimator database: %v", err) + } + res.db = db + + err = res.loadFromDatabase(cfg.ReplaceBucketsOnLoad) + if err != nil { + return nil, fmt.Errorf("error loading estimator data from db: %v", + err) + } + } + + return res, nil +} + +// DumpBuckets returns the internal estimator state as a string. +func (stats *Estimator) DumpBuckets() string { + res := " |" + for c := 0; c < int(stats.maxConfirms); c++ { + if c == int(stats.maxConfirms)-1 { + res += fmt.Sprintf(" %15s", "+Inf") + } else { + res += fmt.Sprintf(" %15d|", c+1) + } + } + res += "\n" + + l := len(stats.bucketFeeBounds) + for i := 0; i < l; i++ { + res += fmt.Sprintf("%10.8f", stats.bucketFeeBounds[i]/1e8) + for c := 0; c < int(stats.maxConfirms); c++ { + avg := float64(0) + count := stats.buckets[i].confirmed[c].txCount + if stats.buckets[i].confirmed[c].txCount > 0 { + avg = stats.buckets[i].confirmed[c].feeSum / + stats.buckets[i].confirmed[c].txCount / 1e8 + } + + res += fmt.Sprintf("| %.8f %6.1f", avg, count) + } + res += "\n" + } + + return res +} + +// loadFromDatabase loads the estimator data from the currently opened database +// and performs any db upgrades if required. After loading, it updates the db +// with the current estimator configuration. +// +// Argument replaceBuckets indicates if the buckets in the current stats should +// be completely replaced by what is stored in the database or if the data +// should be validated against what is current in the estimator. +// +// The database should *not* be used while loading is taking place. +// +// The current code does not support loading from a database created with a +// different set of configuration parameters (fee rate buckets, max confirmation +// range, etc) than the current estimator is configured with. If an incompatible +// file is detected during loading, an error is returned and the user must +// either reconfigure the estimator to use the same parameters to allow the +// database to be loaded or they must ignore the database file (possibly by +// deleting it) so that the new parameters are used. In the future it might be +// possible to load from a different set of configuration parameters. +// +// The current code does not currently save mempool information, since saving +// information in the estimator without saving the corresponding data in the +// mempool itself could result in transactions lingering in the mempool +// estimator forever. +func (stats *Estimator) loadFromDatabase(replaceBuckets bool) error { + if stats.db == nil { + return errors.New("estimator database is not open") + } + + // Database version is currently hardcoded here as this is the only + // place that uses it. + currentDbVersion := []byte{1} + + version, err := stats.db.Get(dbKeyVersion, nil) + if err != nil && !errors.Is(err, leveldb.ErrNotFound) { + return fmt.Errorf("error reading version from db: %v", err) + } + if len(version) < 1 { + // No data in the file. Fill with the current config. + batch := new(leveldb.Batch) + b := bytes.NewBuffer(nil) + var maxConfirmsBytes [4]byte + var bestHeightBytes [8]byte + + batch.Put(dbKeyVersion, currentDbVersion) + + dbByteOrder.PutUint32(maxConfirmsBytes[:], uint32(stats.maxConfirms)) + batch.Put(dbKeyMaxConfirms, maxConfirmsBytes[:]) + + dbByteOrder.PutUint64(bestHeightBytes[:], uint64(stats.bestHeight)) + batch.Put(dbKeyBestHeight, bestHeightBytes[:]) + + err := binary.Write(b, dbByteOrder, stats.bucketFeeBounds) + if err != nil { + return fmt.Errorf("error writing bucket fees to db: %v", err) + } + batch.Put(dbKeyBucketFees, b.Bytes()) + + err = stats.db.Write(batch, nil) + if err != nil { + return fmt.Errorf("error writing initial estimator db file: %v", + err) + } + + err = stats.updateDatabase() + if err != nil { + return fmt.Errorf("error adding initial estimator data to db: %v", + err) + } + + log.Debug("Initialized fee estimator database") + + return nil + } + + if !bytes.Equal(currentDbVersion, version) { + return fmt.Errorf("incompatible database version: %d", version) + } + + maxConfirmsBytes, err := stats.db.Get(dbKeyMaxConfirms, nil) + if err != nil { + return fmt.Errorf("error reading max confirmation range from db file: "+ + "%v", err) + } + if len(maxConfirmsBytes) != 4 { + return errors.New("wrong number of bytes in stored maxConfirms") + } + fileMaxConfirms := int32(dbByteOrder.Uint32(maxConfirmsBytes)) + if fileMaxConfirms > maxAllowedConfirms { + return fmt.Errorf("confirmation count stored in database (%d) larger "+ + "than maximum allowed (%d)", fileMaxConfirms, maxAllowedConfirms) + } + + feesBytes, err := stats.db.Get(dbKeyBucketFees, nil) + if err != nil { + return fmt.Errorf("error reading fee bounds from db file: %v", err) + } + if feesBytes == nil { + return errors.New("fee bounds not found in database file") + } + fileNbBucketFees := len(feesBytes) / 8 + if fileNbBucketFees > maxAllowedBucketFees { + return fmt.Errorf("more fee buckets stored in file (%d) than allowed "+ + "(%d)", fileNbBucketFees, maxAllowedBucketFees) + } + fileBucketFees := make([]feeRate, fileNbBucketFees) + err = binary.Read(bytes.NewReader(feesBytes), dbByteOrder, + &fileBucketFees) + if err != nil { + return fmt.Errorf("error decoding file bucket fees: %v", err) + } + + if !replaceBuckets { + if stats.maxConfirms != fileMaxConfirms { + return errors.New("max confirmation range in database file different " + + "than currently configured max confirmation") + } + + if len(stats.bucketFeeBounds) != len(fileBucketFees) { + return errors.New("number of bucket fees stored in database file " + + "different than currently configured bucket fees") + } + + for i, f := range fileBucketFees { + if stats.bucketFeeBounds[i] != f { + return errors.New("bucket fee rates stored in database file " + + "different than currently configured fees") + } + } + } + + fileBuckets := make([]txConfirmStatBucket, fileNbBucketFees) + + iter := stats.db.NewIterator(ldbutil.BytesPrefix(dbKeyBucketPrefix), nil) + err = nil + var fbytes [8]byte + for iter.Next() { + key := iter.Key() + if len(key) != 8 { + err = fmt.Errorf("bucket key read from db has wrong length (%d)", + len(key)) + break + } + idx := int(int32(dbByteOrder.Uint32(key[4:]))) + if (idx >= len(fileBuckets)) || (idx < 0) { + err = fmt.Errorf("wrong bucket index read from db (%d vs %d)", + idx, len(fileBuckets)) + break + } + value := iter.Value() + if len(value) != 8+8+int(fileMaxConfirms)*16 { + err = errors.New("wrong size of data in bucket read from db") + break + } + + b := bytes.NewBuffer(value) + readf := func() float64 { + // We ignore the error here because the only possible one is EOF and + // we already previously checked the length of the source byte array + // for consistency. + b.Read(fbytes[:]) + return math.Float64frombits(dbByteOrder.Uint64(fbytes[:])) + } + + fileBuckets[idx].confirmCount = readf() + fileBuckets[idx].feeSum = readf() + fileBuckets[idx].confirmed = make([]txConfirmStatBucketCount, fileMaxConfirms) + for i := range fileBuckets[idx].confirmed { + fileBuckets[idx].confirmed[i].txCount = readf() + fileBuckets[idx].confirmed[i].feeSum = readf() + } + } + iter.Release() + if err != nil { + return err + } + err = iter.Error() + if err != nil { + return fmt.Errorf("error on bucket iterator: %v", err) + } + + stats.bucketFeeBounds = fileBucketFees + stats.buckets = fileBuckets + stats.maxConfirms = fileMaxConfirms + log.Debug("Loaded fee estimator database") + + return nil +} + +// updateDatabase updates the current database file with the current bucket +// data. This is called during normal operation after processing mined +// transactions, so it only updates data that might have changed. +func (stats *Estimator) updateDatabase() error { + if stats.db == nil { + return errors.New("estimator database is closed") + } + + batch := new(leveldb.Batch) + buf := bytes.NewBuffer(nil) + + var key [8]byte + copy(key[:], dbKeyBucketPrefix) + var fbytes [8]byte + writef := func(f float64) { + dbByteOrder.PutUint64(fbytes[:], math.Float64bits(f)) + _, err := buf.Write(fbytes[:]) + if err != nil { + panic(err) // only possible error is ErrTooLarge + } + } + + for i, b := range stats.buckets { + dbByteOrder.PutUint32(key[4:], uint32(i)) + buf.Reset() + writef(b.confirmCount) + writef(b.feeSum) + for _, c := range b.confirmed { + writef(c.txCount) + writef(c.feeSum) + } + batch.Put(key[:], buf.Bytes()) + } + + var bestHeightBytes [8]byte + + dbByteOrder.PutUint64(bestHeightBytes[:], uint64(stats.bestHeight)) + batch.Put(dbKeyBestHeight, bestHeightBytes[:]) + + err := stats.db.Write(batch, nil) + if err != nil { + return fmt.Errorf("error writing update to estimator db file: %v", + err) + } + + return nil +} + +// lowerBucket returns the bucket that has the highest upperBound such that it +// is still lower than rate. +func (stats *Estimator) lowerBucket(rate feeRate) int32 { + res := sort.Search(len(stats.bucketFeeBounds), func(i int) bool { + return stats.bucketFeeBounds[i] >= rate + }) + return int32(res) +} + +// confirmRange returns the confirmation range index to be used for the given +// number of blocks to confirm. The last confirmation range has an upper bound +// of +inf to mean that it represents all confirmations higher than the second +// to last bucket. +func (stats *Estimator) confirmRange(blocksToConfirm int32) int32 { + idx := blocksToConfirm - 1 + if idx >= stats.maxConfirms { + return stats.maxConfirms - 1 + } + return idx +} + +// updateMovingAverages updates the moving averages for the existing confirmed +// statistics and increases the confirmation ranges for mempool txs. This is +// meant to be called when a new block is mined, so that we discount older +// information. +func (stats *Estimator) updateMovingAverages(newHeight int64) { + log.Debugf("Updated moving averages into block %d", newHeight) + + // decay the existing stats so that, over time, we rely on more up to date + // information regarding fees. + for b := 0; b < len(stats.buckets); b++ { + bucket := &stats.buckets[b] + bucket.feeSum *= stats.decay + bucket.confirmCount *= stats.decay + for c := 0; c < len(bucket.confirmed); c++ { + conf := &bucket.confirmed[c] + conf.feeSum *= stats.decay + conf.txCount *= stats.decay + } + } + + // For unconfirmed (mempool) transactions, every transaction will now take + // at least one additional block to confirm. So for every fee bucket, we + // move the stats up one confirmation range. + for b := 0; b < len(stats.memPool); b++ { + bucket := &stats.memPool[b] + + // The last confirmation range represents all txs confirmed at >= than + // the initial maxConfirms, so we *add* the second to last range into + // the last range. + c := len(bucket.confirmed) - 1 + bucket.confirmed[c].txCount += bucket.confirmed[c-1].txCount + bucket.confirmed[c].feeSum += bucket.confirmed[c-1].feeSum + + // For the other ranges, just move up the stats. + for c--; c > 0; c-- { + bucket.confirmed[c] = bucket.confirmed[c-1] + } + + // and finally, the very first confirmation range (ie, what will enter + // the mempool now that a new block has been mined) is zeroed so we can + // start tracking brand new txs. + bucket.confirmed[0].txCount = 0 + bucket.confirmed[0].feeSum = 0 + } + + stats.bestHeight = newHeight +} + +// newMemPoolTx records a new memPool transaction into the stats. A brand new +// mempool transaction has a minimum confirmation range of 1, so it is inserted +// into the very first confirmation range bucket of the appropriate fee rate +// bucket. +func (stats *Estimator) newMemPoolTx(bucketIdx int32, fees feeRate) { + conf := &stats.memPool[bucketIdx].confirmed[0] + conf.feeSum += float64(fees) + conf.txCount++ +} + +// newMinedTx moves a mined tx from the mempool into the confirmed statistics. +// Note that this should only be called if the transaction had been seen and +// previously tracked by calling newMemPoolTx for it. Failing to observe that +// will result in undefined statistical results. +func (stats *Estimator) newMinedTx(blocksToConfirm int32, rate feeRate) { + bucketIdx := stats.lowerBucket(rate) + confirmIdx := stats.confirmRange(blocksToConfirm) + bucket := &stats.buckets[bucketIdx] + + // increase the counts for all confirmation ranges starting at the first + // confirmIdx because it took at least `blocksToConfirm` for this tx to be + // mined. This is used to simplify the bucket selection during estimation, + // so that we only need to check a single confirmation range (instead of + // iterating to sum all confirmations with <= `minConfs`). + for c := int(confirmIdx); c < len(bucket.confirmed); c++ { + conf := &bucket.confirmed[c] + conf.feeSum += float64(rate) + conf.txCount++ + } + bucket.confirmCount++ + bucket.feeSum += float64(rate) +} + +func (stats *Estimator) removeFromMemPool(blocksInMemPool int32, rate feeRate) { + bucketIdx := stats.lowerBucket(rate) + confirmIdx := stats.confirmRange(blocksInMemPool + 1) + bucket := &stats.memPool[bucketIdx] + conf := &bucket.confirmed[confirmIdx] + conf.feeSum -= float64(rate) + conf.txCount-- + if conf.txCount < 0 { + // If this happens, it means a transaction has been called on this + // function but not on a previous newMemPoolTx. This leaves the fee db + // in an undefined state and should never happen in regular use. If this + // happens, then there is a logic or coding error somewhere, either in + // the estimator itself or on its hooking to the mempool/network sync + // manager. Either way, the easiest way to fix this is to completely + // delete the database and start again. During development, you can use + // a panic() here and we might return it after being confident that the + // estimator is completely bug free. + log.Errorf("Transaction count in bucket index %d and confirmation "+ + "index %d became < 0", bucketIdx, confirmIdx) + } +} + +// estimateMedianFee estimates the median fee rate for the current recorded +// statistics such that at least successPct transactions have been mined on all +// tracked fee rate buckets with fee >= to the median. +// In other words, this is the median fee of the lowest bucket such that it and +// all higher fee buckets have >= successPct transactions confirmed in at most +// `targetConfs` confirmations. +// Note that sometimes the requested combination of targetConfs and successPct is +// not achievable (hypothetical example: 99% of txs confirmed within 1 block) +// or there are not enough recorded statistics to derive a successful estimate +// (eg: confirmation tracking has only started or there was a period of very few +// transactions). In those situations, the appropriate error is returned. +func (stats *Estimator) estimateMedianFee(targetConfs int32, successPct float64) (feeRate, error) { + if targetConfs <= 0 { + return 0, errors.New("target confirmation range cannot be <= 0") + } + + const minTxCount float64 = 1 + + if (targetConfs - 1) >= stats.maxConfirms { + // We might want to add support to use a targetConf at +infinity to + // allow us to make estimates at confirmation interval higher than what + // we currently track. + return 0, ErrTargetConfTooLarge{MaxConfirms: stats.maxConfirms, + ReqConfirms: targetConfs} + } + + startIdx := len(stats.buckets) - 1 + confirmRangeIdx := stats.confirmRange(targetConfs) + + var totalTxs, confirmedTxs float64 + bestBucketsStt := startIdx + bestBucketsEnd := startIdx + curBucketsEnd := startIdx + + for b := startIdx; b >= 0; b-- { + totalTxs += stats.buckets[b].confirmCount + confirmedTxs += stats.buckets[b].confirmed[confirmRangeIdx].txCount + + // Add the mempool (unconfirmed) transactions to the total tx count + // since a very large mempool for the given bucket might mean that + // miners are reluctant to include these in their mined blocks. + totalTxs += stats.memPool[b].confirmed[confirmRangeIdx].txCount + + if totalTxs > minTxCount { + if confirmedTxs/totalTxs < successPct { + if curBucketsEnd == startIdx { + return 0, ErrNoSuccessPctBucketFound + } + break + } + + bestBucketsStt = b + bestBucketsEnd = curBucketsEnd + curBucketsEnd = b - 1 + totalTxs = 0 + confirmedTxs = 0 + } + } + + txCount := float64(0) + for b := bestBucketsStt; b <= bestBucketsEnd; b++ { + txCount += stats.buckets[b].confirmCount + } + if txCount <= 0 { + return 0, ErrNotEnoughTxsForEstimate + } + txCount /= 2 + for b := bestBucketsStt; b <= bestBucketsEnd; b++ { + if stats.buckets[b].confirmCount < txCount { + txCount -= stats.buckets[b].confirmCount + } else { + median := stats.buckets[b].feeSum / stats.buckets[b].confirmCount + return feeRate(median), nil + } + } + + return 0, errors.New("this isn't supposed to be reached") +} + +// EstimateFee is the public version of estimateMedianFee. It calculates the +// suggested fee for a transaction to be confirmed in at most `targetConf` +// blocks after publishing with a high degree of certainty. +// +// This function is safe to be called from multiple goroutines but might block +// until concurrent modifications to the internal database state are complete. +func (stats *Estimator) EstimateFee(targetConfs int32) (dcrutil.Amount, error) { + stats.lock.RLock() + rate, err := stats.estimateMedianFee(targetConfs, 0.95) + stats.lock.RUnlock() + + if err != nil { + return 0, err + } + + rate = feeRate(math.Round(float64(rate))) + if rate < stats.bucketFeeBounds[0] { + // Prevent our public facing api to ever return something lower than the + // minimum fee + rate = stats.bucketFeeBounds[0] + } + + return dcrutil.Amount(rate), nil +} + +// Enable establishes the current best height of the blockchain after +// initializing the chain. All new mempool transactions will be added at this +// block height. +func (stats *Estimator) Enable(bestHeight int64) { + log.Debugf("Setting best height as %d", bestHeight) + stats.lock.Lock() + stats.bestHeight = bestHeight + stats.lock.Unlock() +} + +// IsEnabled returns whether the fee estimator is ready to accept new mined and +// mempool transactions. +func (stats *Estimator) IsEnabled() bool { + stats.lock.RLock() + enabled := stats.bestHeight > -1 + stats.lock.RUnlock() + return enabled +} + +// AddMemPoolTransaction adds a mempool transaction to the estimator in order to +// account for it in the estimations. It assumes that this transaction is +// entering the mempool at the currently recorded best chain hash, using the +// total fee amount (in atoms) and with the provided size (in bytes). +// +// This is safe to be called from multiple goroutines. +func (stats *Estimator) AddMemPoolTransaction(txHash *chainhash.Hash, fee, size int64, txType stake.TxType) { + stats.lock.Lock() + defer stats.lock.Unlock() + + if stats.bestHeight < 0 { + return + } + + if _, exists := stats.memPoolTxs[*txHash]; exists { + // we should not double count transactions + return + } + + // Ignore tspends for the purposes of fee estimation, since they remain + // in the mempool for a long time and have special rules about when + // they can be included in blocks. + if txType == stake.TxTypeTSpend { + return + } + + // Note that we use this less exact version instead of fee * 1000 / size + // (using ints) because it naturally "downsamples" the fee rates towards the + // minimum at values less than 0.001 DCR/KB. This is needed because due to + // how the wallet estimates the final fee given an input rate and the final + // tx size, there's usually a small discrepancy towards a higher effective + // rate in the published tx. + rate := feeRate(fee / size * 1000) + + if rate < stats.bucketFeeBounds[0] { + // Transactions paying less than the current relaying fee can only + // possibly be included in the high priority/zero fee area of blocks, + // which are usually of limited size, so we explicitly don't track + // those. + // This also naturally handles votes (SSGen transactions) which don't + // carry a tx fee and are required for inclusion in blocks. Note that + // the test is explicitly < instead of <= so that we *can* track + // transactions that pay *exactly* the minimum fee. + return + } + + log.Debugf("Adding mempool tx %s using fee rate %.8f", txHash, rate/1e8) + + tx := memPoolTxDesc{ + addedHeight: stats.bestHeight, + bucketIndex: stats.lowerBucket(rate), + fees: rate, + } + stats.memPoolTxs[*txHash] = tx + stats.newMemPoolTx(tx.bucketIndex, rate) +} + +// RemoveMemPoolTransaction removes a mempool transaction from statistics +// tracking. +// +// This is safe to be called from multiple goroutines. +func (stats *Estimator) RemoveMemPoolTransaction(txHash *chainhash.Hash) { + stats.lock.Lock() + defer stats.lock.Unlock() + + desc, exists := stats.memPoolTxs[*txHash] + if !exists { + return + } + + log.Debugf("Removing tx %s from mempool", txHash) + + stats.removeFromMemPool(int32(stats.bestHeight-desc.addedHeight), desc.fees) + delete(stats.memPoolTxs, *txHash) +} + +// processMinedTransaction moves the transaction that exist in the currently +// tracked mempool into a mined state. +// +// This function is *not* safe to be called from multiple goroutines. +func (stats *Estimator) processMinedTransaction(blockHeight int64, txh *chainhash.Hash) { + desc, exists := stats.memPoolTxs[*txh] + if !exists { + // We cannot use transactions that we didn't know about to estimate + // because that opens up the possibility of miners introducing dummy, + // high fee transactions which would tend to then increase the average + // fee estimate. + // Tracking only previously known transactions forces miners trying to + // pull off this attack to broadcast their transactions and possibly + // forfeit their coins by having the transaction mined by a competitor. + log.Tracef("Processing previously unknown mined tx %s", txh) + return + } + + stats.removeFromMemPool(int32(blockHeight-desc.addedHeight), desc.fees) + delete(stats.memPoolTxs, *txh) + + if blockHeight <= desc.addedHeight { + // This shouldn't usually happen but we need to explicitly test for + // because we can't account for non positive confirmation ranges in + // mined transactions. + log.Errorf("Mined transaction %s (%d) that was known from "+ + "mempool at a higher block height (%d)", txh, blockHeight, + desc.addedHeight) + return + } + + mineDelay := int32(blockHeight - desc.addedHeight) + log.Debugf("Processing mined tx %s (rate %.8f, delay %d)", txh, + desc.fees/1e8, mineDelay) + stats.newMinedTx(mineDelay, desc.fees) +} + +// ProcessBlock processes all mined transactions in the provided block. +// +// This function is safe to be called from multiple goroutines. +func (stats *Estimator) ProcessBlock(block *dcrutil.Block) error { + stats.lock.Lock() + defer stats.lock.Unlock() + + if stats.bestHeight < 0 { + return nil + } + + blockHeight := block.Height() + if blockHeight <= stats.bestHeight { + // we don't explicitly track reorgs right now + log.Warnf("Trying to process mined transactions at block %d when "+ + "previous best block was at height %d", blockHeight, + stats.bestHeight) + return nil + } + + stats.updateMovingAverages(blockHeight) + + for _, tx := range block.Transactions() { + stats.processMinedTransaction(blockHeight, tx.Hash()) + } + + for _, tx := range block.STransactions() { + stats.processMinedTransaction(blockHeight, tx.Hash()) + } + + if stats.db != nil { + return stats.updateDatabase() + } + + return nil +} + +// Close closes the database (if it is currently opened). +func (stats *Estimator) Close() { + stats.lock.Lock() + + if stats.db != nil { + log.Trace("Closing fee estimator database") + stats.db.Close() + stats.db = nil + } + + stats.lock.Unlock() +} diff --git a/fees/log.go b/fees/log.go new file mode 100644 index 00000000..08793d69 --- /dev/null +++ b/fees/log.go @@ -0,0 +1,21 @@ +// Copyright (c) 2018-2019 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package fees + +import ( + "github.com/decred/slog" +) + +// log is a logger that is initialized with no output filters. This means the +// package will not perform any logging by default until the caller requests it. +// The default amount of logging is none. +var log = slog.Disabled + +// UseLogger uses a specified Logger to output fee estimator logging info. This +// should be used in preference to SetLogWriter if the caller is also using +// slog. +func UseLogger(logger slog.Logger) { + log = logger +}