diff --git a/claimtrie/block/blockrepo/pebble.go b/claimtrie/block/blockrepo/pebble.go new file mode 100644 index 00000000..8bf0b1d2 --- /dev/null +++ b/claimtrie/block/blockrepo/pebble.go @@ -0,0 +1,77 @@ +package blockrepo + +import ( + "encoding/binary" + + "github.com/pkg/errors" + + "github.com/lbryio/lbcd/chaincfg/chainhash" + + "github.com/cockroachdb/pebble" +) + +type Pebble struct { + db *pebble.DB +} + +func NewPebble(path string) (*Pebble, error) { + + db, err := pebble.Open(path, &pebble.Options{MaxOpenFiles: 2000}) + repo := &Pebble{db: db} + + return repo, errors.Wrapf(err, "unable to open %s", path) +} + +func (repo *Pebble) Load() (int32, error) { + + iter := repo.db.NewIter(nil) + if !iter.Last() { + err := iter.Close() + return 0, errors.Wrap(err, "closing iterator with no last") + } + + height := int32(binary.BigEndian.Uint32(iter.Key())) + err := iter.Close() + return height, errors.Wrap(err, "closing iterator") +} + +func (repo *Pebble) Get(height int32) (*chainhash.Hash, error) { + + key := make([]byte, 4) + binary.BigEndian.PutUint32(key, uint32(height)) + + b, closer, err := repo.db.Get(key) + if closer != nil { + defer closer.Close() + } + if err != nil { + return nil, errors.Wrap(err, "in get") + } + hash, err := chainhash.NewHash(b) + return hash, errors.Wrap(err, "creating hash") +} + +func (repo *Pebble) Set(height int32, hash *chainhash.Hash) error { + + key := make([]byte, 4) + binary.BigEndian.PutUint32(key, uint32(height)) + + return errors.WithStack(repo.db.Set(key, hash[:], pebble.NoSync)) +} + +func (repo *Pebble) Close() error { + + err := repo.db.Flush() + if err != nil { + // if we fail to close are we going to try again later? + return errors.Wrap(err, "on flush") + } + + err = repo.db.Close() + return errors.Wrap(err, "on close") +} + +func (repo *Pebble) Flush() error { + _, err := repo.db.AsyncFlush() + return err +} diff --git a/claimtrie/block/repo.go b/claimtrie/block/repo.go new file mode 100644 index 00000000..eaa0b7d1 --- /dev/null +++ b/claimtrie/block/repo.go @@ -0,0 +1,14 @@ +package block + +import ( + "github.com/lbryio/lbcd/chaincfg/chainhash" +) + +// Repo defines APIs for Block to access persistence layer. +type Repo interface { + Load() (int32, error) + Set(height int32, hash *chainhash.Hash) error + Get(height int32) (*chainhash.Hash, error) + Close() error + Flush() error +} diff --git a/claimtrie/chain/chainrepo/pebble.go b/claimtrie/chain/chainrepo/pebble.go new file mode 100644 index 00000000..4100d6ac --- /dev/null +++ b/claimtrie/chain/chainrepo/pebble.go @@ -0,0 +1,77 @@ +package chainrepo + +import ( + "encoding/binary" + + "github.com/pkg/errors" + + "github.com/lbryio/lbcd/claimtrie/change" + "github.com/vmihailenco/msgpack/v5" + + "github.com/cockroachdb/pebble" +) + +type Pebble struct { + db *pebble.DB +} + +func NewPebble(path string) (*Pebble, error) { + + db, err := pebble.Open(path, &pebble.Options{BytesPerSync: 64 << 20, MaxOpenFiles: 2000}) + repo := &Pebble{db: db} + + return repo, errors.Wrapf(err, "open %s", path) +} + +func (repo *Pebble) Save(height int32, changes []change.Change) error { + + if len(changes) == 0 { + return nil + } + + var key [4]byte + binary.BigEndian.PutUint32(key[:], uint32(height)) + + value, err := msgpack.Marshal(changes) + if err != nil { + return errors.Wrap(err, "in marshaller") + } + + err = repo.db.Set(key[:], value, pebble.NoSync) + return errors.Wrap(err, "in set") +} + +func (repo *Pebble) Load(height int32) ([]change.Change, error) { + + var key [4]byte + binary.BigEndian.PutUint32(key[:], uint32(height)) + + b, closer, err := repo.db.Get(key[:]) + if closer != nil { + defer closer.Close() + } + if err != nil { + return nil, errors.Wrap(err, "in get") + } + + var changes []change.Change + err = msgpack.Unmarshal(b, &changes) + return changes, errors.Wrap(err, "in unmarshaller") +} + +func (repo *Pebble) Close() error { + + err := repo.db.Flush() + if err != nil { + // if we fail to close are we going to try again later? + return errors.Wrap(err, "on flush") + } + + err = repo.db.Close() + return errors.Wrap(err, "on close") +} + +func (repo *Pebble) Flush() error { + _, err := repo.db.AsyncFlush() + return err +} diff --git a/claimtrie/chain/repo.go b/claimtrie/chain/repo.go new file mode 100644 index 00000000..7d3aa978 --- /dev/null +++ b/claimtrie/chain/repo.go @@ -0,0 +1,10 @@ +package chain + +import "github.com/lbryio/lbcd/claimtrie/change" + +type Repo interface { + Save(height int32, changes []change.Change) error + Load(height int32) ([]change.Change, error) + Close() error + Flush() error +} diff --git a/claimtrie/change/change.go b/claimtrie/change/change.go new file mode 100644 index 00000000..aac349c6 --- /dev/null +++ b/claimtrie/change/change.go @@ -0,0 +1,112 @@ +package change + +import ( + "bytes" + "encoding/binary" + + "github.com/lbryio/lbcd/chaincfg/chainhash" + "github.com/lbryio/lbcd/wire" +) + +type ChangeType uint32 + +const ( + AddClaim ChangeType = iota + SpendClaim + UpdateClaim + AddSupport + SpendSupport +) + +type Change struct { + Type ChangeType + Height int32 + + Name []byte + ClaimID ClaimID + OutPoint wire.OutPoint + Amount int64 + + ActiveHeight int32 + VisibleHeight int32 // aka, CreatedAt; used for normalization fork + + SpentChildren map[string]bool +} + +func NewChange(typ ChangeType) Change { + return Change{Type: typ} +} + +func (c Change) SetHeight(height int32) Change { + c.Height = height + return c +} + +func (c Change) SetName(name []byte) Change { + c.Name = name // need to clone it? + return c +} + +func (c Change) SetOutPoint(op *wire.OutPoint) Change { + c.OutPoint = *op + return c +} + +func (c Change) SetAmount(amt int64) Change { + c.Amount = amt + return c +} + +func (c *Change) Marshal(enc *bytes.Buffer) error { + enc.Write(c.ClaimID[:]) + enc.Write(c.OutPoint.Hash[:]) + var temp [8]byte + binary.BigEndian.PutUint32(temp[:4], c.OutPoint.Index) + enc.Write(temp[:4]) + binary.BigEndian.PutUint32(temp[:4], uint32(c.Type)) + enc.Write(temp[:4]) + binary.BigEndian.PutUint32(temp[:4], uint32(c.Height)) + enc.Write(temp[:4]) + binary.BigEndian.PutUint32(temp[:4], uint32(c.ActiveHeight)) + enc.Write(temp[:4]) + binary.BigEndian.PutUint32(temp[:4], uint32(c.VisibleHeight)) + enc.Write(temp[:4]) + binary.BigEndian.PutUint64(temp[:], uint64(c.Amount)) + enc.Write(temp[:]) + + if c.SpentChildren != nil { + binary.BigEndian.PutUint32(temp[:4], uint32(len(c.SpentChildren))) + enc.Write(temp[:4]) + for key := range c.SpentChildren { + binary.BigEndian.PutUint16(temp[:2], uint16(len(key))) // technically limited to 255; not sure we trust it + enc.Write(temp[:2]) + enc.WriteString(key) + } + } else { + binary.BigEndian.PutUint32(temp[:4], 0) + enc.Write(temp[:4]) + } + return nil +} + +func (c *Change) Unmarshal(dec *bytes.Buffer) error { + copy(c.ClaimID[:], dec.Next(ClaimIDSize)) + copy(c.OutPoint.Hash[:], dec.Next(chainhash.HashSize)) + c.OutPoint.Index = binary.BigEndian.Uint32(dec.Next(4)) + c.Type = ChangeType(binary.BigEndian.Uint32(dec.Next(4))) + c.Height = int32(binary.BigEndian.Uint32(dec.Next(4))) + c.ActiveHeight = int32(binary.BigEndian.Uint32(dec.Next(4))) + c.VisibleHeight = int32(binary.BigEndian.Uint32(dec.Next(4))) + c.Amount = int64(binary.BigEndian.Uint64(dec.Next(8))) + keys := binary.BigEndian.Uint32(dec.Next(4)) + if keys > 0 { + c.SpentChildren = map[string]bool{} + } + for keys > 0 { + keys-- + keySize := int(binary.BigEndian.Uint16(dec.Next(2))) + key := string(dec.Next(keySize)) + c.SpentChildren[key] = true + } + return nil +} diff --git a/claimtrie/change/claimid.go b/claimtrie/change/claimid.go new file mode 100644 index 00000000..e7a92565 --- /dev/null +++ b/claimtrie/change/claimid.go @@ -0,0 +1,54 @@ +package change + +import ( + "encoding/binary" + "encoding/hex" + + "github.com/lbryio/lbcd/chaincfg/chainhash" + "github.com/lbryio/lbcd/wire" + btcutil "github.com/lbryio/lbcutil" +) + +// ClaimID represents a Claim's ClaimID. +const ClaimIDSize = 20 + +type ClaimID [ClaimIDSize]byte + +// NewClaimID returns a Claim ID calculated from Ripemd160(Sha256(OUTPOINT). +func NewClaimID(op wire.OutPoint) (id ClaimID) { + + var buffer [chainhash.HashSize + 4]byte // hoping for stack alloc + copy(buffer[:], op.Hash[:]) + binary.BigEndian.PutUint32(buffer[chainhash.HashSize:], op.Index) + copy(id[:], btcutil.Hash160(buffer[:])) + return id +} + +// NewIDFromString returns a Claim ID from a string. +func NewIDFromString(s string) (id ClaimID, err error) { + + if len(s) == 40 { + _, err = hex.Decode(id[:], []byte(s)) + } else { + copy(id[:], s) + } + for i, j := 0, len(id)-1; i < j; i, j = i+1, j-1 { + id[i], id[j] = id[j], id[i] + } + return id, err +} + +// Key is for in-memory maps +func (id ClaimID) Key() string { + return string(id[:]) +} + +// String is for anything written to a DB +func (id ClaimID) String() string { + + for i, j := 0, len(id)-1; i < j; i, j = i+1, j-1 { + id[i], id[j] = id[j], id[i] + } + + return hex.EncodeToString(id[:]) +} diff --git a/claimtrie/claimtrie.go b/claimtrie/claimtrie.go new file mode 100644 index 00000000..f99a147e --- /dev/null +++ b/claimtrie/claimtrie.go @@ -0,0 +1,483 @@ +package claimtrie + +import ( + "bytes" + "fmt" + "path/filepath" + "runtime" + "sort" + "sync" + + "github.com/pkg/errors" + + "github.com/lbryio/lbcd/claimtrie/block" + "github.com/lbryio/lbcd/claimtrie/block/blockrepo" + "github.com/lbryio/lbcd/claimtrie/change" + "github.com/lbryio/lbcd/claimtrie/config" + "github.com/lbryio/lbcd/claimtrie/merkletrie" + "github.com/lbryio/lbcd/claimtrie/merkletrie/merkletrierepo" + "github.com/lbryio/lbcd/claimtrie/node" + "github.com/lbryio/lbcd/claimtrie/node/noderepo" + "github.com/lbryio/lbcd/claimtrie/normalization" + "github.com/lbryio/lbcd/claimtrie/param" + "github.com/lbryio/lbcd/claimtrie/temporal" + "github.com/lbryio/lbcd/claimtrie/temporal/temporalrepo" + + "github.com/lbryio/lbcd/chaincfg/chainhash" + "github.com/lbryio/lbcd/wire" +) + +// ClaimTrie implements a Merkle Trie supporting linear history of commits. +type ClaimTrie struct { + + // Repository for calculated block hashes. + blockRepo block.Repo + + // Repository for storing temporal information of nodes at each block height. + // For example, which nodes (by name) should be refreshed at each block height + // due to stake expiration or delayed activation. + temporalRepo temporal.Repo + + // Cache layer of Nodes. + nodeManager node.Manager + + // Prefix tree (trie) that manages merkle hash of each node. + merkleTrie merkletrie.MerkleTrie + + // Current block height, which is increased by one when AppendBlock() is called. + height int32 + + // Registrered cleanup functions which are invoked in the Close() in reverse order. + cleanups []func() error +} + +func New(cfg config.Config) (*ClaimTrie, error) { + + var cleanups []func() error + + // The passed in cfg.DataDir has been prepended with netname. + dataDir := filepath.Join(cfg.DataDir, "claim_dbs") + + dbPath := filepath.Join(dataDir, cfg.BlockRepoPebble.Path) + blockRepo, err := blockrepo.NewPebble(dbPath) + if err != nil { + return nil, errors.Wrap(err, "creating block repo") + } + cleanups = append(cleanups, blockRepo.Close) + err = blockRepo.Set(0, merkletrie.EmptyTrieHash) + if err != nil { + return nil, errors.Wrap(err, "setting block repo genesis") + } + + dbPath = filepath.Join(dataDir, cfg.TemporalRepoPebble.Path) + temporalRepo, err := temporalrepo.NewPebble(dbPath) + if err != nil { + return nil, errors.Wrap(err, "creating temporal repo") + } + cleanups = append(cleanups, temporalRepo.Close) + + // Initialize repository for changes to nodes. + // The cleanup is delegated to the Node Manager. + dbPath = filepath.Join(dataDir, cfg.NodeRepoPebble.Path) + nodeRepo, err := noderepo.NewPebble(dbPath) + if err != nil { + return nil, errors.Wrap(err, "creating node repo") + } + + baseManager, err := node.NewBaseManager(nodeRepo) + if err != nil { + return nil, errors.Wrap(err, "creating node base manager") + } + normalizingManager := node.NewNormalizingManager(baseManager) + nodeManager := &node.HashV2Manager{Manager: normalizingManager} + cleanups = append(cleanups, nodeManager.Close) + + var trie merkletrie.MerkleTrie + if cfg.RamTrie { + trie = merkletrie.NewRamTrie() + } else { + + // Initialize repository for MerkleTrie. The cleanup is delegated to MerkleTrie. + dbPath = filepath.Join(dataDir, cfg.MerkleTrieRepoPebble.Path) + trieRepo, err := merkletrierepo.NewPebble(dbPath) + if err != nil { + return nil, errors.Wrap(err, "creating trie repo") + } + + persistentTrie := merkletrie.NewPersistentTrie(trieRepo) + cleanups = append(cleanups, persistentTrie.Close) + trie = persistentTrie + } + + // Restore the last height. + previousHeight, err := blockRepo.Load() + if err != nil { + return nil, errors.Wrap(err, "load block tip") + } + + ct := &ClaimTrie{ + blockRepo: blockRepo, + temporalRepo: temporalRepo, + + nodeManager: nodeManager, + merkleTrie: trie, + + height: previousHeight, + } + + ct.cleanups = cleanups + + if previousHeight > 0 { + hash, err := blockRepo.Get(previousHeight) + if err != nil { + ct.Close() // TODO: the cleanups aren't run when we exit with an err above here (but should be) + return nil, errors.Wrap(err, "block repo get") + } + _, err = nodeManager.IncrementHeightTo(previousHeight) + if err != nil { + ct.Close() + return nil, errors.Wrap(err, "increment height to") + } + err = trie.SetRoot(hash) // keep this after IncrementHeightTo + if err == merkletrie.ErrFullRebuildRequired { + ct.runFullTrieRebuild(nil, cfg.Interrupt) + } + + if interruptRequested(cfg.Interrupt) || !ct.MerkleHash().IsEqual(hash) { + ct.Close() + return nil, errors.Errorf("unable to restore the claim hash to %s at height %d", hash.String(), previousHeight) + } + } + + return ct, nil +} + +// AddClaim adds a Claim to the ClaimTrie. +func (ct *ClaimTrie) AddClaim(name []byte, op wire.OutPoint, id change.ClaimID, amt int64) error { + + chg := change.Change{ + Type: change.AddClaim, + Name: name, + OutPoint: op, + Amount: amt, + ClaimID: id, + } + + return ct.forwardNodeChange(chg) +} + +// UpdateClaim updates a Claim in the ClaimTrie. +func (ct *ClaimTrie) UpdateClaim(name []byte, op wire.OutPoint, amt int64, id change.ClaimID) error { + + chg := change.Change{ + Type: change.UpdateClaim, + Name: name, + OutPoint: op, + Amount: amt, + ClaimID: id, + } + + return ct.forwardNodeChange(chg) +} + +// SpendClaim spends a Claim in the ClaimTrie. +func (ct *ClaimTrie) SpendClaim(name []byte, op wire.OutPoint, id change.ClaimID) error { + + chg := change.Change{ + Type: change.SpendClaim, + Name: name, + OutPoint: op, + ClaimID: id, + } + + return ct.forwardNodeChange(chg) +} + +// AddSupport adds a Support to the ClaimTrie. +func (ct *ClaimTrie) AddSupport(name []byte, op wire.OutPoint, amt int64, id change.ClaimID) error { + + chg := change.Change{ + Type: change.AddSupport, + Name: name, + OutPoint: op, + Amount: amt, + ClaimID: id, + } + + return ct.forwardNodeChange(chg) +} + +// SpendSupport spends a Support in the ClaimTrie. +func (ct *ClaimTrie) SpendSupport(name []byte, op wire.OutPoint, id change.ClaimID) error { + + chg := change.Change{ + Type: change.SpendSupport, + Name: name, + OutPoint: op, + ClaimID: id, + } + + return ct.forwardNodeChange(chg) +} + +// AppendBlock increases block by one. +func (ct *ClaimTrie) AppendBlock() error { + + ct.height++ + + names, err := ct.nodeManager.IncrementHeightTo(ct.height) + if err != nil { + return errors.Wrap(err, "node manager increment") + } + + expirations, err := ct.temporalRepo.NodesAt(ct.height) + if err != nil { + return errors.Wrap(err, "temporal repo get") + } + + names = removeDuplicates(names) // comes out sorted + + updateNames := make([][]byte, 0, len(names)+len(expirations)) + updateHeights := make([]int32, 0, len(names)+len(expirations)) + updateNames = append(updateNames, names...) + for range names { // log to the db that we updated a name at this height for rollback purposes + updateHeights = append(updateHeights, ct.height) + } + names = append(names, expirations...) + names = removeDuplicates(names) + + nhns := ct.makeNameHashNext(names, false, nil) + for nhn := range nhns { + + ct.merkleTrie.Update(nhn.Name, nhn.Hash, true) + if nhn.Next <= 0 { + continue + } + + newName := normalization.NormalizeIfNecessary(nhn.Name, nhn.Next) + updateNames = append(updateNames, newName) + updateHeights = append(updateHeights, nhn.Next) + } + if len(updateNames) != 0 { + err = ct.temporalRepo.SetNodesAt(updateNames, updateHeights) + if err != nil { + return errors.Wrap(err, "temporal repo set") + } + } + + hitFork := ct.updateTrieForHashForkIfNecessary() + + h := ct.MerkleHash() + ct.blockRepo.Set(ct.height, h) + + if hitFork { + err = ct.merkleTrie.SetRoot(h) // for clearing the memory entirely + } + + return errors.Wrap(err, "merkle trie clear memory") +} + +func (ct *ClaimTrie) updateTrieForHashForkIfNecessary() bool { + if ct.height != param.ActiveParams.AllClaimsInMerkleForkHeight { + return false + } + + node.LogOnce(fmt.Sprintf("Rebuilding all trie nodes for the hash fork at %d...", ct.height)) + ct.runFullTrieRebuild(nil, nil) // I don't think it's safe to allow interrupt during fork + return true +} + +func removeDuplicates(names [][]byte) [][]byte { // this might be too expensive; we'll have to profile it + sort.Slice(names, func(i, j int) bool { // put names in order so we can skip duplicates + return bytes.Compare(names[i], names[j]) < 0 + }) + + for i := len(names) - 2; i >= 0; i-- { + if bytes.Equal(names[i], names[i+1]) { + names = append(names[:i], names[i+1:]...) + } + } + return names +} + +// ResetHeight resets the ClaimTrie to a previous known height.. +func (ct *ClaimTrie) ResetHeight(height int32) error { + + names := make([][]byte, 0) + for h := height + 1; h <= ct.height; h++ { + results, err := ct.temporalRepo.NodesAt(h) + if err != nil { + return err + } + names = append(names, results...) + } + err := ct.nodeManager.DecrementHeightTo(names, height) + if err != nil { + return err + } + + passedHashFork := ct.height >= param.ActiveParams.AllClaimsInMerkleForkHeight && height < param.ActiveParams.AllClaimsInMerkleForkHeight + hash, err := ct.blockRepo.Get(height) + if err != nil { + return err + } + + ct.height = height // keep this before the rebuild + + if passedHashFork { + names = nil // force them to reconsider all names + } + err = ct.merkleTrie.SetRoot(hash) + if err == merkletrie.ErrFullRebuildRequired { + ct.runFullTrieRebuild(names, nil) + } + + if !ct.MerkleHash().IsEqual(hash) { + return errors.Errorf("unable to restore the hash at height %d", height) + } + return nil +} + +func (ct *ClaimTrie) runFullTrieRebuild(names [][]byte, interrupt <-chan struct{}) { + var nhns chan NameHashNext + if names == nil { + node.LogOnce("Building the entire claim trie in RAM...") + + nhns = ct.makeNameHashNext(nil, true, interrupt) + } else { + nhns = ct.makeNameHashNext(names, false, interrupt) + } + + for nhn := range nhns { + ct.merkleTrie.Update(nhn.Name, nhn.Hash, false) + } +} + +// MerkleHash returns the Merkle Hash of the claimTrie. +func (ct *ClaimTrie) MerkleHash() *chainhash.Hash { + if ct.height >= param.ActiveParams.AllClaimsInMerkleForkHeight { + return ct.merkleTrie.MerkleHashAllClaims() + } + return ct.merkleTrie.MerkleHash() +} + +// Height returns the current block height. +func (ct *ClaimTrie) Height() int32 { + return ct.height +} + +// Close persists states. +// Any calls to the ClaimTrie after Close() being called results undefined behaviour. +func (ct *ClaimTrie) Close() { + + for i := len(ct.cleanups) - 1; i >= 0; i-- { + cleanup := ct.cleanups[i] + err := cleanup() + if err != nil { // it would be better to cleanup what we can than exit early + node.LogOnce("On cleanup: " + err.Error()) + } + } + ct.cleanups = nil +} + +func (ct *ClaimTrie) forwardNodeChange(chg change.Change) error { + + chg.Height = ct.Height() + 1 + ct.nodeManager.AppendChange(chg) + return nil +} + +func (ct *ClaimTrie) NodeAt(height int32, name []byte) (*node.Node, error) { + return ct.nodeManager.NodeAt(height, name) +} + +func (ct *ClaimTrie) NamesChangedInBlock(height int32) ([]string, error) { + hits, err := ct.temporalRepo.NodesAt(height) + r := make([]string, len(hits)) + for i := range hits { + r[i] = string(hits[i]) + } + return r, err +} + +func (ct *ClaimTrie) FlushToDisk() { + // maybe the user can fix the file lock shown in the warning before they shut down + if err := ct.nodeManager.Flush(); err != nil { + node.Warn("During nodeManager flush: " + err.Error()) + } + if err := ct.temporalRepo.Flush(); err != nil { + node.Warn("During temporalRepo flush: " + err.Error()) + } + if err := ct.merkleTrie.Flush(); err != nil { + node.Warn("During merkleTrie flush: " + err.Error()) + } + if err := ct.blockRepo.Flush(); err != nil { + node.Warn("During blockRepo flush: " + err.Error()) + } +} + +type NameHashNext struct { + Name []byte + Hash *chainhash.Hash + Next int32 +} + +func interruptRequested(interrupted <-chan struct{}) bool { + select { + case <-interrupted: // should never block on nil + return true + default: + } + + return false +} + +func (ct *ClaimTrie) makeNameHashNext(names [][]byte, all bool, interrupt <-chan struct{}) chan NameHashNext { + inputs := make(chan []byte, 512) + outputs := make(chan NameHashNext, 512) + + var wg sync.WaitGroup + hashComputationWorker := func() { + for name := range inputs { + hash, next := ct.nodeManager.Hash(name) + outputs <- NameHashNext{name, hash, next} + } + wg.Done() + } + + threads := int(0.8 * float32(runtime.NumCPU())) + if threads < 1 { + threads = 1 + } + for threads > 0 { + threads-- + wg.Add(1) + go hashComputationWorker() + } + go func() { + if all { + ct.nodeManager.IterateNames(func(name []byte) bool { + if interruptRequested(interrupt) { + return false + } + clone := make([]byte, len(name)) + copy(clone, name) // iteration name buffer is reused on future loops + inputs <- clone + return true + }) + } else { + for _, name := range names { + if interruptRequested(interrupt) { + break + } + inputs <- name + } + } + close(inputs) + }() + go func() { + wg.Wait() + close(outputs) + }() + return outputs +} diff --git a/claimtrie/claimtrie_test.go b/claimtrie/claimtrie_test.go new file mode 100644 index 00000000..7cd1432b --- /dev/null +++ b/claimtrie/claimtrie_test.go @@ -0,0 +1,1027 @@ +package claimtrie + +import ( + "math/rand" + "testing" + "time" + + "github.com/lbryio/lbcd/claimtrie/change" + "github.com/lbryio/lbcd/claimtrie/config" + "github.com/lbryio/lbcd/claimtrie/merkletrie" + "github.com/lbryio/lbcd/claimtrie/param" + + "github.com/lbryio/lbcd/chaincfg/chainhash" + "github.com/lbryio/lbcd/wire" + + "github.com/stretchr/testify/require" +) + +var cfg = config.DefaultConfig + +func setup(t *testing.T) { + param.SetNetwork(wire.TestNet) + cfg.DataDir = t.TempDir() +} + +func b(s string) []byte { + return []byte(s) +} + +func buildTx(hash chainhash.Hash) *wire.MsgTx { + tx := wire.NewMsgTx(1) + txIn := wire.NewTxIn(wire.NewOutPoint(&hash, 0), nil, nil) + tx.AddTxIn(txIn) + tx.AddTxOut(wire.NewTxOut(0, nil)) + return tx +} + +func TestFixedHashes(t *testing.T) { + + r := require.New(t) + + setup(t) + ct, err := New(cfg) + r.NoError(err) + defer ct.Close() + + r.Equal(merkletrie.EmptyTrieHash[:], ct.MerkleHash()[:]) + + tx1 := buildTx(*merkletrie.EmptyTrieHash) + tx2 := buildTx(tx1.TxHash()) + tx3 := buildTx(tx2.TxHash()) + tx4 := buildTx(tx3.TxHash()) + + err = ct.AddClaim(b("test"), tx1.TxIn[0].PreviousOutPoint, change.NewClaimID(tx1.TxIn[0].PreviousOutPoint), 50) + r.NoError(err) + + err = ct.AddClaim(b("test2"), tx2.TxIn[0].PreviousOutPoint, change.NewClaimID(tx2.TxIn[0].PreviousOutPoint), 50) + r.NoError(err) + + err = ct.AddClaim(b("test"), tx3.TxIn[0].PreviousOutPoint, change.NewClaimID(tx3.TxIn[0].PreviousOutPoint), 50) + r.NoError(err) + + err = ct.AddClaim(b("tes"), tx4.TxIn[0].PreviousOutPoint, change.NewClaimID(tx4.TxIn[0].PreviousOutPoint), 50) + r.NoError(err) + + incrementBlock(r, ct, 1) + + expected, err := chainhash.NewHashFromStr("938fb93364bf8184e0b649c799ae27274e8db5221f1723c99fb2acd3386cfb00") + r.NoError(err) + r.Equal(expected[:], ct.MerkleHash()[:]) +} + +func TestEmptyHashFork(t *testing.T) { + r := require.New(t) + + setup(t) + param.ActiveParams.AllClaimsInMerkleForkHeight = 2 + ct, err := New(cfg) + r.NoError(err) + r.NotNil(ct) + defer ct.Close() + + for i := 0; i < 5; i++ { + err := ct.AppendBlock() + r.NoError(err) + } +} + +func TestNormalizationFork(t *testing.T) { + r := require.New(t) + + setup(t) + param.ActiveParams.NormalizedNameForkHeight = 2 + ct, err := New(cfg) + r.NoError(err) + r.NotNil(ct) + defer ct.Close() + + hash := chainhash.HashH([]byte{1, 2, 3}) + + o1 := wire.OutPoint{Hash: hash, Index: 1} + err = ct.AddClaim([]byte("AÑEJO"), o1, change.NewClaimID(o1), 10) + r.NoError(err) + + o2 := wire.OutPoint{Hash: hash, Index: 2} + err = ct.AddClaim([]byte("AÑejo"), o2, change.NewClaimID(o2), 5) + r.NoError(err) + + o3 := wire.OutPoint{Hash: hash, Index: 3} + err = ct.AddClaim([]byte("あてはまる"), o3, change.NewClaimID(o3), 5) + r.NoError(err) + + o4 := wire.OutPoint{Hash: hash, Index: 4} + err = ct.AddClaim([]byte("Aḿlie"), o4, change.NewClaimID(o4), 5) + r.NoError(err) + + o5 := wire.OutPoint{Hash: hash, Index: 5} + err = ct.AddClaim([]byte("TEST"), o5, change.NewClaimID(o5), 5) + r.NoError(err) + + o6 := wire.OutPoint{Hash: hash, Index: 6} + err = ct.AddClaim([]byte("test"), o6, change.NewClaimID(o6), 7) + r.NoError(err) + + o7 := wire.OutPoint{Hash: hash, Index: 7} + err = ct.AddSupport([]byte("test"), o7, 11, change.NewClaimID(o6)) + r.NoError(err) + + incrementBlock(r, ct, 1) + r.NotEqual(merkletrie.EmptyTrieHash[:], ct.MerkleHash()[:]) + + n, err := ct.nodeManager.NodeAt(ct.nodeManager.Height(), []byte("AÑEJO")) + r.NoError(err) + r.NotNil(n.BestClaim) + r.Equal(int32(1), n.TakenOverAt) + + o8 := wire.OutPoint{Hash: hash, Index: 8} + err = ct.AddClaim([]byte("aÑEJO"), o8, change.NewClaimID(o8), 8) + r.NoError(err) + + incrementBlock(r, ct, 1) + r.NotEqual(merkletrie.EmptyTrieHash[:], ct.MerkleHash()[:]) + + n, err = ct.nodeManager.NodeAt(ct.nodeManager.Height(), []byte("añejo")) + r.NoError(err) + r.Equal(3, len(n.Claims)) + r.Equal(uint32(1), n.BestClaim.OutPoint.Index) + r.Equal(int32(2), n.TakenOverAt) + + n, err = ct.nodeManager.NodeAt(ct.nodeManager.Height(), []byte("test")) + r.NoError(err) + r.Equal(int64(18), n.BestClaim.Amount+n.SupportSums[n.BestClaim.ClaimID.Key()]) +} + +func TestActivationsOnNormalizationFork(t *testing.T) { + + r := require.New(t) + + setup(t) + param.ActiveParams.NormalizedNameForkHeight = 4 + ct, err := New(cfg) + r.NoError(err) + r.NotNil(ct) + defer ct.Close() + + hash := chainhash.HashH([]byte{1, 2, 3}) + + o7 := wire.OutPoint{Hash: hash, Index: 7} + err = ct.AddClaim([]byte("A"), o7, change.NewClaimID(o7), 1) + r.NoError(err) + incrementBlock(r, ct, 3) + verifyBestIndex(t, ct, "A", 7, 1) + + o8 := wire.OutPoint{Hash: hash, Index: 8} + err = ct.AddClaim([]byte("A"), o8, change.NewClaimID(o8), 2) + r.NoError(err) + incrementBlock(r, ct, 1) + verifyBestIndex(t, ct, "a", 8, 2) + + incrementBlock(r, ct, 2) + verifyBestIndex(t, ct, "a", 8, 2) + + err = ct.ResetHeight(3) + r.NoError(err) + verifyBestIndex(t, ct, "A", 7, 1) +} + +func TestNormalizationSortOrder(t *testing.T) { + + r := require.New(t) + // this was an unfortunate bug; the normalization fork should not have activated anything + // alas, it's now part of our history; we hereby test it to keep it that way + setup(t) + param.ActiveParams.NormalizedNameForkHeight = 2 + ct, err := New(cfg) + r.NoError(err) + r.NotNil(ct) + defer ct.Close() + + hash := chainhash.HashH([]byte{1, 2, 3}) + + o1 := wire.OutPoint{Hash: hash, Index: 1} + err = ct.AddClaim([]byte("A"), o1, change.NewClaimID(o1), 1) + r.NoError(err) + + o2 := wire.OutPoint{Hash: hash, Index: 2} + err = ct.AddClaim([]byte("A"), o2, change.NewClaimID(o2), 2) + r.NoError(err) + + o3 := wire.OutPoint{Hash: hash, Index: 3} + err = ct.AddClaim([]byte("a"), o3, change.NewClaimID(o3), 3) + r.NoError(err) + + incrementBlock(r, ct, 1) + verifyBestIndex(t, ct, "A", 2, 2) + verifyBestIndex(t, ct, "a", 3, 1) + + incrementBlock(r, ct, 1) + verifyBestIndex(t, ct, "a", 3, 3) +} + +func verifyBestIndex(t *testing.T, ct *ClaimTrie, name string, idx uint32, claims int) { + + r := require.New(t) + + n, err := ct.nodeManager.NodeAt(ct.nodeManager.Height(), []byte(name)) + r.NoError(err) + r.Equal(claims, len(n.Claims)) + if claims > 0 { + r.Equal(idx, n.BestClaim.OutPoint.Index) + } +} + +func TestRebuild(t *testing.T) { + r := require.New(t) + setup(t) + ct, err := New(cfg) + r.NoError(err) + r.NotNil(ct) + defer ct.Close() + + hash := chainhash.HashH([]byte{1, 2, 3}) + + o1 := wire.OutPoint{Hash: hash, Index: 1} + err = ct.AddClaim([]byte("test1"), o1, change.NewClaimID(o1), 1) + r.NoError(err) + + o2 := wire.OutPoint{Hash: hash, Index: 2} + err = ct.AddClaim([]byte("test2"), o2, change.NewClaimID(o2), 2) + r.NoError(err) + + incrementBlock(r, ct, 1) + + m := ct.MerkleHash() + r.NotNil(m) + r.NotEqual(*merkletrie.EmptyTrieHash, *m) + + ct.merkleTrie = merkletrie.NewRamTrie() + ct.runFullTrieRebuild(nil, nil) + + m2 := ct.MerkleHash() + r.NotNil(m2) + r.Equal(*m, *m2) +} + +func BenchmarkClaimTrie_AppendBlock256(b *testing.B) { + + addUpdateRemoveRandoms(b, 256) +} + +func BenchmarkClaimTrie_AppendBlock4(b *testing.B) { + + addUpdateRemoveRandoms(b, 4) +} + +func addUpdateRemoveRandoms(b *testing.B, inBlock int) { + rand.Seed(42) + names := make([][]byte, 0, b.N) + + for i := 0; i < b.N; i++ { + names = append(names, randomName()) + } + + var hashes []*chainhash.Hash + + param.SetNetwork(wire.TestNet) + param.ActiveParams.OriginalClaimExpirationTime = 1000000 + param.ActiveParams.ExtendedClaimExpirationTime = 1000000 + cfg.DataDir = b.TempDir() + + r := require.New(b) + ct, err := New(cfg) + r.NoError(err) + defer ct.Close() + h1 := chainhash.Hash{100, 200} + + start := time.Now() + b.ResetTimer() + + c := 0 + for i := 0; i < b.N; i++ { + op := wire.OutPoint{Hash: h1, Index: uint32(i)} + id := change.NewClaimID(op) + err = ct.AddClaim(names[i], op, id, 500) + r.NoError(err) + if c++; c%inBlock == inBlock-1 { + incrementBlock(r, ct, 1) + hashes = append(hashes, ct.MerkleHash()) + } + } + + for i := 0; i < b.N; i++ { + op := wire.OutPoint{Hash: h1, Index: uint32(i)} + id := change.NewClaimID(op) + op.Hash[0] = 1 + err = ct.UpdateClaim(names[i], op, 400, id) + r.NoError(err) + if c++; c%inBlock == inBlock-1 { + incrementBlock(r, ct, 1) + hashes = append(hashes, ct.MerkleHash()) + } + } + + for i := 0; i < b.N; i++ { + op := wire.OutPoint{Hash: h1, Index: uint32(i)} + id := change.NewClaimID(op) + op.Hash[0] = 2 + err = ct.UpdateClaim(names[i], op, 300, id) + r.NoError(err) + if c++; c%inBlock == inBlock-1 { + incrementBlock(r, ct, 1) + hashes = append(hashes, ct.MerkleHash()) + } + } + + for i := 0; i < b.N; i++ { + op := wire.OutPoint{Hash: h1, Index: uint32(i)} + id := change.NewClaimID(op) + op.Hash[0] = 3 + err = ct.SpendClaim(names[i], op, id) + r.NoError(err) + if c++; c%inBlock == inBlock-1 { + incrementBlock(r, ct, 1) + hashes = append(hashes, ct.MerkleHash()) + } + } + incrementBlock(r, ct, 1) + hashes = append(hashes, ct.MerkleHash()) + + b.StopTimer() + ht := ct.height + h1 = *ct.MerkleHash() + b.Logf("Running AppendBlock bench with %d names in %f sec. Height: %d, Hash: %s", + b.N, time.Since(start).Seconds(), ht, h1.String()) + + // a very important test of the functionality: + for ct.height > 0 { + r.True(hashes[ct.height-1].IsEqual(ct.MerkleHash())) + err = ct.ResetHeight(ct.height - 1) + r.NoError(err) + } +} + +func randomName() []byte { + name := make([]byte, rand.Intn(30)+10) + rand.Read(name) + for i := range name { + name[i] %= 56 + name[i] += 65 + } + return name +} + +func incrementBlock(r *require.Assertions, ct *ClaimTrie, c int32) { + h := ct.height + c + if c < 0 { + err := ct.ResetHeight(ct.height + c) + r.NoError(err) + } else { + for ; c > 0; c-- { + err := ct.AppendBlock() + r.NoError(err) + } + } + r.Equal(h, ct.height) +} + +func TestNormalizationRollback(t *testing.T) { + param.SetNetwork(wire.TestNet) + param.ActiveParams.OriginalClaimExpirationTime = 1000000 + param.ActiveParams.ExtendedClaimExpirationTime = 1000000 + cfg.DataDir = t.TempDir() + + r := require.New(t) + ct, err := New(cfg) + r.NoError(err) + defer ct.Close() + + r.Equal(int32(250), param.ActiveParams.NormalizedNameForkHeight) + incrementBlock(r, ct, 247) + + h1 := chainhash.Hash{100, 200} + op := wire.OutPoint{Hash: h1, Index: 1} + id := change.NewClaimID(op) + err = ct.AddClaim([]byte("TEST"), op, id, 1000) + r.NoError(err) + + incrementBlock(r, ct, 5) + incrementBlock(r, ct, -4) + err = ct.SpendClaim([]byte("TEST"), op, id) + r.NoError(err) + incrementBlock(r, ct, 1) + h := ct.MerkleHash() + r.True(h.IsEqual(merkletrie.EmptyTrieHash)) + incrementBlock(r, ct, 3) + h2 := ct.MerkleHash() + r.True(h.IsEqual(h2)) +} + +func TestNormalizationRollbackFuzz(t *testing.T) { + rand.Seed(42) + var hashes []*chainhash.Hash + + param.SetNetwork(wire.TestNet) + param.ActiveParams.OriginalClaimExpirationTime = 1000000 + param.ActiveParams.ExtendedClaimExpirationTime = 1000000 + cfg.DataDir = t.TempDir() + + r := require.New(t) + ct, err := New(cfg) + r.NoError(err) + defer ct.Close() + h1 := chainhash.Hash{100, 200} + + r.Equal(int32(250), param.ActiveParams.NormalizedNameForkHeight) + incrementBlock(r, ct, 240) + + for j := 0; j < 10; j++ { + c := 0 + for i := 0; i < 200; i++ { + op := wire.OutPoint{Hash: h1, Index: uint32(i)} + id := change.NewClaimID(op) + err = ct.AddClaim(randomName(), op, id, 500) + r.NoError(err) + if c++; c%10 == 9 { + incrementBlock(r, ct, 1) + hashes = append(hashes, ct.MerkleHash()) + } + } + if j > 7 { + ct.runFullTrieRebuild(nil, nil) + h := ct.MerkleHash() + r.True(h.IsEqual(hashes[len(hashes)-1])) + } + for ct.height > 240 { + r.True(hashes[ct.height-1-240].IsEqual(ct.MerkleHash())) + err = ct.ResetHeight(ct.height - 1) + r.NoError(err) + } + hashes = hashes[:0] + } +} + +func TestClaimReplace(t *testing.T) { + r := require.New(t) + setup(t) + ct, err := New(cfg) + r.NoError(err) + r.NotNil(ct) + defer ct.Close() + + hash := chainhash.HashH([]byte{1, 2, 3}) + o1 := wire.OutPoint{Hash: hash, Index: 1} + err = ct.AddClaim([]byte("bass"), o1, change.NewClaimID(o1), 8) + r.NoError(err) + + o2 := wire.OutPoint{Hash: hash, Index: 2} + err = ct.AddClaim([]byte("basso"), o2, change.NewClaimID(o2), 10) + r.NoError(err) + + incrementBlock(r, ct, 1) + n, err := ct.NodeAt(ct.height, []byte("bass")) + r.Equal(o1.String(), n.BestClaim.OutPoint.String()) + + err = ct.SpendClaim([]byte("bass"), o1, n.BestClaim.ClaimID) + r.NoError(err) + + o4 := wire.OutPoint{Hash: hash, Index: 4} + err = ct.AddClaim([]byte("bassfisher"), o4, change.NewClaimID(o4), 12) + r.NoError(err) + + incrementBlock(r, ct, 1) + n, err = ct.NodeAt(ct.height, []byte("bass")) + r.NoError(err) + r.True(n == nil || !n.HasActiveBestClaim()) + n, err = ct.NodeAt(ct.height, []byte("bassfisher")) + r.Equal(o4.String(), n.BestClaim.OutPoint.String()) +} + +func TestGeneralClaim(t *testing.T) { + r := require.New(t) + setup(t) + ct, err := New(cfg) + r.NoError(err) + r.NotNil(ct) + defer ct.Close() + + incrementBlock(r, ct, 1) + + hash := chainhash.HashH([]byte{1, 2, 3}) + o1 := wire.OutPoint{Hash: hash, Index: 1} + err = ct.AddClaim([]byte("test"), o1, change.NewClaimID(o1), 8) + r.NoError(err) + + incrementBlock(r, ct, 1) + err = ct.ResetHeight(ct.height - 1) + r.NoError(err) + n, err := ct.NodeAt(ct.height, []byte("test")) + r.NoError(err) + r.True(n == nil || !n.HasActiveBestClaim()) + + err = ct.AddClaim([]byte("test"), o1, change.NewClaimID(o1), 8) + o2 := wire.OutPoint{Hash: hash, Index: 2} + err = ct.AddClaim([]byte("test"), o2, change.NewClaimID(o2), 8) + r.NoError(err) + + incrementBlock(r, ct, 1) + incrementBlock(r, ct, -1) + n, err = ct.NodeAt(ct.height, []byte("test")) + r.NoError(err) + r.True(n == nil || !n.HasActiveBestClaim()) + + err = ct.AddClaim([]byte("test"), o1, change.NewClaimID(o1), 8) + r.NoError(err) + incrementBlock(r, ct, 1) + err = ct.AddClaim([]byte("test"), o2, change.NewClaimID(o2), 8) + r.NoError(err) + incrementBlock(r, ct, 1) + + incrementBlock(r, ct, -2) + n, err = ct.NodeAt(ct.height, []byte("test")) + r.NoError(err) + r.True(n == nil || !n.HasActiveBestClaim()) +} + +func TestClaimTakeover(t *testing.T) { + r := require.New(t) + setup(t) + param.ActiveParams.ActiveDelayFactor = 1 + + ct, err := New(cfg) + r.NoError(err) + r.NotNil(ct) + defer ct.Close() + + incrementBlock(r, ct, 1) + + hash := chainhash.HashH([]byte{1, 2, 3}) + o1 := wire.OutPoint{Hash: hash, Index: 1} + err = ct.AddClaim([]byte("test"), o1, change.NewClaimID(o1), 8) + r.NoError(err) + + incrementBlock(r, ct, 10) + + o2 := wire.OutPoint{Hash: hash, Index: 2} + err = ct.AddClaim([]byte("test"), o2, change.NewClaimID(o2), 18) + r.NoError(err) + + incrementBlock(r, ct, 10) + + n, err := ct.NodeAt(ct.height, []byte("test")) + r.NoError(err) + r.Equal(o1.String(), n.BestClaim.OutPoint.String()) + + incrementBlock(r, ct, 1) + + n, err = ct.NodeAt(ct.height, []byte("test")) + r.NoError(err) + r.Equal(o2.String(), n.BestClaim.OutPoint.String()) + + incrementBlock(r, ct, -1) + n, err = ct.NodeAt(ct.height, []byte("test")) + r.NoError(err) + r.Equal(o1.String(), n.BestClaim.OutPoint.String()) +} + +func TestSpendClaim(t *testing.T) { + r := require.New(t) + setup(t) + param.ActiveParams.ActiveDelayFactor = 1 + + ct, err := New(cfg) + r.NoError(err) + r.NotNil(ct) + defer ct.Close() + + incrementBlock(r, ct, 1) + + hash := chainhash.HashH([]byte{1, 2, 3}) + o1 := wire.OutPoint{Hash: hash, Index: 1} + err = ct.AddClaim([]byte("test"), o1, change.NewClaimID(o1), 18) + r.NoError(err) + o2 := wire.OutPoint{Hash: hash, Index: 2} + err = ct.AddClaim([]byte("test"), o2, change.NewClaimID(o2), 8) + r.NoError(err) + + incrementBlock(r, ct, 1) + + err = ct.SpendClaim([]byte("test"), o1, change.NewClaimID(o1)) + r.NoError(err) + + incrementBlock(r, ct, 1) + + n, err := ct.NodeAt(ct.height, []byte("test")) + r.NoError(err) + r.Equal(o2.String(), n.BestClaim.OutPoint.String()) + + incrementBlock(r, ct, -1) + + o3 := wire.OutPoint{Hash: hash, Index: 3} + err = ct.AddClaim([]byte("test"), o3, change.NewClaimID(o3), 22) + r.NoError(err) + + incrementBlock(r, ct, 10) + + o4 := wire.OutPoint{Hash: hash, Index: 4} + err = ct.AddClaim([]byte("test"), o4, change.NewClaimID(o4), 28) + r.NoError(err) + + incrementBlock(r, ct, 1) + + n, err = ct.NodeAt(ct.height, []byte("test")) + r.NoError(err) + r.Equal(o3.String(), n.BestClaim.OutPoint.String()) + + err = ct.SpendClaim([]byte("test"), o3, n.BestClaim.ClaimID) + r.NoError(err) + + incrementBlock(r, ct, 1) + + n, err = ct.NodeAt(ct.height, []byte("test")) + r.NoError(err) + r.Equal(o4.String(), n.BestClaim.OutPoint.String()) + + err = ct.SpendClaim([]byte("test"), o1, change.NewClaimID(o1)) + r.NoError(err) + err = ct.SpendClaim([]byte("test"), o2, change.NewClaimID(o2)) + r.NoError(err) + err = ct.SpendClaim([]byte("test"), o3, change.NewClaimID(o3)) + r.NoError(err) + err = ct.SpendClaim([]byte("test"), o4, change.NewClaimID(o4)) + r.NoError(err) + + incrementBlock(r, ct, 1) + + n, err = ct.NodeAt(ct.height, []byte("test")) + r.NoError(err) + r.True(n == nil || !n.HasActiveBestClaim()) + + h := ct.MerkleHash() + r.Equal(merkletrie.EmptyTrieHash.String(), h.String()) +} + +func TestSupportDelay(t *testing.T) { + r := require.New(t) + setup(t) + param.ActiveParams.ActiveDelayFactor = 1 + + ct, err := New(cfg) + r.NoError(err) + r.NotNil(ct) + defer ct.Close() + + incrementBlock(r, ct, 1) + + hash := chainhash.HashH([]byte{1, 2, 3}) + o1 := wire.OutPoint{Hash: hash, Index: 1} + err = ct.AddClaim([]byte("test"), o1, change.NewClaimID(o1), 18) + r.NoError(err) + o2 := wire.OutPoint{Hash: hash, Index: 2} + err = ct.AddClaim([]byte("test"), o2, change.NewClaimID(o2), 8) + r.NoError(err) + + o3 := wire.OutPoint{Hash: hash, Index: 3} + err = ct.AddSupport([]byte("test"), o3, 18, change.NewClaimID(o3)) // using bad ClaimID on purpose + r.NoError(err) + o4 := wire.OutPoint{Hash: hash, Index: 4} + err = ct.AddSupport([]byte("test"), o4, 18, change.NewClaimID(o2)) + r.NoError(err) + + incrementBlock(r, ct, 1) + + n, err := ct.NodeAt(ct.height, []byte("test")) + r.NoError(err) + r.Equal(o2.String(), n.BestClaim.OutPoint.String()) + + incrementBlock(r, ct, 10) + + o5 := wire.OutPoint{Hash: hash, Index: 5} + err = ct.AddSupport([]byte("test"), o5, 18, change.NewClaimID(o1)) + r.NoError(err) + + incrementBlock(r, ct, 1) + + n, err = ct.NodeAt(ct.height, []byte("test")) + r.NoError(err) + r.Equal(o2.String(), n.BestClaim.OutPoint.String()) + + incrementBlock(r, ct, 11) + + n, err = ct.NodeAt(ct.height, []byte("test")) + r.NoError(err) + r.Equal(o1.String(), n.BestClaim.OutPoint.String()) + + incrementBlock(r, ct, -1) + + n, err = ct.NodeAt(ct.height, []byte("test")) + r.NoError(err) + r.Equal(o2.String(), n.BestClaim.OutPoint.String()) +} + +func TestSupportSpending(t *testing.T) { + r := require.New(t) + setup(t) + param.ActiveParams.ActiveDelayFactor = 1 + + ct, err := New(cfg) + r.NoError(err) + r.NotNil(ct) + defer ct.Close() + + incrementBlock(r, ct, 1) + + hash := chainhash.HashH([]byte{1, 2, 3}) + o1 := wire.OutPoint{Hash: hash, Index: 1} + err = ct.AddClaim([]byte("test"), o1, change.NewClaimID(o1), 18) + r.NoError(err) + + incrementBlock(r, ct, 1) + + o3 := wire.OutPoint{Hash: hash, Index: 3} + err = ct.AddSupport([]byte("test"), o3, 18, change.NewClaimID(o1)) + r.NoError(err) + + err = ct.SpendClaim([]byte("test"), o1, change.NewClaimID(o1)) + r.NoError(err) + + incrementBlock(r, ct, 1) + + n, err := ct.NodeAt(ct.height, []byte("test")) + r.NoError(err) + r.True(n == nil || !n.HasActiveBestClaim()) +} + +func TestSupportOnUpdate(t *testing.T) { + r := require.New(t) + setup(t) + param.ActiveParams.ActiveDelayFactor = 1 + + ct, err := New(cfg) + r.NoError(err) + r.NotNil(ct) + defer ct.Close() + + incrementBlock(r, ct, 1) + + hash := chainhash.HashH([]byte{1, 2, 3}) + o1 := wire.OutPoint{Hash: hash, Index: 1} + err = ct.AddClaim([]byte("test"), o1, change.NewClaimID(o1), 18) + r.NoError(err) + + err = ct.SpendClaim([]byte("test"), o1, change.NewClaimID(o1)) + r.NoError(err) + + o2 := wire.OutPoint{Hash: hash, Index: 2} + err = ct.UpdateClaim([]byte("test"), o2, 28, change.NewClaimID(o1)) + r.NoError(err) + + incrementBlock(r, ct, 1) + + n, err := ct.NodeAt(ct.height, []byte("test")) + r.NoError(err) + r.Equal(int64(28), n.BestClaim.Amount) + + incrementBlock(r, ct, 1) + + err = ct.SpendClaim([]byte("test"), o2, change.NewClaimID(o1)) + r.NoError(err) + + o3 := wire.OutPoint{Hash: hash, Index: 3} + err = ct.UpdateClaim([]byte("test"), o3, 38, change.NewClaimID(o1)) + r.NoError(err) + + o4 := wire.OutPoint{Hash: hash, Index: 4} + err = ct.AddSupport([]byte("test"), o4, 2, change.NewClaimID(o1)) + r.NoError(err) + + o5 := wire.OutPoint{Hash: hash, Index: 5} + err = ct.AddClaim([]byte("test"), o5, change.NewClaimID(o5), 39) + r.NoError(err) + + incrementBlock(r, ct, 1) + + n, err = ct.NodeAt(ct.height, []byte("test")) + r.NoError(err) + r.Equal(int64(40), n.BestClaim.Amount+n.SupportSums[n.BestClaim.ClaimID.Key()]) + + err = ct.SpendSupport([]byte("test"), o4, n.BestClaim.ClaimID) + r.NoError(err) + + incrementBlock(r, ct, 1) + + // NOTE: LBRYcrd did not test that supports can trigger a takeover correctly (and it doesn't work here): + // n, err = ct.NodeAt(ct.height, []byte("test")) + // r.NoError(err) + // r.Equal(int64(39), n.BestClaim.Amount + n.SupportSums[n.BestClaim.ClaimID.Key()]) +} + +func TestSupportPreservation(t *testing.T) { + r := require.New(t) + setup(t) + param.ActiveParams.ActiveDelayFactor = 1 + + ct, err := New(cfg) + r.NoError(err) + r.NotNil(ct) + defer ct.Close() + + incrementBlock(r, ct, 1) + + hash := chainhash.HashH([]byte{1, 2, 3}) + o1 := wire.OutPoint{Hash: hash, Index: 1} + o2 := wire.OutPoint{Hash: hash, Index: 2} + o3 := wire.OutPoint{Hash: hash, Index: 3} + o4 := wire.OutPoint{Hash: hash, Index: 4} + o5 := wire.OutPoint{Hash: hash, Index: 5} + + err = ct.AddSupport([]byte("test"), o2, 10, change.NewClaimID(o1)) + r.NoError(err) + + incrementBlock(r, ct, 1) + + err = ct.AddClaim([]byte("test"), o1, change.NewClaimID(o1), 18) + r.NoError(err) + + err = ct.AddClaim([]byte("test"), o3, change.NewClaimID(o3), 7) + r.NoError(err) + + incrementBlock(r, ct, 10) + + n, err := ct.NodeAt(ct.height, []byte("test")) + r.NoError(err) + r.Equal(int64(28), n.BestClaim.Amount+n.SupportSums[n.BestClaim.ClaimID.Key()]) + + err = ct.AddSupport([]byte("test"), o4, 10, change.NewClaimID(o1)) + r.NoError(err) + err = ct.AddSupport([]byte("test"), o5, 100, change.NewClaimID(o3)) + r.NoError(err) + + incrementBlock(r, ct, 1) + + n, err = ct.NodeAt(ct.height, []byte("test")) + r.NoError(err) + r.Equal(int64(38), n.BestClaim.Amount+n.SupportSums[n.BestClaim.ClaimID.Key()]) + + incrementBlock(r, ct, 10) + + n, err = ct.NodeAt(ct.height, []byte("test")) + r.NoError(err) + r.Equal(int64(107), n.BestClaim.Amount+n.SupportSums[n.BestClaim.ClaimID.Key()]) +} + +func TestInvalidClaimID(t *testing.T) { + r := require.New(t) + setup(t) + param.ActiveParams.ActiveDelayFactor = 1 + + ct, err := New(cfg) + r.NoError(err) + r.NotNil(ct) + defer ct.Close() + + incrementBlock(r, ct, 1) + + hash := chainhash.HashH([]byte{1, 2, 3}) + o1 := wire.OutPoint{Hash: hash, Index: 1} + o2 := wire.OutPoint{Hash: hash, Index: 2} + o3 := wire.OutPoint{Hash: hash, Index: 3} + + err = ct.AddClaim([]byte("test"), o1, change.NewClaimID(o1), 10) + r.NoError(err) + + incrementBlock(r, ct, 1) + + err = ct.SpendClaim([]byte("test"), o3, change.NewClaimID(o1)) + r.NoError(err) + + err = ct.UpdateClaim([]byte("test"), o2, 18, change.NewClaimID(o3)) + r.NoError(err) + + incrementBlock(r, ct, 12) + + n, err := ct.NodeAt(ct.height, []byte("test")) + r.NoError(err) + r.Len(n.Claims, 1) + r.Len(n.Supports, 0) + r.Equal(int64(10), n.BestClaim.Amount+n.SupportSums[n.BestClaim.ClaimID.Key()]) +} + +func TestStableTrieHash(t *testing.T) { + r := require.New(t) + setup(t) + param.ActiveParams.ActiveDelayFactor = 1 + param.ActiveParams.AllClaimsInMerkleForkHeight = 8 // changes on this one + + ct, err := New(cfg) + r.NoError(err) + r.NotNil(ct) + defer ct.Close() + + hash := chainhash.HashH([]byte{1, 2, 3}) + o1 := wire.OutPoint{Hash: hash, Index: 1} + + err = ct.AddClaim([]byte("test"), o1, change.NewClaimID(o1), 1) + r.NoError(err) + + incrementBlock(r, ct, 1) + + h := ct.MerkleHash() + r.NotEqual(merkletrie.EmptyTrieHash.String(), h.String()) + + for i := 0; i < 6; i++ { + incrementBlock(r, ct, 1) + r.Equal(h.String(), ct.MerkleHash().String()) + } + + incrementBlock(r, ct, 1) + + r.NotEqual(h.String(), ct.MerkleHash()) + h = ct.MerkleHash() + + for i := 0; i < 16; i++ { + incrementBlock(r, ct, 1) + r.Equal(h.String(), ct.MerkleHash().String()) + } +} + +func TestBlock884431(t *testing.T) { + r := require.New(t) + setup(t) + param.ActiveParams.ActiveDelayFactor = 1 + param.ActiveParams.MaxRemovalWorkaroundHeight = 0 + param.ActiveParams.AllClaimsInMerkleForkHeight = 0 + + ct, err := New(cfg) + r.NoError(err) + r.NotNil(ct) + defer ct.Close() + + // in this block we have a scenario where we update all the child names + // which, in the old code, caused a trie vertex to be removed + // which, in turn, would trigger a premature takeover + + c := byte(10) + + add := func(s string, amt int64) wire.OutPoint { + h := chainhash.HashH([]byte{c}) + c++ + o := wire.OutPoint{Hash: h, Index: 1} + err := ct.AddClaim([]byte(s), o, change.NewClaimID(o), amt) + r.NoError(err) + return o + } + + update := func(s string, o wire.OutPoint, amt int64) wire.OutPoint { + err = ct.SpendClaim([]byte(s), o, change.NewClaimID(o)) + r.NoError(err) + + h := chainhash.HashH([]byte{c}) + c++ + o2 := wire.OutPoint{Hash: h, Index: 2} + + err = ct.UpdateClaim([]byte(s), o2, amt, change.NewClaimID(o)) + r.NoError(err) + return o2 + } + + o1a := add("go", 10) + o1b := add("go", 20) + o2 := add("goop", 10) + o3 := add("gog", 20) + + o4a := add("test", 10) + o4b := add("test", 20) + o5 := add("tester", 10) + o6 := add("testing", 20) + + for i := 0; i < 10; i++ { + err = ct.AppendBlock() + r.NoError(err) + } + n, err := ct.NodeAt(ct.height, []byte("go")) + r.NoError(err) + r.Equal(o1b.String(), n.BestClaim.OutPoint.String()) + n, err = ct.NodeAt(ct.height, []byte("test")) + r.NoError(err) + r.Equal(o4b.String(), n.BestClaim.OutPoint.String()) + + update("go", o1b, 30) + o10 := update("go", o1a, 40) + update("gog", o3, 30) + update("goop", o2, 30) + + update("testing", o6, 30) + o11 := update("test", o4b, 30) + update("test", o4a, 40) + update("tester", o5, 30) + + incrementBlock(r, ct, 1) + + n, err = ct.NodeAt(ct.height, []byte("go")) + r.NoError(err) + r.Equal(o10.String(), n.BestClaim.OutPoint.String()) + n, err = ct.NodeAt(ct.height, []byte("test")) + r.NoError(err) + r.Equal(o11.String(), n.BestClaim.OutPoint.String()) +} diff --git a/claimtrie/cmd/cmd/block.go b/claimtrie/cmd/cmd/block.go new file mode 100644 index 00000000..d0ee7719 --- /dev/null +++ b/claimtrie/cmd/cmd/block.go @@ -0,0 +1,98 @@ +package cmd + +import ( + "fmt" + + "github.com/cockroachdb/errors" + "github.com/spf13/cobra" +) + +func init() { + rootCmd.AddCommand(NewBlocCommands()) +} + +func NewBlocCommands() *cobra.Command { + + cmd := &cobra.Command{ + Use: "block", + Short: "Block related commands", + } + + cmd.AddCommand(NewBlockBestCommand()) + cmd.AddCommand(NewBlockListCommand()) + + return cmd +} + +func NewBlockBestCommand() *cobra.Command { + + cmd := &cobra.Command{ + Use: "best", + Short: "Show the height and hash of the best block", + RunE: func(cmd *cobra.Command, args []string) error { + + db, err := loadBlocksDB() + if err != nil { + return errors.Wrapf(err, "load blocks database") + } + defer db.Close() + + chain, err := loadChain(db) + if err != nil { + return errors.Wrapf(err, "load chain") + } + + state := chain.BestSnapshot() + fmt.Printf("Block %7d: %s\n", state.Height, state.Hash.String()) + + return nil + }, + } + + return cmd +} + +func NewBlockListCommand() *cobra.Command { + + var fromHeight int32 + var toHeight int32 + + cmd := &cobra.Command{ + Use: "list", + Short: "List merkle hash of blocks between ", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + + db, err := loadBlocksDB() + if err != nil { + return errors.Wrapf(err, "load blocks database") + } + defer db.Close() + + chain, err := loadChain(db) + if err != nil { + return errors.Wrapf(err, "load chain") + } + + if toHeight > chain.BestSnapshot().Height { + toHeight = chain.BestSnapshot().Height + } + + for ht := fromHeight; ht <= toHeight; ht++ { + hash, err := chain.BlockHashByHeight(ht) + if err != nil { + return errors.Wrapf(err, "load hash for %d", ht) + } + fmt.Printf("Block %7d: %s\n", ht, hash.String()) + } + + return nil + }, + } + + cmd.Flags().Int32Var(&fromHeight, "from", 0, "From height (inclusive)") + cmd.Flags().Int32Var(&toHeight, "to", 0, "To height (inclusive)") + cmd.Flags().SortFlags = false + + return cmd +} diff --git a/claimtrie/cmd/cmd/chain.go b/claimtrie/cmd/cmd/chain.go new file mode 100644 index 00000000..45e843b5 --- /dev/null +++ b/claimtrie/cmd/cmd/chain.go @@ -0,0 +1,441 @@ +package cmd + +import ( + "os" + "path/filepath" + "sync" + "time" + + "github.com/lbryio/lbcd/blockchain" + "github.com/lbryio/lbcd/claimtrie" + "github.com/lbryio/lbcd/claimtrie/chain" + "github.com/lbryio/lbcd/claimtrie/chain/chainrepo" + "github.com/lbryio/lbcd/claimtrie/change" + "github.com/lbryio/lbcd/claimtrie/config" + "github.com/lbryio/lbcd/database" + _ "github.com/lbryio/lbcd/database/ffldb" + "github.com/lbryio/lbcd/txscript" + "github.com/lbryio/lbcd/wire" + btcutil "github.com/lbryio/lbcutil" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble" + "github.com/spf13/cobra" +) + +func init() { + rootCmd.AddCommand(NewChainCommands()) +} + +func NewChainCommands() *cobra.Command { + + cmd := &cobra.Command{ + Use: "chain", + Short: "chain related command", + } + + cmd.AddCommand(NewChainDumpCommand()) + cmd.AddCommand(NewChainReplayCommand()) + cmd.AddCommand(NewChainConvertCommand()) + + return cmd +} + +func NewChainDumpCommand() *cobra.Command { + + var chainRepoPath string + var fromHeight int32 + var toHeight int32 + + cmd := &cobra.Command{ + Use: "dump", + Short: "Dump the chain changes between and ", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + + dbPath := chainRepoPath + log.Debugf("Open chain repo: %q", dbPath) + chainRepo, err := chainrepo.NewPebble(dbPath) + if err != nil { + return errors.Wrapf(err, "open chain repo") + } + + for height := fromHeight; height <= toHeight; height++ { + changes, err := chainRepo.Load(height) + if errors.Is(err, pebble.ErrNotFound) { + continue + } + if err != nil { + return errors.Wrapf(err, "load charnges for height: %d") + } + for _, chg := range changes { + showChange(chg) + } + } + + return nil + }, + } + + cmd.Flags().StringVar(&chainRepoPath, "chaindb", "chain_db", "Claim operation database") + cmd.Flags().Int32Var(&fromHeight, "from", 0, "From height (inclusive)") + cmd.Flags().Int32Var(&toHeight, "to", 0, "To height (inclusive)") + cmd.Flags().SortFlags = false + + return cmd +} + +func NewChainReplayCommand() *cobra.Command { + + var chainRepoPath string + var fromHeight int32 + var toHeight int32 + + cmd := &cobra.Command{ + Use: "replay", + Short: "Replay the chain changes between and ", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + + for _, dbName := range []string{ + cfg.BlockRepoPebble.Path, + cfg.NodeRepoPebble.Path, + cfg.MerkleTrieRepoPebble.Path, + cfg.TemporalRepoPebble.Path, + } { + dbPath := filepath.Join(dataDir, netName, "claim_dbs", dbName) + log.Debugf("Delete repo: %q", dbPath) + err := os.RemoveAll(dbPath) + if err != nil { + return errors.Wrapf(err, "delete repo: %q", dbPath) + } + } + + log.Debugf("Open chain repo: %q", chainRepoPath) + chainRepo, err := chainrepo.NewPebble(chainRepoPath) + if err != nil { + return errors.Wrapf(err, "open chain repo") + } + + cfg := config.DefaultConfig + cfg.RamTrie = true + cfg.DataDir = filepath.Join(dataDir, netName) + + ct, err := claimtrie.New(cfg) + if err != nil { + return errors.Wrapf(err, "create claimtrie") + } + defer ct.Close() + + db, err := loadBlocksDB() + if err != nil { + return errors.Wrapf(err, "load blocks database") + } + + chain, err := loadChain(db) + if err != nil { + return errors.Wrapf(err, "load chain") + } + + startTime := time.Now() + for ht := fromHeight; ht < toHeight; ht++ { + + changes, err := chainRepo.Load(ht + 1) + if errors.Is(err, pebble.ErrNotFound) { + // do nothing. + } else if err != nil { + return errors.Wrapf(err, "load changes for block %d", ht) + } + + for _, chg := range changes { + + switch chg.Type { + case change.AddClaim: + err = ct.AddClaim(chg.Name, chg.OutPoint, chg.ClaimID, chg.Amount) + case change.UpdateClaim: + err = ct.UpdateClaim(chg.Name, chg.OutPoint, chg.Amount, chg.ClaimID) + case change.SpendClaim: + err = ct.SpendClaim(chg.Name, chg.OutPoint, chg.ClaimID) + case change.AddSupport: + err = ct.AddSupport(chg.Name, chg.OutPoint, chg.Amount, chg.ClaimID) + case change.SpendSupport: + err = ct.SpendSupport(chg.Name, chg.OutPoint, chg.ClaimID) + default: + err = errors.Errorf("invalid change type: %v", chg) + } + + if err != nil { + return errors.Wrapf(err, "execute change %v", chg) + } + } + err = appendBlock(ct, chain) + if err != nil { + return errors.Wrapf(err, "appendBlock") + } + + if time.Since(startTime) > 5*time.Second { + log.Infof("Block: %d", ct.Height()) + startTime = time.Now() + } + } + + return nil + }, + } + + cmd.Flags().StringVar(&chainRepoPath, "chaindb", "chain_db", "Claim operation database") + cmd.Flags().Int32Var(&fromHeight, "from", 0, "From height") + cmd.Flags().Int32Var(&toHeight, "to", 0, "To height") + cmd.Flags().SortFlags = false + + return cmd +} + +func appendBlock(ct *claimtrie.ClaimTrie, chain *blockchain.BlockChain) error { + + err := ct.AppendBlock() + if err != nil { + return errors.Wrapf(err, "append block: %w") + } + + blockHash, err := chain.BlockHashByHeight(ct.Height()) + if err != nil { + return errors.Wrapf(err, "load from block repo: %w") + } + + header, err := chain.HeaderByHash(blockHash) + + if err != nil { + return errors.Wrapf(err, "load from block repo: %w") + } + + if *ct.MerkleHash() != header.ClaimTrie { + return errors.Errorf("hash mismatched at height %5d: exp: %s, got: %s", + ct.Height(), header.ClaimTrie, ct.MerkleHash()) + } + + return nil +} + +func NewChainConvertCommand() *cobra.Command { + + var chainRepoPath string + var toHeight int32 + + cmd := &cobra.Command{ + Use: "convert", + Short: "convert changes from 0 to ", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + + db, err := loadBlocksDB() + if err != nil { + return errors.Wrapf(err, "load block db") + } + defer db.Close() + + chain, err := loadChain(db) + if err != nil { + return errors.Wrapf(err, "load block db") + } + + if toHeight > chain.BestSnapshot().Height { + toHeight = chain.BestSnapshot().Height + } + + chainRepo, err := chainrepo.NewPebble(chainRepoPath) + if err != nil { + return errors.Wrapf(err, "open chain repo: %v") + } + defer chainRepo.Close() + + converter := chainConverter{ + db: db, + chain: chain, + chainRepo: chainRepo, + toHeight: toHeight, + blockChan: make(chan *btcutil.Block, 1000), + changesChan: make(chan []change.Change, 1000), + wg: &sync.WaitGroup{}, + stat: &stat{}, + } + + startTime := time.Now() + err = converter.start() + if err != nil { + return errors.Wrapf(err, "start Converter") + } + + converter.wait() + log.Infof("Convert chain: took %s", time.Since(startTime)) + + return nil + }, + } + + cmd.Flags().StringVar(&chainRepoPath, "chaindb", "chain_db", "Claim operation database") + cmd.Flags().Int32Var(&toHeight, "to", 0, "toHeight") + cmd.Flags().SortFlags = false + return cmd +} + +type stat struct { + blocksFetched int + blocksProcessed int + changesSaved int +} + +type chainConverter struct { + db database.DB + chain *blockchain.BlockChain + chainRepo chain.Repo + toHeight int32 + + blockChan chan *btcutil.Block + changesChan chan []change.Change + + wg *sync.WaitGroup + + stat *stat +} + +func (cc *chainConverter) wait() { + cc.wg.Wait() +} + +func (cb *chainConverter) start() error { + + go cb.reportStats() + + cb.wg.Add(3) + go cb.getBlock() + go cb.processBlock() + go cb.saveChanges() + + return nil +} + +func (cb *chainConverter) getBlock() { + defer cb.wg.Done() + defer close(cb.blockChan) + + for ht := int32(0); ht < cb.toHeight; ht++ { + block, err := cb.chain.BlockByHeight(ht) + if err != nil { + if errors.Cause(err).Error() == "too many open files" { + err = errors.WithHintf(err, "try ulimit -n 2048") + } + log.Errorf("load changes at %d: %s", ht, err) + return + } + cb.stat.blocksFetched++ + cb.blockChan <- block + } +} + +func (cb *chainConverter) processBlock() { + defer cb.wg.Done() + defer close(cb.changesChan) + + utxoPubScripts := map[wire.OutPoint][]byte{} + for block := range cb.blockChan { + var changes []change.Change + for _, tx := range block.Transactions() { + + if blockchain.IsCoinBase(tx) { + continue + } + + for _, txIn := range tx.MsgTx().TxIn { + prevOutpoint := txIn.PreviousOutPoint + pkScript := utxoPubScripts[prevOutpoint] + cs, err := txscript.DecodeClaimScript(pkScript) + if err == txscript.ErrNotClaimScript { + continue + } + if err != nil { + log.Criticalf("Can't parse claim script: %s", err) + } + + chg := change.Change{ + Height: block.Height(), + Name: cs.Name(), + OutPoint: txIn.PreviousOutPoint, + } + delete(utxoPubScripts, prevOutpoint) + + switch cs.Opcode() { + case txscript.OP_CLAIMNAME: + chg.Type = change.SpendClaim + chg.ClaimID = change.NewClaimID(chg.OutPoint) + case txscript.OP_UPDATECLAIM: + chg.Type = change.SpendClaim + copy(chg.ClaimID[:], cs.ClaimID()) + case txscript.OP_SUPPORTCLAIM: + chg.Type = change.SpendSupport + copy(chg.ClaimID[:], cs.ClaimID()) + } + + changes = append(changes, chg) + } + + op := *wire.NewOutPoint(tx.Hash(), 0) + for i, txOut := range tx.MsgTx().TxOut { + cs, err := txscript.DecodeClaimScript(txOut.PkScript) + if err == txscript.ErrNotClaimScript { + continue + } + + op.Index = uint32(i) + chg := change.Change{ + Height: block.Height(), + Name: cs.Name(), + OutPoint: op, + Amount: txOut.Value, + } + utxoPubScripts[op] = txOut.PkScript + + switch cs.Opcode() { + case txscript.OP_CLAIMNAME: + chg.Type = change.AddClaim + chg.ClaimID = change.NewClaimID(op) + case txscript.OP_SUPPORTCLAIM: + chg.Type = change.AddSupport + copy(chg.ClaimID[:], cs.ClaimID()) + case txscript.OP_UPDATECLAIM: + chg.Type = change.UpdateClaim + copy(chg.ClaimID[:], cs.ClaimID()) + } + changes = append(changes, chg) + } + } + cb.stat.blocksProcessed++ + + if len(changes) != 0 { + cb.changesChan <- changes + } + } +} + +func (cb *chainConverter) saveChanges() { + defer cb.wg.Done() + + for changes := range cb.changesChan { + err := cb.chainRepo.Save(changes[0].Height, changes) + if err != nil { + log.Errorf("save to chain repo: %s", err) + return + } + cb.stat.changesSaved++ + } +} + +func (cb *chainConverter) reportStats() { + stat := cb.stat + tick := time.NewTicker(5 * time.Second) + for range tick.C { + log.Infof("block : %7d / %7d, changes saved: %d", + stat.blocksFetched, stat.blocksProcessed, stat.changesSaved) + + } +} diff --git a/claimtrie/cmd/cmd/helper.go b/claimtrie/cmd/cmd/helper.go new file mode 100644 index 00000000..e75da402 --- /dev/null +++ b/claimtrie/cmd/cmd/helper.go @@ -0,0 +1,62 @@ +package cmd + +import ( + "path/filepath" + "time" + + "github.com/lbryio/lbcd/blockchain" + "github.com/lbryio/lbcd/chaincfg" + "github.com/lbryio/lbcd/database" + "github.com/lbryio/lbcd/txscript" + + "github.com/cockroachdb/errors" +) + +func loadBlocksDB() (database.DB, error) { + + dbPath := filepath.Join(dataDir, netName, "blocks_ffldb") + log.Infof("Loading blocks database: %s", dbPath) + db, err := database.Open("ffldb", dbPath, chainPramas().Net) + if err != nil { + return nil, errors.Wrapf(err, "open blocks database") + } + + return db, nil +} + +func loadChain(db database.DB) (*blockchain.BlockChain, error) { + paramsCopy := chaincfg.MainNetParams + + log.Infof("Loading chain from database") + + startTime := time.Now() + chain, err := blockchain.New(&blockchain.Config{ + DB: db, + ChainParams: ¶msCopy, + TimeSource: blockchain.NewMedianTime(), + SigCache: txscript.NewSigCache(1000), + }) + if err != nil { + return nil, errors.Wrapf(err, "create blockchain") + } + + log.Infof("Loaded chain from database (%s)", time.Since(startTime)) + + return chain, err + +} + +func chainPramas() chaincfg.Params { + + // Make a copy so the user won't modify the global instance. + params := chaincfg.MainNetParams + switch netName { + case "mainnet": + params = chaincfg.MainNetParams + case "testnet": + params = chaincfg.TestNet3Params + case "regtest": + params = chaincfg.RegressionNetParams + } + return params +} diff --git a/claimtrie/cmd/cmd/merkletrie.go b/claimtrie/cmd/cmd/merkletrie.go new file mode 100644 index 00000000..66694c98 --- /dev/null +++ b/claimtrie/cmd/cmd/merkletrie.go @@ -0,0 +1,105 @@ +package cmd + +import ( + "fmt" + "path/filepath" + + "github.com/lbryio/lbcd/claimtrie/merkletrie" + "github.com/lbryio/lbcd/claimtrie/merkletrie/merkletrierepo" + "github.com/lbryio/lbcd/claimtrie/temporal/temporalrepo" + + "github.com/cockroachdb/errors" + "github.com/spf13/cobra" +) + +func init() { + rootCmd.AddCommand(NewTrieCommands()) +} + +func NewTrieCommands() *cobra.Command { + + cmd := &cobra.Command{ + Use: "trie", + Short: "MerkleTrie related commands", + } + + cmd.AddCommand(NewTrieNameCommand()) + + return cmd +} + +func NewTrieNameCommand() *cobra.Command { + + var height int32 + var name string + + cmd := &cobra.Command{ + Use: "name", + Short: "List the claim and child hashes at vertex name of block at height", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + + db, err := loadBlocksDB() + if err != nil { + return errors.Wrapf(err, "load blocks database") + } + defer db.Close() + + chain, err := loadChain(db) + if err != nil { + return errors.Wrapf(err, "load chain") + } + + state := chain.BestSnapshot() + fmt.Printf("Block %7d: %s\n", state.Height, state.Hash.String()) + + if height > state.Height { + return errors.New("requested height is unavailable") + } + + hash := state.Hash + + dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.MerkleTrieRepoPebble.Path) + log.Debugf("Open merkletrie repo: %q", dbPath) + trieRepo, err := merkletrierepo.NewPebble(dbPath) + if err != nil { + return errors.Wrapf(err, "open merkle trie repo") + } + + trie := merkletrie.NewPersistentTrie(trieRepo) + defer trie.Close() + + trie.SetRoot(&hash) + + if len(name) > 1 { + trie.Dump(name) + return nil + } + + dbPath = filepath.Join(dataDir, netName, "claim_dbs", cfg.TemporalRepoPebble.Path) + log.Debugf("Open temporal repo: %q", dbPath) + tmpRepo, err := temporalrepo.NewPebble(dbPath) + if err != nil { + return errors.Wrapf(err, "open temporal repo") + } + + nodes, err := tmpRepo.NodesAt(height) + if err != nil { + return errors.Wrapf(err, "read temporal repo at %d", height) + } + + for _, name := range nodes { + fmt.Printf("Name: %s, ", string(name)) + trie.Dump(string(name)) + } + + return nil + }, + } + + cmd.Flags().Int32Var(&height, "height", 0, "Height") + cmd.Flags().StringVar(&name, "name", "", "Name") + cmd.Flags().SortFlags = false + + return cmd +} diff --git a/claimtrie/cmd/cmd/node.go b/claimtrie/cmd/cmd/node.go new file mode 100644 index 00000000..08112e94 --- /dev/null +++ b/claimtrie/cmd/cmd/node.go @@ -0,0 +1,194 @@ +package cmd + +import ( + "encoding/hex" + "fmt" + "math" + "path/filepath" + + "github.com/cockroachdb/errors" + "github.com/lbryio/lbcd/claimtrie/change" + "github.com/lbryio/lbcd/claimtrie/node" + "github.com/lbryio/lbcd/claimtrie/node/noderepo" + + "github.com/spf13/cobra" +) + +func init() { + rootCmd.AddCommand(NewNodeCommands()) +} + +func NewNodeCommands() *cobra.Command { + + cmd := &cobra.Command{ + Use: "node", + Short: "Replay the application of changes on a node up to certain height", + } + + cmd.AddCommand(NewNodeDumpCommand()) + cmd.AddCommand(NewNodeReplayCommand()) + cmd.AddCommand(NewNodeChildrenCommand()) + cmd.AddCommand(NewNodeStatsCommand()) + + return cmd +} + +func NewNodeDumpCommand() *cobra.Command { + + var name string + var height int32 + + cmd := &cobra.Command{ + Use: "dump", + Short: "Replay the application of changes on a node up to certain height", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + + dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.NodeRepoPebble.Path) + log.Debugf("Open node repo: %q", dbPath) + repo, err := noderepo.NewPebble(dbPath) + if err != nil { + return errors.Wrapf(err, "open node repo") + } + defer repo.Close() + + changes, err := repo.LoadChanges([]byte(name)) + if err != nil { + return errors.Wrapf(err, "load commands") + } + + for _, chg := range changes { + if chg.Height > height { + break + } + showChange(chg) + } + + return nil + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Name") + cmd.MarkFlagRequired("name") + cmd.Flags().Int32Var(&height, "height", math.MaxInt32, "Height") + + return cmd +} + +func NewNodeReplayCommand() *cobra.Command { + + var name string + var height int32 + + cmd := &cobra.Command{ + Use: "replay", + Short: "Replay the changes of up to ", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + + dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.NodeRepoPebble.Path) + log.Debugf("Open node repo: %q", dbPath) + repo, err := noderepo.NewPebble(dbPath) + if err != nil { + return errors.Wrapf(err, "open node repo") + } + + bm, err := node.NewBaseManager(repo) + if err != nil { + return errors.Wrapf(err, "create node manager") + } + defer bm.Close() + + nm := node.NewNormalizingManager(bm) + + n, err := nm.NodeAt(height, []byte(name)) + if err != nil || n == nil { + return errors.Wrapf(err, "get node: %s", name) + } + + showNode(n) + return nil + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Name") + cmd.MarkFlagRequired("name") + cmd.Flags().Int32Var(&height, "height", 0, "Height (inclusive)") + cmd.Flags().SortFlags = false + + return cmd +} + +func NewNodeChildrenCommand() *cobra.Command { + + var name string + + cmd := &cobra.Command{ + Use: "children", + Short: "Show all the children names of a given node name", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + + dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.NodeRepoPebble.Path) + log.Debugf("Open node repo: %q", dbPath) + repo, err := noderepo.NewPebble(dbPath) + if err != nil { + return errors.Wrapf(err, "open node repo") + } + defer repo.Close() + + fn := func(changes []change.Change) bool { + fmt.Printf("Name: %s, Height: %d, %d\n", changes[0].Name, changes[0].Height, + changes[len(changes)-1].Height) + return true + } + + err = repo.IterateChildren([]byte(name), fn) + if err != nil { + return errors.Wrapf(err, "iterate children: %s", name) + } + + return nil + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Name") + cmd.MarkFlagRequired("name") + + return cmd +} + +func NewNodeStatsCommand() *cobra.Command { + + cmd := &cobra.Command{ + Use: "stat", + Short: "Determine the number of unique names, average changes per name, etc.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + + dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.NodeRepoPebble.Path) + log.Debugf("Open node repo: %q", dbPath) + repo, err := noderepo.NewPebble(dbPath) + if err != nil { + return errors.Wrapf(err, "open node repo") + } + defer repo.Close() + + n := 0 + c := 0 + err = repo.IterateChildren([]byte{}, func(changes []change.Change) bool { + c += len(changes) + n++ + if len(changes) > 5000 { + fmt.Printf("Name: %s, Hex: %s, Changes: %d\n", string(changes[0].Name), + hex.EncodeToString(changes[0].Name), len(changes)) + } + return true + }) + fmt.Printf("\nNames: %d, Average changes: %.2f\n", n, float64(c)/float64(n)) + return errors.Wrapf(err, "iterate node repo") + }, + } + + return cmd +} diff --git a/claimtrie/cmd/cmd/root.go b/claimtrie/cmd/cmd/root.go new file mode 100644 index 00000000..8b2fb75f --- /dev/null +++ b/claimtrie/cmd/cmd/root.go @@ -0,0 +1,61 @@ +package cmd + +import ( + "os" + + "github.com/btcsuite/btclog" + "github.com/lbryio/lbcd/claimtrie/config" + "github.com/lbryio/lbcd/claimtrie/param" + "github.com/lbryio/lbcd/limits" + "github.com/lbryio/lbcd/wire" + + "github.com/spf13/cobra" +) + +var ( + log btclog.Logger + cfg = config.DefaultConfig + netName string + dataDir string +) + +var rootCmd = NewRootCommand() + +func NewRootCommand() *cobra.Command { + + cmd := &cobra.Command{ + Use: "claimtrie", + Short: "ClaimTrie Command Line Interface", + SilenceUsage: true, + PersistentPreRun: func(cmd *cobra.Command, args []string) { + switch netName { + case "mainnet": + param.SetNetwork(wire.MainNet) + case "testnet": + param.SetNetwork(wire.TestNet3) + case "regtest": + param.SetNetwork(wire.TestNet) + } + }, + } + + cmd.PersistentFlags().StringVar(&netName, "netname", "mainnet", "Net name") + cmd.PersistentFlags().StringVarP(&dataDir, "datadir", "b", cfg.DataDir, "Data dir") + + return cmd +} + +func Execute() { + + backendLogger := btclog.NewBackend(os.Stdout) + defer os.Stdout.Sync() + log = backendLogger.Logger("CMDL") + log.SetLevel(btclog.LevelDebug) + + // Up some limits. + if err := limits.SetLimits(); err != nil { + log.Errorf("failed to set limits: %v\n", err) + } + + rootCmd.Execute() // nolint : errchk +} diff --git a/claimtrie/cmd/cmd/temporal.go b/claimtrie/cmd/cmd/temporal.go new file mode 100644 index 00000000..67d3397c --- /dev/null +++ b/claimtrie/cmd/cmd/temporal.go @@ -0,0 +1,60 @@ +package cmd + +import ( + "path/filepath" + + "github.com/lbryio/lbcd/claimtrie/temporal/temporalrepo" + + "github.com/cockroachdb/errors" + "github.com/spf13/cobra" +) + +func init() { + rootCmd.AddCommand(NewTemporalCommand()) +} + +func NewTemporalCommand() *cobra.Command { + + var fromHeight int32 + var toHeight int32 + + cmd := &cobra.Command{ + Use: "temporal", + Short: "List which nodes are update in a range of heights", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + + dbPath := filepath.Join(dataDir, netName, "claim_dbs", cfg.TemporalRepoPebble.Path) + log.Debugf("Open temporal repo: %s", dbPath) + repo, err := temporalrepo.NewPebble(dbPath) + if err != nil { + return errors.Wrapf(err, "open temporal repo") + } + + if toHeight <= 0 { + toHeight = fromHeight + } + + for ht := fromHeight; ht <= toHeight; ht++ { + names, err := repo.NodesAt(ht) + if err != nil { + return errors.Wrapf(err, "get node names from temporal") + } + + if len(names) == 0 { + continue + } + + showTemporalNames(ht, names) + } + + return nil + }, + } + + cmd.Flags().Int32Var(&fromHeight, "from", 0, "From height (inclusive)") + cmd.Flags().Int32Var(&toHeight, "to", 0, "To height (inclusive)") + cmd.Flags().SortFlags = false + + return cmd +} diff --git a/claimtrie/cmd/cmd/ui.go b/claimtrie/cmd/cmd/ui.go new file mode 100644 index 00000000..9882b474 --- /dev/null +++ b/claimtrie/cmd/cmd/ui.go @@ -0,0 +1,76 @@ +package cmd + +import ( + "fmt" + "strings" + + "github.com/lbryio/lbcd/claimtrie/change" + "github.com/lbryio/lbcd/claimtrie/node" +) + +var status = map[node.Status]string{ + node.Accepted: "Accepted", + node.Activated: "Activated", + node.Deactivated: "Deactivated", +} + +func changeType(c change.ChangeType) string { + switch c { + case change.AddClaim: + return "AddClaim" + case change.SpendClaim: + return "SpendClaim" + case change.UpdateClaim: + return "UpdateClaim" + case change.AddSupport: + return "AddSupport" + case change.SpendSupport: + return "SpendSupport" + } + return "Unknown" +} + +func showChange(chg change.Change) { + fmt.Printf(">>> Height: %6d: %s for %04s, %15d, %s - %s\n", + chg.Height, changeType(chg.Type), chg.ClaimID, chg.Amount, chg.OutPoint, chg.Name) +} + +func showClaim(c *node.Claim, n *node.Node) { + mark := " " + if c == n.BestClaim { + mark = "*" + } + + fmt.Printf("%s C ID: %s, TXO: %s\n %5d/%-5d, Status: %9s, Amount: %15d, Support Amount: %15d\n", + mark, c.ClaimID, c.OutPoint, c.AcceptedAt, c.ActiveAt, status[c.Status], c.Amount, n.SupportSums[c.ClaimID.Key()]) +} + +func showSupport(c *node.Claim) { + fmt.Printf(" S id: %s, op: %s, %5d/%-5d, %9s, amt: %15d\n", + c.ClaimID, c.OutPoint, c.AcceptedAt, c.ActiveAt, status[c.Status], c.Amount) +} + +func showNode(n *node.Node) { + + fmt.Printf("%s\n", strings.Repeat("-", 200)) + fmt.Printf("Last Node Takeover: %d\n\n", n.TakenOverAt) + n.SortClaimsByBid() + for _, c := range n.Claims { + showClaim(c, n) + for _, s := range n.Supports { + if s.ClaimID != c.ClaimID { + continue + } + showSupport(s) + } + } + fmt.Printf("\n\n") +} + +func showTemporalNames(height int32, names [][]byte) { + fmt.Printf("%7d: %q", height, names[0]) + for _, name := range names[1:] { + fmt.Printf(", %q ", name) + } + fmt.Printf("\n") +} diff --git a/claimtrie/cmd/main.go b/claimtrie/cmd/main.go new file mode 100644 index 00000000..b87adc7d --- /dev/null +++ b/claimtrie/cmd/main.go @@ -0,0 +1,9 @@ +package main + +import ( + "github.com/lbryio/lbcd/claimtrie/cmd/cmd" +) + +func main() { + cmd.Execute() +} diff --git a/claimtrie/config/config.go b/claimtrie/config/config.go new file mode 100644 index 00000000..4920ca17 --- /dev/null +++ b/claimtrie/config/config.go @@ -0,0 +1,49 @@ +package config + +import ( + "path/filepath" + + "github.com/lbryio/lbcd/claimtrie/param" + btcutil "github.com/lbryio/lbcutil" +) + +var DefaultConfig = Config{ + Params: param.MainNet, + + RamTrie: true, // as it stands the other trie uses more RAM, more time, and 40GB+ of disk space + + DataDir: filepath.Join(btcutil.AppDataDir("chain", false), "data"), + + BlockRepoPebble: pebbleConfig{ + Path: "blocks_pebble_db", + }, + NodeRepoPebble: pebbleConfig{ + Path: "node_change_pebble_db", + }, + TemporalRepoPebble: pebbleConfig{ + Path: "temporal_pebble_db", + }, + MerkleTrieRepoPebble: pebbleConfig{ + Path: "merkletrie_pebble_db", + }, +} + +// Config is the container of all configurations. +type Config struct { + Params param.ClaimTrieParams + + RamTrie bool + + DataDir string + + BlockRepoPebble pebbleConfig + NodeRepoPebble pebbleConfig + TemporalRepoPebble pebbleConfig + MerkleTrieRepoPebble pebbleConfig + + Interrupt <-chan struct{} +} + +type pebbleConfig struct { + Path string +} diff --git a/claimtrie/merkletrie/collapsedtrie.go b/claimtrie/merkletrie/collapsedtrie.go new file mode 100644 index 00000000..18af30a0 --- /dev/null +++ b/claimtrie/merkletrie/collapsedtrie.go @@ -0,0 +1,235 @@ +package merkletrie + +import ( + "github.com/lbryio/lbcd/chaincfg/chainhash" +) + +type KeyType []byte + +type collapsedVertex struct { + children []*collapsedVertex + key KeyType + merkleHash *chainhash.Hash + claimHash *chainhash.Hash +} + +// insertAt inserts v into s at index i and returns the new slice. +// https://stackoverflow.com/questions/42746972/golang-insert-to-a-sorted-slice +func insertAt(data []*collapsedVertex, i int, v *collapsedVertex) []*collapsedVertex { + if i == len(data) { + // Insert at end is the easy case. + return append(data, v) + } + + // Make space for the inserted element by shifting + // values at the insertion index up one index. The call + // to append does not allocate memory when cap(data) is + // greater than len(data). + data = append(data[:i+1], data[i:]...) + data[i] = v + return data +} + +func (ptn *collapsedVertex) Insert(value *collapsedVertex) *collapsedVertex { + // keep it sorted (and sort.Sort is too slow) + index := sortSearch(ptn.children, value.key[0]) + ptn.children = insertAt(ptn.children, index, value) + + return value +} + +// this sort.Search is stolen shamelessly from search.go, +// and modified for performance to not need a closure +func sortSearch(nodes []*collapsedVertex, b byte) int { + i, j := 0, len(nodes) + for i < j { + h := int(uint(i+j) >> 1) // avoid overflow when computing h + // i ≤ h < j + if nodes[h].key[0] < b { + i = h + 1 // preserves f(i-1) == false + } else { + j = h // preserves f(j) == true + } + } + // i == j, f(i-1) == false, and f(j) (= f(i)) == true => answer is i. + return i +} + +func (ptn *collapsedVertex) findNearest(key KeyType) (int, *collapsedVertex) { + // none of the children overlap on the first char or we would have a parent node with that char + index := sortSearch(ptn.children, key[0]) + hits := ptn.children[index:] + if len(hits) > 0 { + return index, hits[0] + } + return -1, nil +} + +type collapsedTrie struct { + Root *collapsedVertex + Nodes int +} + +func NewCollapsedTrie() *collapsedTrie { + // we never delete the Root node + return &collapsedTrie{Root: &collapsedVertex{key: make(KeyType, 0)}, Nodes: 1} +} + +func (pt *collapsedTrie) NodeCount() int { + return pt.Nodes +} + +func matchLength(a, b KeyType) int { + minLen := len(a) + if len(b) < minLen { + minLen = len(b) + } + for i := 0; i < minLen; i++ { + if a[i] != b[i] { + return i + } + } + return minLen +} + +func (pt *collapsedTrie) insert(value KeyType, node *collapsedVertex) (bool, *collapsedVertex) { + index, child := node.findNearest(value) + match := 0 + if index >= 0 { // if we found a child + child.merkleHash = nil + match = matchLength(value, child.key) + if len(value) == match && len(child.key) == match { + return false, child + } + } + if match <= 0 { + pt.Nodes++ + return true, node.Insert(&collapsedVertex{key: value}) + } + if match < len(child.key) { + grandChild := collapsedVertex{key: child.key[match:], children: child.children, + claimHash: child.claimHash, merkleHash: child.merkleHash} + newChild := collapsedVertex{key: child.key[0:match], children: []*collapsedVertex{&grandChild}} + child = &newChild + node.children[index] = child + pt.Nodes++ + if len(value) == match { + return true, child + } + } + return pt.insert(value[match:], child) +} + +func (pt *collapsedTrie) InsertOrFind(value KeyType) (bool, *collapsedVertex) { + pt.Root.merkleHash = nil + if len(value) <= 0 { + return false, pt.Root + } + + // we store the name so we need to make our own copy of it + // this avoids errors where this function is called via the DB iterator + v2 := make([]byte, len(value)) + copy(v2, value) + return pt.insert(v2, pt.Root) +} + +func find(value KeyType, node *collapsedVertex, pathIndexes *[]int, path *[]*collapsedVertex) *collapsedVertex { + index, child := node.findNearest(value) + if index < 0 { + return nil + } + match := matchLength(value, child.key) + if len(value) == match && len(child.key) == match { + if pathIndexes != nil { + *pathIndexes = append(*pathIndexes, index) + } + if path != nil { + *path = append(*path, child) + } + return child + } + if match < len(child.key) || match == len(value) { + return nil + } + if pathIndexes != nil { + *pathIndexes = append(*pathIndexes, index) + } + if path != nil { + *path = append(*path, child) + } + return find(value[match:], child, pathIndexes, path) +} + +func (pt *collapsedTrie) Find(value KeyType) *collapsedVertex { + if len(value) <= 0 { + return pt.Root + } + return find(value, pt.Root, nil, nil) +} + +func (pt *collapsedTrie) FindPath(value KeyType) ([]int, []*collapsedVertex) { + pathIndexes := []int{-1} + path := []*collapsedVertex{pt.Root} + if len(value) > 0 { + result := find(value, pt.Root, &pathIndexes, &path) + if result == nil { // not sure I want this line + return nil, nil + } + } + return pathIndexes, path +} + +// IterateFrom can be used to find a value and run a function on that value. +// If the handler returns true it continues to iterate through the children of value. +func (pt *collapsedTrie) IterateFrom(start KeyType, handler func(name KeyType, value *collapsedVertex) bool) { + node := find(start, pt.Root, nil, nil) + if node == nil { + return + } + iterateFrom(start, node, handler) +} + +func iterateFrom(name KeyType, node *collapsedVertex, handler func(name KeyType, value *collapsedVertex) bool) { + for handler(name, node) { + for _, child := range node.children { + iterateFrom(append(name, child.key...), child, handler) + } + } +} + +func (pt *collapsedTrie) Erase(value KeyType) bool { + indexes, path := pt.FindPath(value) + if path == nil || len(path) <= 1 { + if len(path) == 1 { + path[0].merkleHash = nil + path[0].claimHash = nil + } + return false + } + nodes := pt.Nodes + i := len(path) - 1 + path[i].claimHash = nil // this is the thing we are erasing; the rest is book-keeping + for ; i > 0; i-- { + childCount := len(path[i].children) + noClaimData := path[i].claimHash == nil + path[i].merkleHash = nil + if childCount == 1 && noClaimData { + path[i].key = append(path[i].key, path[i].children[0].key...) + path[i].claimHash = path[i].children[0].claimHash + path[i].children = path[i].children[0].children + pt.Nodes-- + continue + } + if childCount == 0 && noClaimData { + index := indexes[i] + path[i-1].children = append(path[i-1].children[:index], path[i-1].children[index+1:]...) + pt.Nodes-- + continue + } + break + } + for ; i >= 0; i-- { + path[i].merkleHash = nil + } + return nodes > pt.Nodes +} diff --git a/claimtrie/merkletrie/collapsedtrie_test.go b/claimtrie/merkletrie/collapsedtrie_test.go new file mode 100644 index 00000000..ce41c35f --- /dev/null +++ b/claimtrie/merkletrie/collapsedtrie_test.go @@ -0,0 +1,113 @@ +package merkletrie + +import ( + "bytes" + "math/rand" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func b(value string) []byte { return []byte(value) } +func eq(x []byte, y string) bool { return bytes.Equal(x, b(y)) } + +func TestInsertAndErase(t *testing.T) { + trie := NewCollapsedTrie() + assert.True(t, trie.NodeCount() == 1) + inserted, node := trie.InsertOrFind(b("abc")) + assert.True(t, inserted) + assert.NotNil(t, node) + assert.Equal(t, 2, trie.NodeCount()) + inserted, node = trie.InsertOrFind(b("abd")) + assert.True(t, inserted) + assert.Equal(t, 4, trie.NodeCount()) + assert.NotNil(t, node) + hit := trie.Find(b("ab")) + assert.True(t, eq(hit.key, "ab")) + assert.Equal(t, 2, len(hit.children)) + hit = trie.Find(b("abc")) + assert.True(t, eq(hit.key, "c")) + hit = trie.Find(b("abd")) + assert.True(t, eq(hit.key, "d")) + hit = trie.Find(b("a")) + assert.Nil(t, hit) + indexes, path := trie.FindPath(b("abd")) + assert.Equal(t, 3, len(indexes)) + assert.True(t, eq(path[1].key, "ab")) + erased := trie.Erase(b("ab")) + assert.False(t, erased) + assert.Equal(t, 4, trie.NodeCount()) + erased = trie.Erase(b("abc")) + assert.True(t, erased) + assert.Equal(t, 2, trie.NodeCount()) + erased = trie.Erase(b("abd")) + assert.True(t, erased) + assert.Equal(t, 1, trie.NodeCount()) +} + +func TestNilNameHandling(t *testing.T) { + trie := NewCollapsedTrie() + inserted, n := trie.InsertOrFind([]byte("test")) + assert.True(t, inserted) + n.claimHash = EmptyTrieHash + inserted, n = trie.InsertOrFind(nil) + assert.False(t, inserted) + n.claimHash = EmptyTrieHash + n.merkleHash = EmptyTrieHash + inserted, n = trie.InsertOrFind(nil) + assert.False(t, inserted) + assert.NotNil(t, n.claimHash) + assert.Nil(t, n.merkleHash) + nodeRemoved := trie.Erase(nil) + assert.False(t, nodeRemoved) + inserted, n = trie.InsertOrFind(nil) + assert.False(t, inserted) + assert.Nil(t, n.claimHash) +} + +func TestCollapsedTriePerformance(t *testing.T) { + inserts := 100000 // increase this to 1M for more interesting results + data := make([][]byte, inserts) + rand.Seed(42) + for i := 0; i < inserts; i++ { + size := rand.Intn(70) + 4 + data[i] = make([]byte, size) + rand.Read(data[i]) + for j := 0; j < size; j++ { + data[i][j] %= byte(62) // shrink the range to match the old test + } + } + + trie := NewCollapsedTrie() + // doing my own timing because I couldn't get the B.Run method to work: + start := time.Now() + for i := 0; i < inserts; i++ { + _, node := trie.InsertOrFind(data[i]) + assert.NotNil(t, node, "Failure at %d of %d", i, inserts) + } + t.Logf("Insertion in %f sec.", time.Since(start).Seconds()) + + start = time.Now() + for i := 0; i < inserts; i++ { + node := trie.Find(data[i]) + assert.True(t, bytes.HasSuffix(data[i], node.key), "Failure on %d of %d", i, inserts) + } + t.Logf("Lookup in %f sec. on %d nodes.", time.Since(start).Seconds(), trie.NodeCount()) + + start = time.Now() + for i := 0; i < inserts; i++ { + indexes, path := trie.FindPath(data[i]) + assert.True(t, len(indexes) == len(path)) + assert.True(t, len(path) > 1) + assert.True(t, bytes.HasSuffix(data[i], path[len(path)-1].key)) + } + t.Logf("Parents in %f sec.", time.Since(start).Seconds()) + + start = time.Now() + for i := 0; i < inserts; i++ { + trie.Erase(data[i]) + } + t.Logf("Deletion in %f sec.", time.Since(start).Seconds()) + assert.Equal(t, 1, trie.NodeCount()) +} diff --git a/claimtrie/merkletrie/merkletrie.go b/claimtrie/merkletrie/merkletrie.go new file mode 100644 index 00000000..3bc525fe --- /dev/null +++ b/claimtrie/merkletrie/merkletrie.go @@ -0,0 +1,255 @@ +package merkletrie + +import ( + "bytes" + "fmt" + "runtime" + "sort" + "sync" + + "github.com/pkg/errors" + + "github.com/lbryio/lbcd/chaincfg/chainhash" + "github.com/lbryio/lbcd/claimtrie/node" +) + +var ( + // EmptyTrieHash represents the Merkle Hash of an empty PersistentTrie. + // "0000000000000000000000000000000000000000000000000000000000000001" + EmptyTrieHash = &chainhash.Hash{1} + NoChildrenHash = &chainhash.Hash{2} + NoClaimsHash = &chainhash.Hash{3} +) + +// PersistentTrie implements a 256-way prefix tree. +type PersistentTrie struct { + repo Repo + + root *vertex + bufs *sync.Pool +} + +// NewPersistentTrie returns a PersistentTrie. +func NewPersistentTrie(repo Repo) *PersistentTrie { + + tr := &PersistentTrie{ + repo: repo, + bufs: &sync.Pool{ + New: func() interface{} { + return new(bytes.Buffer) + }, + }, + root: newVertex(EmptyTrieHash), + } + + return tr +} + +// SetRoot drops all resolved nodes in the PersistentTrie, and set the Root with specified hash. +func (t *PersistentTrie) SetRoot(h *chainhash.Hash) error { + t.root = newVertex(h) + runtime.GC() + return nil +} + +// Update updates the nodes along the path to the key. +// Each node is resolved or created with their Hash cleared. +func (t *PersistentTrie) Update(name []byte, hash *chainhash.Hash, restoreChildren bool) { + + n := t.root + for i, ch := range name { + if restoreChildren && len(n.childLinks) == 0 { + t.resolveChildLinks(n, name[:i]) + } + if n.childLinks[ch] == nil { + n.childLinks[ch] = newVertex(nil) + } + n.merkleHash = nil + n = n.childLinks[ch] + } + + if restoreChildren && len(n.childLinks) == 0 { + t.resolveChildLinks(n, name) + } + n.merkleHash = nil + n.claimsHash = hash +} + +// resolveChildLinks updates the links on n +func (t *PersistentTrie) resolveChildLinks(n *vertex, key []byte) { + + if n.merkleHash == nil { + return + } + + b := t.bufs.Get().(*bytes.Buffer) + defer t.bufs.Put(b) + b.Reset() + b.Write(key) + b.Write(n.merkleHash[:]) + + result, closer, err := t.repo.Get(b.Bytes()) + if result == nil { + return + } else if err != nil { + panic(err) + } + defer closer.Close() + + nb := nbuf(result) + _, n.claimsHash = nb.hasValue() + for i := 0; i < nb.entries(); i++ { + p, h := nb.entry(i) + n.childLinks[p] = newVertex(h) + } +} + +// MerkleHash returns the Merkle Hash of the PersistentTrie. +// All nodes must have been resolved before calling this function. +func (t *PersistentTrie) MerkleHash() *chainhash.Hash { + buf := make([]byte, 0, 256) + if h := t.merkle(buf, t.root); h == nil { + return EmptyTrieHash + } + return t.root.merkleHash +} + +// merkle recursively resolves the hashes of the node. +// All nodes must have been resolved before calling this function. +func (t *PersistentTrie) merkle(prefix []byte, v *vertex) *chainhash.Hash { + if v.merkleHash != nil { + return v.merkleHash + } + + b := t.bufs.Get().(*bytes.Buffer) + defer t.bufs.Put(b) + b.Reset() + + keys := keysInOrder(v) + + for _, ch := range keys { + child := v.childLinks[ch] + if child == nil { + continue + } + p := append(prefix, ch) + h := t.merkle(p, child) + if h != nil { + b.WriteByte(ch) // nolint : errchk + b.Write(h[:]) // nolint : errchk + } + if h == nil || len(prefix) > 4 { // TODO: determine the right number here + delete(v.childLinks, ch) // keep the RAM down (they get recreated on Update) + } + } + + if v.claimsHash != nil { + b.Write(v.claimsHash[:]) + } + + if b.Len() > 0 { + h := chainhash.DoubleHashH(b.Bytes()) + v.merkleHash = &h + t.repo.Set(append(prefix, h[:]...), b.Bytes()) + } + + return v.merkleHash +} + +func keysInOrder(v *vertex) []byte { + keys := make([]byte, 0, len(v.childLinks)) + for key := range v.childLinks { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) + return keys +} + +func (t *PersistentTrie) MerkleHashAllClaims() *chainhash.Hash { + buf := make([]byte, 0, 256) + if h := t.merkleAllClaims(buf, t.root); h == nil { + return EmptyTrieHash + } + return t.root.merkleHash +} + +func (t *PersistentTrie) merkleAllClaims(prefix []byte, v *vertex) *chainhash.Hash { + if v.merkleHash != nil { + return v.merkleHash + } + b := t.bufs.Get().(*bytes.Buffer) + defer t.bufs.Put(b) + b.Reset() + + keys := keysInOrder(v) + childHashes := make([]*chainhash.Hash, 0, len(keys)) + for _, ch := range keys { + n := v.childLinks[ch] + if n == nil { + continue + } + p := append(prefix, ch) + h := t.merkleAllClaims(p, n) + if h != nil { + childHashes = append(childHashes, h) + b.WriteByte(ch) // nolint : errchk + b.Write(h[:]) // nolint : errchk + } + if h == nil || len(prefix) > 4 { // TODO: determine the right number here + delete(v.childLinks, ch) // keep the RAM down (they get recreated on Update) + } + } + + if len(childHashes) > 1 || v.claimsHash != nil { // yeah, about that 1 there -- old code used the condensed trie + left := NoChildrenHash + if len(childHashes) > 0 { + left = node.ComputeMerkleRoot(childHashes) + } + right := NoClaimsHash + if v.claimsHash != nil { + b.Write(v.claimsHash[:]) // for Has Value, nolint : errchk + right = v.claimsHash + } + + h := node.HashMerkleBranches(left, right) + v.merkleHash = h + t.repo.Set(append(prefix, h[:]...), b.Bytes()) + } else if len(childHashes) == 1 { + v.merkleHash = childHashes[0] // pass it up the tree + t.repo.Set(append(prefix, v.merkleHash[:]...), b.Bytes()) + } + + return v.merkleHash +} + +func (t *PersistentTrie) Close() error { + return errors.WithStack(t.repo.Close()) +} + +func (t *PersistentTrie) Dump(s string) { + // TODO: this function is in the wrong spot; either it goes with its caller or it needs to be a generic iterator + // we don't want fmt used in here either way + + v := t.root + + for i := 0; i < len(s); i++ { + t.resolveChildLinks(v, []byte(s[:i])) + ch := s[i] + v = v.childLinks[ch] + if v == nil { + fmt.Printf("Missing child at %s\n", s[:i+1]) + return + } + } + t.resolveChildLinks(v, []byte(s)) + + fmt.Printf("Node hash: %s, has value: %t\n", v.merkleHash.String(), v.claimsHash != nil) + + for key, value := range v.childLinks { + fmt.Printf(" Child %s hash: %s\n", string(key), value.merkleHash.String()) + } +} + +func (t *PersistentTrie) Flush() error { + return t.repo.Flush() +} diff --git a/claimtrie/merkletrie/merkletrie_test.go b/claimtrie/merkletrie/merkletrie_test.go new file mode 100644 index 00000000..fc95a7b4 --- /dev/null +++ b/claimtrie/merkletrie/merkletrie_test.go @@ -0,0 +1,25 @@ +package merkletrie + +import ( + "testing" + + "github.com/lbryio/lbcd/chaincfg/chainhash" + "github.com/lbryio/lbcd/claimtrie/node" + + "github.com/stretchr/testify/require" +) + +func TestName(t *testing.T) { + + r := require.New(t) + + target, _ := chainhash.NewHashFromStr("e9ffb584c62449f157c8be88257bd1eebb2d8ef824f5c86b43c4f8fd9e800d6a") + + data := []*chainhash.Hash{EmptyTrieHash} + root := node.ComputeMerkleRoot(data) + r.True(EmptyTrieHash.IsEqual(root)) + + data = append(data, NoChildrenHash, NoClaimsHash) + root = node.ComputeMerkleRoot(data) + r.True(target.IsEqual(root)) +} diff --git a/claimtrie/merkletrie/merkletrierepo/pebble.go b/claimtrie/merkletrie/merkletrierepo/pebble.go new file mode 100644 index 00000000..c903794e --- /dev/null +++ b/claimtrie/merkletrie/merkletrierepo/pebble.go @@ -0,0 +1,67 @@ +package merkletrierepo + +import ( + "io" + + "github.com/cockroachdb/pebble" + "github.com/pkg/errors" +) + +type Pebble struct { + db *pebble.DB +} + +func NewPebble(path string) (*Pebble, error) { + + cache := pebble.NewCache(512 << 20) + //defer cache.Unref() + // + //go func() { + // tick := time.NewTicker(60 * time.Second) + // for range tick.C { + // + // m := cache.Metrics() + // fmt.Printf("cnt: %s, objs: %s, hits: %s, miss: %s, hitrate: %.2f\n", + // humanize.Bytes(uint64(m.Size)), + // humanize.Comma(m.Count), + // humanize.Comma(m.Hits), + // humanize.Comma(m.Misses), + // float64(m.Hits)/float64(m.Hits+m.Misses)) + // + // } + //}() + + db, err := pebble.Open(path, &pebble.Options{Cache: cache, BytesPerSync: 32 << 20, MaxOpenFiles: 2000}) + repo := &Pebble{db: db} + + return repo, errors.Wrapf(err, "unable to open %s", path) +} + +func (repo *Pebble) Get(key []byte) ([]byte, io.Closer, error) { + d, c, e := repo.db.Get(key) + if e == pebble.ErrNotFound { + return nil, c, nil + } + return d, c, e +} + +func (repo *Pebble) Set(key, value []byte) error { + return repo.db.Set(key, value, pebble.NoSync) +} + +func (repo *Pebble) Close() error { + + err := repo.db.Flush() + if err != nil { + // if we fail to close are we going to try again later? + return errors.Wrap(err, "on flush") + } + + err = repo.db.Close() + return errors.Wrap(err, "on close") +} + +func (repo *Pebble) Flush() error { + _, err := repo.db.AsyncFlush() + return err +} diff --git a/claimtrie/merkletrie/ramtrie.go b/claimtrie/merkletrie/ramtrie.go new file mode 100644 index 00000000..7b426655 --- /dev/null +++ b/claimtrie/merkletrie/ramtrie.go @@ -0,0 +1,139 @@ +package merkletrie + +import ( + "bytes" + "errors" + "runtime" + "sync" + + "github.com/lbryio/lbcd/chaincfg/chainhash" + "github.com/lbryio/lbcd/claimtrie/node" +) + +type MerkleTrie interface { + SetRoot(h *chainhash.Hash) error + Update(name []byte, h *chainhash.Hash, restoreChildren bool) + MerkleHash() *chainhash.Hash + MerkleHashAllClaims() *chainhash.Hash + Flush() error +} + +type RamTrie struct { + collapsedTrie + bufs *sync.Pool +} + +func NewRamTrie() *RamTrie { + return &RamTrie{ + bufs: &sync.Pool{ + New: func() interface{} { + return new(bytes.Buffer) + }, + }, + collapsedTrie: collapsedTrie{Root: &collapsedVertex{merkleHash: EmptyTrieHash}}, + } +} + +var ErrFullRebuildRequired = errors.New("a full rebuild is required") + +func (rt *RamTrie) SetRoot(h *chainhash.Hash) error { + if rt.Root.merkleHash.IsEqual(h) { + runtime.GC() + return nil + } + + // should technically clear the old trie first, but this is abused for partial rebuilds so don't + return ErrFullRebuildRequired +} + +func (rt *RamTrie) Update(name []byte, h *chainhash.Hash, _ bool) { + if h == nil { + rt.Erase(name) + } else { + _, n := rt.InsertOrFind(name) + n.claimHash = h + } +} + +func (rt *RamTrie) MerkleHash() *chainhash.Hash { + if h := rt.merkleHash(rt.Root); h == nil { + return EmptyTrieHash + } + return rt.Root.merkleHash +} + +func (rt *RamTrie) merkleHash(v *collapsedVertex) *chainhash.Hash { + if v.merkleHash != nil { + return v.merkleHash + } + + b := rt.bufs.Get().(*bytes.Buffer) + defer rt.bufs.Put(b) + b.Reset() + + for _, ch := range v.children { + h := rt.merkleHash(ch) // h is a pointer; don't destroy its data + b.WriteByte(ch.key[0]) // nolint : errchk + b.Write(rt.completeHash(h, ch.key)) // nolint : errchk + } + + if v.claimHash != nil { + b.Write(v.claimHash[:]) + } + + if b.Len() > 0 { + h := chainhash.DoubleHashH(b.Bytes()) + v.merkleHash = &h + } + + return v.merkleHash +} + +func (rt *RamTrie) completeHash(h *chainhash.Hash, childKey KeyType) []byte { + var data [chainhash.HashSize + 1]byte + copy(data[1:], h[:]) + for i := len(childKey) - 1; i > 0; i-- { + data[0] = childKey[i] + copy(data[1:], chainhash.DoubleHashB(data[:])) + } + return data[1:] +} + +func (rt *RamTrie) MerkleHashAllClaims() *chainhash.Hash { + if h := rt.merkleHashAllClaims(rt.Root); h == nil { + return EmptyTrieHash + } + return rt.Root.merkleHash +} + +func (rt *RamTrie) merkleHashAllClaims(v *collapsedVertex) *chainhash.Hash { + if v.merkleHash != nil { + return v.merkleHash + } + + childHashes := make([]*chainhash.Hash, 0, len(v.children)) + for _, ch := range v.children { + h := rt.merkleHashAllClaims(ch) + childHashes = append(childHashes, h) + } + + claimHash := NoClaimsHash + if v.claimHash != nil { + claimHash = v.claimHash + } else if len(childHashes) == 0 { + return nil + } + + childHash := NoChildrenHash + if len(childHashes) > 0 { + // this shouldn't be referencing node; where else can we put this merkle root func? + childHash = node.ComputeMerkleRoot(childHashes) + } + + v.merkleHash = node.HashMerkleBranches(childHash, claimHash) + return v.merkleHash +} + +func (rt *RamTrie) Flush() error { + return nil +} diff --git a/claimtrie/merkletrie/repo.go b/claimtrie/merkletrie/repo.go new file mode 100644 index 00000000..68b6c8d6 --- /dev/null +++ b/claimtrie/merkletrie/repo.go @@ -0,0 +1,13 @@ +package merkletrie + +import ( + "io" +) + +// Repo defines APIs for PersistentTrie to access persistence layer. +type Repo interface { + Get(key []byte) ([]byte, io.Closer, error) + Set(key, value []byte) error + Close() error + Flush() error +} diff --git a/claimtrie/merkletrie/vertex.go b/claimtrie/merkletrie/vertex.go new file mode 100644 index 00000000..77f1f04a --- /dev/null +++ b/claimtrie/merkletrie/vertex.go @@ -0,0 +1,43 @@ +package merkletrie + +import ( + "github.com/lbryio/lbcd/chaincfg/chainhash" +) + +type vertex struct { + merkleHash *chainhash.Hash + claimsHash *chainhash.Hash + childLinks map[byte]*vertex +} + +func newVertex(hash *chainhash.Hash) *vertex { + return &vertex{childLinks: map[byte]*vertex{}, merkleHash: hash} +} + +// TODO: more professional to use msgpack here? + +// nbuf decodes the on-disk format of a node, which has the following form: +// ch(1B) hash(32B) +// ... +// ch(1B) hash(32B) +// vhash(32B) +type nbuf []byte + +func (nb nbuf) entries() int { + return len(nb) / 33 +} + +func (nb nbuf) entry(i int) (byte, *chainhash.Hash) { + h := chainhash.Hash{} + copy(h[:], nb[33*i+1:]) + return nb[33*i], &h +} + +func (nb nbuf) hasValue() (bool, *chainhash.Hash) { + if len(nb)%33 == 0 { + return false, nil + } + h := chainhash.Hash{} + copy(h[:], nb[len(nb)-32:]) + return true, &h +} diff --git a/claimtrie/node/claim.go b/claimtrie/node/claim.go new file mode 100644 index 00000000..8f385913 --- /dev/null +++ b/claimtrie/node/claim.go @@ -0,0 +1,92 @@ +package node + +import ( + "bytes" + "strconv" + "strings" + + "github.com/lbryio/lbcd/chaincfg/chainhash" + "github.com/lbryio/lbcd/claimtrie/change" + "github.com/lbryio/lbcd/claimtrie/param" + "github.com/lbryio/lbcd/wire" +) + +type Status int + +const ( + Accepted Status = iota + Activated + Deactivated +) + +// Claim defines a structure of stake, which could be a Claim or Support. +type Claim struct { + OutPoint wire.OutPoint + ClaimID change.ClaimID + Amount int64 + // CreatedAt int32 // the very first block, unused at present + AcceptedAt int32 // the latest update height + ActiveAt int32 // AcceptedAt + actual delay + VisibleAt int32 + + Status Status `msgpack:",omitempty"` + Sequence int32 `msgpack:",omitempty"` +} + +func (c *Claim) setOutPoint(op wire.OutPoint) *Claim { + c.OutPoint = op + return c +} + +func (c *Claim) SetAmt(amt int64) *Claim { + c.Amount = amt + return c +} + +func (c *Claim) setAccepted(height int32) *Claim { + c.AcceptedAt = height + return c +} + +func (c *Claim) setActiveAt(height int32) *Claim { + c.ActiveAt = height + return c +} + +func (c *Claim) setStatus(status Status) *Claim { + c.Status = status + return c +} + +func (c *Claim) ExpireAt() int32 { + + if c.AcceptedAt+param.ActiveParams.OriginalClaimExpirationTime > param.ActiveParams.ExtendedClaimExpirationForkHeight { + return c.AcceptedAt + param.ActiveParams.ExtendedClaimExpirationTime + } + + return c.AcceptedAt + param.ActiveParams.OriginalClaimExpirationTime +} + +func OutPointLess(a, b wire.OutPoint) bool { + + switch cmp := bytes.Compare(a.Hash[:], b.Hash[:]); { + case cmp < 0: + return true + case cmp > 0: + return false + default: + return a.Index < b.Index + } +} + +func NewOutPointFromString(str string) *wire.OutPoint { + + f := strings.Split(str, ":") + if len(f) != 2 { + return nil + } + hash, _ := chainhash.NewHashFromStr(f[0]) + idx, _ := strconv.Atoi(f[1]) + + return wire.NewOutPoint(hash, uint32(idx)) +} diff --git a/claimtrie/node/claim_list.go b/claimtrie/node/claim_list.go new file mode 100644 index 00000000..007a1b6b --- /dev/null +++ b/claimtrie/node/claim_list.go @@ -0,0 +1,33 @@ +package node + +import ( + "github.com/lbryio/lbcd/claimtrie/change" + "github.com/lbryio/lbcd/wire" +) + +type ClaimList []*Claim + +type comparator func(c *Claim) bool + +func byID(id change.ClaimID) comparator { + return func(c *Claim) bool { + return c.ClaimID == id + } +} + +func byOut(out wire.OutPoint) comparator { + return func(c *Claim) bool { + return c.OutPoint == out // assuming value comparison + } +} + +func (l ClaimList) find(cmp comparator) *Claim { + + for i := range l { + if cmp(l[i]) { + return l[i] + } + } + + return nil +} diff --git a/claimtrie/node/hashfork_manager.go b/claimtrie/node/hashfork_manager.go new file mode 100644 index 00000000..bbd814ee --- /dev/null +++ b/claimtrie/node/hashfork_manager.go @@ -0,0 +1,39 @@ +package node + +import ( + "github.com/lbryio/lbcd/chaincfg/chainhash" + "github.com/lbryio/lbcd/claimtrie/param" +) + +type HashV2Manager struct { + Manager +} + +func (nm *HashV2Manager) computeClaimHashes(name []byte) (*chainhash.Hash, int32) { + + n, err := nm.NodeAt(nm.Height(), name) + if err != nil || n == nil { + return nil, 0 + } + + n.SortClaimsByBid() + claimHashes := make([]*chainhash.Hash, 0, len(n.Claims)) + for _, c := range n.Claims { + if c.Status == Activated { // TODO: unit test this line + claimHashes = append(claimHashes, calculateNodeHash(c.OutPoint, n.TakenOverAt)) + } + } + if len(claimHashes) > 0 { + return ComputeMerkleRoot(claimHashes), n.NextUpdate() + } + return nil, n.NextUpdate() +} + +func (nm *HashV2Manager) Hash(name []byte) (*chainhash.Hash, int32) { + + if nm.Height() >= param.ActiveParams.AllClaimsInMerkleForkHeight { + return nm.computeClaimHashes(name) + } + + return nm.Manager.Hash(name) +} diff --git a/claimtrie/node/hashfunc.go b/claimtrie/node/hashfunc.go new file mode 100644 index 00000000..deec78bb --- /dev/null +++ b/claimtrie/node/hashfunc.go @@ -0,0 +1,57 @@ +package node + +import ( + "crypto/sha256" + "encoding/binary" + "strconv" + + "github.com/lbryio/lbcd/chaincfg/chainhash" + "github.com/lbryio/lbcd/wire" +) + +func HashMerkleBranches(left *chainhash.Hash, right *chainhash.Hash) *chainhash.Hash { + // Concatenate the left and right nodes. + var hash [chainhash.HashSize * 2]byte + copy(hash[:chainhash.HashSize], left[:]) + copy(hash[chainhash.HashSize:], right[:]) + + newHash := chainhash.DoubleHashH(hash[:]) + return &newHash +} + +func ComputeMerkleRoot(hashes []*chainhash.Hash) *chainhash.Hash { + if len(hashes) <= 0 { + return nil + } + for len(hashes) > 1 { + if (len(hashes) & 1) > 0 { // odd count + hashes = append(hashes, hashes[len(hashes)-1]) + } + for i := 0; i < len(hashes); i += 2 { // TODO: parallelize this loop (or use a lib that does it) + hashes[i>>1] = HashMerkleBranches(hashes[i], hashes[i+1]) + } + hashes = hashes[:len(hashes)>>1] + } + return hashes[0] +} + +func calculateNodeHash(op wire.OutPoint, takeover int32) *chainhash.Hash { + + txHash := chainhash.DoubleHashH(op.Hash[:]) + + nOut := []byte(strconv.Itoa(int(op.Index))) + nOutHash := chainhash.DoubleHashH(nOut) + + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, uint64(takeover)) + heightHash := chainhash.DoubleHashH(buf) + + h := make([]byte, 0, sha256.Size*3) + h = append(h, txHash[:]...) + h = append(h, nOutHash[:]...) + h = append(h, heightHash[:]...) + + hh := chainhash.DoubleHashH(h) + + return &hh +} diff --git a/claimtrie/node/log.go b/claimtrie/node/log.go new file mode 100644 index 00000000..86293b58 --- /dev/null +++ b/claimtrie/node/log.go @@ -0,0 +1,47 @@ +package node + +import ( + "sync" + + "github.com/btcsuite/btclog" +) + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log btclog.Logger + +// The default amount of logging is none. +func init() { + DisableLog() +} + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = btclog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using btclog. +func UseLogger(logger btclog.Logger) { + log = logger +} + +var loggedStrings = map[string]bool{} // is this gonna get too large? +var loggedStringsMutex sync.Mutex + +func LogOnce(s string) { + loggedStringsMutex.Lock() + defer loggedStringsMutex.Unlock() + if loggedStrings[s] { + return + } + loggedStrings[s] = true + log.Info(s) +} + +func Warn(s string) { + log.Warn(s) +} diff --git a/claimtrie/node/manager.go b/claimtrie/node/manager.go new file mode 100644 index 00000000..605b7123 --- /dev/null +++ b/claimtrie/node/manager.go @@ -0,0 +1,374 @@ +package node + +import ( + "fmt" + "sort" + + "github.com/pkg/errors" + + "github.com/lbryio/lbcd/chaincfg/chainhash" + "github.com/lbryio/lbcd/claimtrie/change" + "github.com/lbryio/lbcd/claimtrie/param" +) + +type Manager interface { + AppendChange(chg change.Change) + IncrementHeightTo(height int32) ([][]byte, error) + DecrementHeightTo(affectedNames [][]byte, height int32) error + Height() int32 + Close() error + NodeAt(height int32, name []byte) (*Node, error) + IterateNames(predicate func(name []byte) bool) + Hash(name []byte) (*chainhash.Hash, int32) + Flush() error +} + +type BaseManager struct { + repo Repo + + height int32 + changes []change.Change +} + +func NewBaseManager(repo Repo) (*BaseManager, error) { + + nm := &BaseManager{ + repo: repo, + } + + return nm, nil +} + +func (nm *BaseManager) NodeAt(height int32, name []byte) (*Node, error) { + + changes, err := nm.repo.LoadChanges(name) + if err != nil { + return nil, errors.Wrap(err, "in load changes") + } + + n, err := nm.newNodeFromChanges(changes, height) + if err != nil { + return nil, errors.Wrap(err, "in new node") + } + + return n, nil +} + +// Node returns a node at the current height. +// The returned node may have pending changes. +func (nm *BaseManager) node(name []byte) (*Node, error) { + return nm.NodeAt(nm.height, name) +} + +// newNodeFromChanges returns a new Node constructed from the changes. +// The changes must preserve their order received. +func (nm *BaseManager) newNodeFromChanges(changes []change.Change, height int32) (*Node, error) { + + if len(changes) == 0 { + return nil, nil + } + + n := New() + previous := changes[0].Height + count := len(changes) + + for i, chg := range changes { + if chg.Height < previous { + panic("expected the changes to be in order by height") + } + if chg.Height > height { + count = i + break + } + + if previous < chg.Height { + n.AdjustTo(previous, chg.Height-1, chg.Name) // update bids and activation + previous = chg.Height + } + + delay := nm.getDelayForName(n, chg) + err := n.ApplyChange(chg, delay) + if err != nil { + return nil, errors.Wrap(err, "in apply change") + } + } + + if count <= 0 { + return nil, nil + } + lastChange := changes[count-1] + return n.AdjustTo(lastChange.Height, height, lastChange.Name), nil +} + +func (nm *BaseManager) AppendChange(chg change.Change) { + + nm.changes = append(nm.changes, chg) + + // worth putting in this kind of thing pre-emptively? + // log.Debugf("CHG: %d, %s, %v, %s, %d", chg.Height, chg.Name, chg.Type, chg.ClaimID, chg.Amount) +} + +func collectChildNames(changes []change.Change) { + // we need to determine which children (names that start with the same name) go with which change + // if we have the names in order then we can avoid iterating through all names in the change list + // and we can possibly reuse the previous list. + + // what would happen in the old code: + // spending a claim (which happens before every update) could remove a node from the cached trie + // in which case we would fall back on the data from the previous block (where it obviously wasn't spent). + // It would only delete the node if it had no children, but have even some rare situations + // Where all of the children happen to be deleted first. That's what we must detect here. + + // Algorithm: + // For each non-spend change + // Loop through all the spends before you and add them to your child list if they are your child + + type pair struct { + name string + order int + } + + spends := make([]pair, 0, len(changes)) + for i := range changes { + t := changes[i].Type + if t != change.SpendClaim { + continue + } + spends = append(spends, pair{string(changes[i].Name), i}) + } + sort.Slice(spends, func(i, j int) bool { + return spends[i].name < spends[j].name + }) + + for i := range changes { + t := changes[i].Type + if t == change.SpendClaim || t == change.SpendSupport { + continue + } + a := string(changes[i].Name) + sc := map[string]bool{} + idx := sort.Search(len(spends), func(k int) bool { + return spends[k].name > a + }) + for idx < len(spends) { + b := spends[idx].name + if len(b) <= len(a) || a != b[:len(a)] { + break // since they're ordered alphabetically, we should be able to break out once we're past matches + } + if spends[idx].order < i { + sc[b] = true + } + idx++ + } + changes[i].SpentChildren = sc + } +} + +// to understand the above function, it may be helpful to refer to the slower implementation: +//func collectChildNamesSlow(changes []change.Change) { +// for i := range changes { +// t := changes[i].Type +// if t == change.SpendClaim || t == change.SpendSupport { +// continue +// } +// a := changes[i].Name +// sc := map[string]bool{} +// for j := 0; j < i; j++ { +// t = changes[j].Type +// if t != change.SpendClaim { +// continue +// } +// b := changes[j].Name +// if len(b) >= len(a) && bytes.Equal(a, b[:len(a)]) { +// sc[string(b)] = true +// } +// } +// changes[i].SpentChildren = sc +// } +//} + +func (nm *BaseManager) IncrementHeightTo(height int32) ([][]byte, error) { + + if height <= nm.height { + panic("invalid height") + } + + if height >= param.ActiveParams.MaxRemovalWorkaroundHeight { + // not technically needed until block 884430, but to be true to the arbitrary rollback length... + collectChildNames(nm.changes) + } + + names := make([][]byte, 0, len(nm.changes)) + for i := range nm.changes { + names = append(names, nm.changes[i].Name) + } + + if err := nm.repo.AppendChanges(nm.changes); err != nil { // destroys names + return nil, errors.Wrap(err, "in append changes") + } + + // Truncate the buffer size to zero. + if len(nm.changes) > 1000 { // TODO: determine a good number here + nm.changes = nil // release the RAM + } else { + nm.changes = nm.changes[:0] + } + nm.height = height + + return names, nil +} + +func (nm *BaseManager) DecrementHeightTo(affectedNames [][]byte, height int32) error { + if height >= nm.height { + return errors.Errorf("invalid height of %d for %d", height, nm.height) + } + + for _, name := range affectedNames { + if err := nm.repo.DropChanges(name, height); err != nil { + return errors.Wrap(err, "in drop changes") + } + } + + nm.height = height + + return nil +} + +func (nm *BaseManager) getDelayForName(n *Node, chg change.Change) int32 { + // Note: we don't consider the active status of BestClaim here on purpose. + // That's because we deactivate and reactivate as part of claim updates. + // However, the final status will be accounted for when we compute the takeover heights; + // claims may get activated early at that point. + + hasBest := n.BestClaim != nil + if hasBest && n.BestClaim.ClaimID == chg.ClaimID { + return 0 + } + if chg.ActiveHeight >= chg.Height { // ActiveHeight is usually unset (aka, zero) + return chg.ActiveHeight - chg.Height + } + if !hasBest { + return 0 + } + + delay := calculateDelay(chg.Height, n.TakenOverAt) + if delay > 0 && nm.aWorkaroundIsNeeded(n, chg) { + if chg.Height >= nm.height { + LogOnce(fmt.Sprintf("Delay workaround applies to %s at %d, ClaimID: %s", + chg.Name, chg.Height, chg.ClaimID)) + } + return 0 + } + return delay +} + +func hasZeroActiveClaims(n *Node) bool { + // this isn't quite the same as having an active best (since that is only updated after all changes are processed) + for _, c := range n.Claims { + if c.Status == Activated { + return false + } + } + return true +} + +// aWorkaroundIsNeeded handles bugs that existed in previous versions +func (nm *BaseManager) aWorkaroundIsNeeded(n *Node, chg change.Change) bool { + + if chg.Type == change.SpendClaim || chg.Type == change.SpendSupport { + return false + } + + if chg.Height >= param.ActiveParams.MaxRemovalWorkaroundHeight { + // TODO: hard fork this out; it's a bug from previous versions: + + // old 17.3 C++ code we're trying to mimic (where empty means no active claims): + // auto it = nodesToAddOrUpdate.find(name); // nodesToAddOrUpdate is the working changes, base is previous block + // auto answer = (it || (it = base->find(name))) && !it->empty() ? nNextHeight - it->nHeightOfLastTakeover : 0; + + return hasZeroActiveClaims(n) && nm.hasChildren(chg.Name, chg.Height, chg.SpentChildren, 2) + } else if len(n.Claims) > 0 { + // NOTE: old code had a bug in it where nodes with no claims but with children would get left in the cache after removal. + // This would cause the getNumBlocksOfContinuousOwnership to return zero (causing incorrect takeover height calc). + w, ok := param.DelayWorkarounds[string(chg.Name)] + if ok { + for _, h := range w { + if chg.Height == h { + return true + } + } + } + } + return false +} + +func calculateDelay(curr, tookOver int32) int32 { + + delay := (curr - tookOver) / param.ActiveParams.ActiveDelayFactor + if delay > param.ActiveParams.MaxActiveDelay { + return param.ActiveParams.MaxActiveDelay + } + + return delay +} + +func (nm *BaseManager) Height() int32 { + return nm.height +} + +func (nm *BaseManager) Close() error { + return errors.WithStack(nm.repo.Close()) +} + +func (nm *BaseManager) hasChildren(name []byte, height int32, spentChildren map[string]bool, required int) bool { + c := map[byte]bool{} + if spentChildren == nil { + spentChildren = map[string]bool{} + } + + err := nm.repo.IterateChildren(name, func(changes []change.Change) bool { + // if the key is unseen, generate a node for it to height + // if that node is active then increase the count + if len(changes) == 0 { + return true + } + if c[changes[0].Name[len(name)]] { // assuming all names here are longer than starter name + return true // we already checked a similar name + } + if spentChildren[string(changes[0].Name)] { + return true // children that are spent in the same block cannot count as active children + } + n, _ := nm.newNodeFromChanges(changes, height) + if n != nil && n.HasActiveBestClaim() { + c[changes[0].Name[len(name)]] = true + if len(c) >= required { + return false + } + } + return true + }) + return err == nil && len(c) >= required +} + +func (nm *BaseManager) IterateNames(predicate func(name []byte) bool) { + nm.repo.IterateAll(predicate) +} + +func (nm *BaseManager) Hash(name []byte) (*chainhash.Hash, int32) { + + n, err := nm.node(name) + if err != nil || n == nil { + return nil, 0 + } + if len(n.Claims) > 0 { + if n.BestClaim != nil && n.BestClaim.Status == Activated { + h := calculateNodeHash(n.BestClaim.OutPoint, n.TakenOverAt) + return h, n.NextUpdate() + } + } + return nil, n.NextUpdate() +} + +func (nm *BaseManager) Flush() error { + return nm.repo.Flush() +} diff --git a/claimtrie/node/manager_test.go b/claimtrie/node/manager_test.go new file mode 100644 index 00000000..0f5ca93e --- /dev/null +++ b/claimtrie/node/manager_test.go @@ -0,0 +1,249 @@ +package node + +import ( + "fmt" + "testing" + + "github.com/lbryio/lbcd/claimtrie/change" + "github.com/lbryio/lbcd/claimtrie/node/noderepo" + "github.com/lbryio/lbcd/claimtrie/param" + "github.com/lbryio/lbcd/wire" + + "github.com/stretchr/testify/require" +) + +var ( + out1 = NewOutPointFromString("0000000000000000000000000000000000000000000000000000000000000000:1") + out2 = NewOutPointFromString("0000000000000000000000000000000000000000000000000000000000000000:2") + out3 = NewOutPointFromString("0100000000000000000000000000000000000000000000000000000000000000:1") + out4 = NewOutPointFromString("0100000000000000000000000000000000000000000000000000000000000000:2") + name1 = []byte("name1") + name2 = []byte("name2") +) + +// verify that we can round-trip bytes to strings +func TestStringRoundTrip(t *testing.T) { + + r := require.New(t) + + data := [][]byte{ + {97, 98, 99, 0, 100, 255}, + {0xc3, 0x28}, + {0xa0, 0xa1}, + {0xe2, 0x28, 0xa1}, + {0xf0, 0x28, 0x8c, 0x28}, + } + for _, d := range data { + s := string(d) + r.Equal(s, fmt.Sprintf("%s", d)) // nolint + d2 := []byte(s) + r.Equal(len(d), len(s)) + r.Equal(d, d2) + } +} + +func TestSimpleAddClaim(t *testing.T) { + + r := require.New(t) + + param.SetNetwork(wire.TestNet) + repo, err := noderepo.NewPebble(t.TempDir()) + r.NoError(err) + + m, err := NewBaseManager(repo) + r.NoError(err) + defer m.Close() + + _, err = m.IncrementHeightTo(10) + r.NoError(err) + + chg := change.NewChange(change.AddClaim).SetName(name1).SetOutPoint(out1).SetHeight(11) + m.AppendChange(chg) + _, err = m.IncrementHeightTo(11) + r.NoError(err) + + chg = chg.SetName(name2).SetOutPoint(out2).SetHeight(12) + m.AppendChange(chg) + _, err = m.IncrementHeightTo(12) + r.NoError(err) + + n1, err := m.node(name1) + r.NoError(err) + r.Equal(1, len(n1.Claims)) + r.NotNil(n1.Claims.find(byOut(*out1))) + + n2, err := m.node(name2) + r.NoError(err) + r.Equal(1, len(n2.Claims)) + r.NotNil(n2.Claims.find(byOut(*out2))) + + err = m.DecrementHeightTo([][]byte{name2}, 11) + r.NoError(err) + n2, err = m.node(name2) + r.NoError(err) + r.Nil(n2) + + err = m.DecrementHeightTo([][]byte{name1}, 1) + r.NoError(err) + n2, err = m.node(name1) + r.NoError(err) + r.Nil(n2) +} + +func TestSupportAmounts(t *testing.T) { + + r := require.New(t) + + param.SetNetwork(wire.TestNet) + repo, err := noderepo.NewPebble(t.TempDir()) + r.NoError(err) + + m, err := NewBaseManager(repo) + r.NoError(err) + defer m.Close() + + _, err = m.IncrementHeightTo(10) + r.NoError(err) + + chg := change.NewChange(change.AddClaim).SetName(name1).SetOutPoint(out1).SetHeight(11).SetAmount(3) + chg.ClaimID = change.NewClaimID(*out1) + m.AppendChange(chg) + + chg = change.NewChange(change.AddClaim).SetName(name1).SetOutPoint(out2).SetHeight(11).SetAmount(4) + chg.ClaimID = change.NewClaimID(*out2) + m.AppendChange(chg) + + _, err = m.IncrementHeightTo(11) + r.NoError(err) + + chg = change.NewChange(change.AddSupport).SetName(name1).SetOutPoint(out3).SetHeight(12).SetAmount(2) + chg.ClaimID = change.NewClaimID(*out1) + m.AppendChange(chg) + + chg = change.NewChange(change.AddSupport).SetName(name1).SetOutPoint(out4).SetHeight(12).SetAmount(2) + chg.ClaimID = change.NewClaimID(*out2) + m.AppendChange(chg) + + chg = change.NewChange(change.SpendSupport).SetName(name1).SetOutPoint(out4).SetHeight(12).SetAmount(2) + chg.ClaimID = change.NewClaimID(*out2) + m.AppendChange(chg) + + _, err = m.IncrementHeightTo(20) + r.NoError(err) + + n1, err := m.node(name1) + r.NoError(err) + r.Equal(2, len(n1.Claims)) + r.Equal(int64(5), n1.BestClaim.Amount+n1.SupportSums[n1.BestClaim.ClaimID.Key()]) +} + +func TestNodeSort(t *testing.T) { + + r := require.New(t) + + param.ActiveParams.ExtendedClaimExpirationTime = 1000 + + r.True(OutPointLess(*out1, *out2)) + r.True(OutPointLess(*out1, *out3)) + + n := New() + n.Claims = append(n.Claims, &Claim{OutPoint: *out1, AcceptedAt: 3, Amount: 3, ClaimID: change.ClaimID{1}}) + n.Claims = append(n.Claims, &Claim{OutPoint: *out2, AcceptedAt: 3, Amount: 3, ClaimID: change.ClaimID{2}}) + n.handleExpiredAndActivated(3) + n.updateTakeoverHeight(3, []byte{}, true) + + r.Equal(n.Claims.find(byOut(*out1)).OutPoint.String(), n.BestClaim.OutPoint.String()) + + n.Claims = append(n.Claims, &Claim{OutPoint: *out3, AcceptedAt: 3, Amount: 3, ClaimID: change.ClaimID{3}}) + n.handleExpiredAndActivated(3) + n.updateTakeoverHeight(3, []byte{}, true) + r.Equal(n.Claims.find(byOut(*out1)).OutPoint.String(), n.BestClaim.OutPoint.String()) +} + +func TestClaimSort(t *testing.T) { + + r := require.New(t) + + param.ActiveParams.ExtendedClaimExpirationTime = 1000 + + n := New() + n.Claims = append(n.Claims, &Claim{OutPoint: *out2, AcceptedAt: 3, Amount: 3, ClaimID: change.ClaimID{2}}) + n.Claims = append(n.Claims, &Claim{OutPoint: *out3, AcceptedAt: 3, Amount: 2, ClaimID: change.ClaimID{3}}) + n.Claims = append(n.Claims, &Claim{OutPoint: *out3, AcceptedAt: 4, Amount: 2, ClaimID: change.ClaimID{4}}) + n.Claims = append(n.Claims, &Claim{OutPoint: *out1, AcceptedAt: 3, Amount: 4, ClaimID: change.ClaimID{1}}) + n.SortClaimsByBid() + + r.Equal(int64(4), n.Claims[0].Amount) + r.Equal(int64(3), n.Claims[1].Amount) + r.Equal(int64(2), n.Claims[2].Amount) + r.Equal(int32(4), n.Claims[3].AcceptedAt) +} + +func TestHasChildren(t *testing.T) { + r := require.New(t) + + param.SetNetwork(wire.TestNet) + repo, err := noderepo.NewPebble(t.TempDir()) + r.NoError(err) + + m, err := NewBaseManager(repo) + r.NoError(err) + defer m.Close() + + chg := change.NewChange(change.AddClaim).SetName([]byte("a")).SetOutPoint(out1).SetHeight(1).SetAmount(2) + chg.ClaimID = change.NewClaimID(*out1) + m.AppendChange(chg) + _, err = m.IncrementHeightTo(1) + r.NoError(err) + r.False(m.hasChildren([]byte("a"), 1, nil, 1)) + + chg = change.NewChange(change.AddClaim).SetName([]byte("ab")).SetOutPoint(out2).SetHeight(2).SetAmount(2) + chg.ClaimID = change.NewClaimID(*out2) + m.AppendChange(chg) + _, err = m.IncrementHeightTo(2) + r.NoError(err) + r.False(m.hasChildren([]byte("a"), 2, nil, 2)) + r.True(m.hasChildren([]byte("a"), 2, nil, 1)) + + chg = change.NewChange(change.AddClaim).SetName([]byte("abc")).SetOutPoint(out3).SetHeight(3).SetAmount(2) + chg.ClaimID = change.NewClaimID(*out3) + m.AppendChange(chg) + _, err = m.IncrementHeightTo(3) + r.NoError(err) + r.False(m.hasChildren([]byte("a"), 3, nil, 2)) + + chg = change.NewChange(change.AddClaim).SetName([]byte("ac")).SetOutPoint(out1).SetHeight(4).SetAmount(2) + chg.ClaimID = change.NewClaimID(*out4) + m.AppendChange(chg) + _, err = m.IncrementHeightTo(4) + r.NoError(err) + r.True(m.hasChildren([]byte("a"), 4, nil, 2)) +} + +func TestCollectChildren(t *testing.T) { + r := require.New(t) + + c1 := change.Change{Name: []byte("ba"), Type: change.SpendClaim} + c2 := change.Change{Name: []byte("ba"), Type: change.UpdateClaim} + c3 := change.Change{Name: []byte("ac"), Type: change.SpendClaim} + c4 := change.Change{Name: []byte("ac"), Type: change.UpdateClaim} + c5 := change.Change{Name: []byte("a"), Type: change.SpendClaim} + c6 := change.Change{Name: []byte("a"), Type: change.UpdateClaim} + c7 := change.Change{Name: []byte("ab"), Type: change.SpendClaim} + c8 := change.Change{Name: []byte("ab"), Type: change.UpdateClaim} + c := []change.Change{c1, c2, c3, c4, c5, c6, c7, c8} + + collectChildNames(c) + + r.Empty(c[0].SpentChildren) + r.Empty(c[2].SpentChildren) + r.Empty(c[4].SpentChildren) + r.Empty(c[6].SpentChildren) + + r.Len(c[1].SpentChildren, 0) + r.Len(c[3].SpentChildren, 0) + r.Len(c[5].SpentChildren, 1) + r.True(c[5].SpentChildren["ac"]) + + r.Len(c[7].SpentChildren, 0) +} diff --git a/claimtrie/node/node.go b/claimtrie/node/node.go new file mode 100644 index 00000000..34f0b60c --- /dev/null +++ b/claimtrie/node/node.go @@ -0,0 +1,313 @@ +package node + +import ( + "fmt" + "math" + "sort" + + "github.com/lbryio/lbcd/claimtrie/change" + "github.com/lbryio/lbcd/claimtrie/param" +) + +type Node struct { + BestClaim *Claim // The claim that has most effective amount at the current height. + TakenOverAt int32 // The height at when the current BestClaim took over. + Claims ClaimList // List of all Claims. + Supports ClaimList // List of all Supports, including orphaned ones. + SupportSums map[string]int64 +} + +// New returns a new node. +func New() *Node { + return &Node{SupportSums: map[string]int64{}} +} + +func (n *Node) HasActiveBestClaim() bool { + return n.BestClaim != nil && n.BestClaim.Status == Activated +} + +func (n *Node) ApplyChange(chg change.Change, delay int32) error { + + visibleAt := chg.VisibleHeight + if visibleAt <= 0 { + visibleAt = chg.Height + } + + switch chg.Type { + case change.AddClaim: + c := &Claim{ + OutPoint: chg.OutPoint, + Amount: chg.Amount, + ClaimID: chg.ClaimID, + // CreatedAt: chg.Height, + AcceptedAt: chg.Height, + ActiveAt: chg.Height + delay, + VisibleAt: visibleAt, + Sequence: int32(len(n.Claims)), + } + // old := n.Claims.find(byOut(chg.OutPoint)) // TODO: remove this after proving ResetHeight works + // if old != nil { + // return errors.Errorf("CONFLICT WITH EXISTING TXO! Name: %s, Height: %d", chg.Name, chg.Height) + // } + n.Claims = append(n.Claims, c) + + case change.SpendClaim: + c := n.Claims.find(byOut(chg.OutPoint)) + if c != nil { + c.setStatus(Deactivated) + } else { + LogOnce(fmt.Sprintf("Spending claim but missing existing claim with TXO %s, "+ + "Name: %s, ID: %s", chg.OutPoint, chg.Name, chg.ClaimID)) + } + // apparently it's legit to be absent in the map: + // 'two' at 481100, 36a719a156a1df178531f3c712b8b37f8e7cc3b36eea532df961229d936272a1:0 + + case change.UpdateClaim: + // Find and remove the claim, which has just been spent. + c := n.Claims.find(byID(chg.ClaimID)) + if c != nil && c.Status == Deactivated { + + // Keep its ID, which was generated from the spent claim. + // And update the rest of properties. + c.setOutPoint(chg.OutPoint).SetAmt(chg.Amount) + c.setStatus(Accepted) // it was Deactivated in the spend (but we only activate at the end of the block) + // that's because the old code would put all insertions into the "queue" that was processed at block's end + + // This forces us to be newer, which may in an unintentional takeover if there's an older one. + // TODO: reconsider these updates in future hard forks. + c.setAccepted(chg.Height) + c.setActiveAt(chg.Height + delay) + + } else { + LogOnce(fmt.Sprintf("Updating claim but missing existing claim with ID %s", chg.ClaimID)) + } + case change.AddSupport: + n.Supports = append(n.Supports, &Claim{ + OutPoint: chg.OutPoint, + Amount: chg.Amount, + ClaimID: chg.ClaimID, + AcceptedAt: chg.Height, + ActiveAt: chg.Height + delay, + VisibleAt: visibleAt, + }) + + case change.SpendSupport: + s := n.Supports.find(byOut(chg.OutPoint)) + if s != nil { + if s.Status == Activated { + n.SupportSums[s.ClaimID.Key()] -= s.Amount + } + // TODO: we could do without this Deactivated flag if we set expiration instead + // That would eliminate the above Sum update. + // We would also need to track the update situation, though, but that could be done locally. + s.setStatus(Deactivated) + } else { + LogOnce(fmt.Sprintf("Spending support but missing existing claim with TXO %s, "+ + "Name: %s, ID: %s", chg.OutPoint, chg.Name, chg.ClaimID)) + } + } + return nil +} + +// AdjustTo activates claims and computes takeovers until it reaches the specified height. +func (n *Node) AdjustTo(height, maxHeight int32, name []byte) *Node { + changed := n.handleExpiredAndActivated(height) > 0 + n.updateTakeoverHeight(height, name, changed) + if maxHeight > height { + for h := n.NextUpdate(); h <= maxHeight; h = n.NextUpdate() { + changed = n.handleExpiredAndActivated(h) > 0 + n.updateTakeoverHeight(h, name, changed) + height = h + } + } + return n +} + +func (n *Node) updateTakeoverHeight(height int32, name []byte, refindBest bool) { + + candidate := n.BestClaim + if refindBest { + candidate = n.findBestClaim() // so expensive... + } + + hasCandidate := candidate != nil + hasCurrentWinner := n.HasActiveBestClaim() + + takeoverHappening := !hasCandidate || !hasCurrentWinner || candidate.ClaimID != n.BestClaim.ClaimID + + if takeoverHappening { + if n.activateAllClaims(height) > 0 { + candidate = n.findBestClaim() + } + } + + if !takeoverHappening && height < param.ActiveParams.MaxRemovalWorkaroundHeight { + // This is a super ugly hack to work around bug in old code. + // The bug: un/support a name then update it. This will cause its takeover height to be reset to current. + // This is because the old code would add to the cache without setting block originals when dealing in supports. + _, takeoverHappening = param.TakeoverWorkarounds[fmt.Sprintf("%d_%s", height, name)] // TODO: ditch the fmt call + } + + if takeoverHappening { + n.TakenOverAt = height + n.BestClaim = candidate + } +} + +func (n *Node) handleExpiredAndActivated(height int32) int { + + changes := 0 + update := func(items ClaimList, sums map[string]int64) ClaimList { + for i := 0; i < len(items); i++ { + c := items[i] + if c.Status == Accepted && c.ActiveAt <= height && c.VisibleAt <= height { + c.setStatus(Activated) + changes++ + if sums != nil { + sums[c.ClaimID.Key()] += c.Amount + } + } + if c.ExpireAt() <= height || c.Status == Deactivated { + if i < len(items)-1 { + items[i] = items[len(items)-1] + i-- + } + items = items[:len(items)-1] + changes++ + if sums != nil && c.Status != Deactivated { + sums[c.ClaimID.Key()] -= c.Amount + } + } + } + return items + } + n.Claims = update(n.Claims, nil) + n.Supports = update(n.Supports, n.SupportSums) + return changes +} + +// NextUpdate returns the nearest height in the future that the node should +// be refreshed due to changes of claims or supports. +func (n Node) NextUpdate() int32 { + + next := int32(math.MaxInt32) + + for _, c := range n.Claims { + if c.ExpireAt() < next { + next = c.ExpireAt() + } + // if we're not active, we need to go to activeAt unless we're still invisible there + if c.Status == Accepted { + min := c.ActiveAt + if c.VisibleAt > min { + min = c.VisibleAt + } + if min < next { + next = min + } + } + } + + for _, s := range n.Supports { + if s.ExpireAt() < next { + next = s.ExpireAt() + } + if s.Status == Accepted { + min := s.ActiveAt + if s.VisibleAt > min { + min = s.VisibleAt + } + if min < next { + next = min + } + } + } + + return next +} + +func (n Node) findBestClaim() *Claim { + + // WARNING: this method is called billions of times. + // if we just had some easy way to know that our best claim was the first one in the list... + // or it may be faster to cache effective amount in the db at some point. + + var best *Claim + var bestAmount int64 + for _, candidate := range n.Claims { + + // not using switch here for performance reasons + if candidate.Status != Activated { + continue + } + + if best == nil { + best = candidate + continue + } + + candidateAmount := candidate.Amount + n.SupportSums[candidate.ClaimID.Key()] + if bestAmount <= 0 { + bestAmount = best.Amount + n.SupportSums[best.ClaimID.Key()] + } + + switch { + case candidateAmount > bestAmount: + best = candidate + bestAmount = candidateAmount + case candidateAmount < bestAmount: + continue + case candidate.AcceptedAt < best.AcceptedAt: + best = candidate + bestAmount = candidateAmount + case candidate.AcceptedAt > best.AcceptedAt: + continue + case OutPointLess(candidate.OutPoint, best.OutPoint): + best = candidate + bestAmount = candidateAmount + } + } + + return best +} + +func (n *Node) activateAllClaims(height int32) int { + count := 0 + for _, c := range n.Claims { + if c.Status == Accepted && c.ActiveAt > height && c.VisibleAt <= height { + c.setActiveAt(height) // don't necessarily need to change this number? + c.setStatus(Activated) + count++ + } + } + + for _, s := range n.Supports { + if s.Status == Accepted && s.ActiveAt > height && s.VisibleAt <= height { + s.setActiveAt(height) // don't necessarily need to change this number? + s.setStatus(Activated) + count++ + n.SupportSums[s.ClaimID.Key()] += s.Amount + } + } + return count +} + +func (n *Node) SortClaimsByBid() { + + // purposefully sorting by descent + sort.Slice(n.Claims, func(j, i int) bool { + iAmount := n.Claims[i].Amount + n.SupportSums[n.Claims[i].ClaimID.Key()] + jAmount := n.Claims[j].Amount + n.SupportSums[n.Claims[j].ClaimID.Key()] + switch { + case iAmount < jAmount: + return true + case iAmount > jAmount: + return false + case n.Claims[i].AcceptedAt > n.Claims[j].AcceptedAt: + return true + case n.Claims[i].AcceptedAt < n.Claims[j].AcceptedAt: + return false + } + return OutPointLess(n.Claims[j].OutPoint, n.Claims[i].OutPoint) + }) +} diff --git a/claimtrie/node/noderepo/noderepo_test.go b/claimtrie/node/noderepo/noderepo_test.go new file mode 100644 index 00000000..fb0a9764 --- /dev/null +++ b/claimtrie/node/noderepo/noderepo_test.go @@ -0,0 +1,188 @@ +package noderepo + +import ( + "testing" + + "github.com/lbryio/lbcd/claimtrie/change" + "github.com/lbryio/lbcd/claimtrie/node" + + "github.com/stretchr/testify/require" +) + +var ( + out1 = node.NewOutPointFromString("0000000000000000000000000000000000000000000000000000000000000000:1") + testNodeName1 = []byte("name1") +) + +func TestPebble(t *testing.T) { + + r := require.New(t) + + repo, err := NewPebble(t.TempDir()) + r.NoError(err) + defer func() { + err := repo.Close() + r.NoError(err) + }() + + cleanup := func() { + lowerBound := testNodeName1 + upperBound := append(testNodeName1, byte(0)) + err := repo.db.DeleteRange(lowerBound, upperBound, nil) + r.NoError(err) + } + + testNodeRepo(t, repo, func() {}, cleanup) +} + +func testNodeRepo(t *testing.T, repo node.Repo, setup, cleanup func()) { + + r := require.New(t) + + chg := change.NewChange(change.AddClaim).SetName(testNodeName1).SetOutPoint(out1) + + testcases := []struct { + name string + height int32 + changes []change.Change + expected []change.Change + }{ + { + "test 1", + 1, + []change.Change{chg.SetHeight(1), chg.SetHeight(3), chg.SetHeight(5)}, + []change.Change{chg.SetHeight(1)}, + }, + { + "test 2", + 2, + []change.Change{chg.SetHeight(1), chg.SetHeight(3), chg.SetHeight(5)}, + []change.Change{chg.SetHeight(1)}, + }, + { + "test 3", + 3, + []change.Change{chg.SetHeight(1), chg.SetHeight(3), chg.SetHeight(5)}, + []change.Change{chg.SetHeight(1), chg.SetHeight(3)}, + }, + { + "test 4", + 4, + []change.Change{chg.SetHeight(1), chg.SetHeight(3), chg.SetHeight(5)}, + []change.Change{chg.SetHeight(1), chg.SetHeight(3)}, + }, + { + "test 5", + 5, + []change.Change{chg.SetHeight(1), chg.SetHeight(3), chg.SetHeight(5)}, + []change.Change{chg.SetHeight(1), chg.SetHeight(3), chg.SetHeight(5)}, + }, + { + "test 6", + 6, + []change.Change{chg.SetHeight(1), chg.SetHeight(3), chg.SetHeight(5)}, + []change.Change{chg.SetHeight(1), chg.SetHeight(3), chg.SetHeight(5)}, + }, + } + + for _, tt := range testcases { + + setup() + + err := repo.AppendChanges(tt.changes) + r.NoError(err) + + changes, err := repo.LoadChanges(testNodeName1) + r.NoError(err) + r.Equalf(tt.expected, changes[:len(tt.expected)], tt.name) + + cleanup() + } + + testcases2 := []struct { + name string + height int32 + changes [][]change.Change + expected []change.Change + }{ + { + "Save in 2 batches, and load up to 1", + 1, + [][]change.Change{ + {chg.SetHeight(1), chg.SetHeight(3), chg.SetHeight(5)}, + {chg.SetHeight(6), chg.SetHeight(8), chg.SetHeight(9)}, + }, + []change.Change{chg.SetHeight(1)}, + }, + { + "Save in 2 batches, and load up to 9", + 9, + [][]change.Change{ + {chg.SetHeight(1), chg.SetHeight(3), chg.SetHeight(5)}, + {chg.SetHeight(6), chg.SetHeight(8), chg.SetHeight(9)}, + }, + []change.Change{ + chg.SetHeight(1), chg.SetHeight(3), chg.SetHeight(5), + chg.SetHeight(6), chg.SetHeight(8), chg.SetHeight(9), + }, + }, + { + "Save in 3 batches, and load up to 8", + 8, + [][]change.Change{ + {chg.SetHeight(1), chg.SetHeight(3)}, + {chg.SetHeight(5)}, + {chg.SetHeight(6), chg.SetHeight(8), chg.SetHeight(9)}, + }, + []change.Change{ + chg.SetHeight(1), chg.SetHeight(3), chg.SetHeight(5), + chg.SetHeight(6), chg.SetHeight(8), + }, + }, + } + + for _, tt := range testcases2 { + + setup() + + for _, changes := range tt.changes { + err := repo.AppendChanges(changes) + r.NoError(err) + } + + changes, err := repo.LoadChanges(testNodeName1) + r.NoError(err) + r.Equalf(tt.expected, changes[:len(tt.expected)], tt.name) + + cleanup() + } +} + +func TestIterator(t *testing.T) { + + r := require.New(t) + + repo, err := NewPebble(t.TempDir()) + r.NoError(err) + defer func() { + err := repo.Close() + r.NoError(err) + }() + + creation := []change.Change{ + {Name: []byte("test\x00"), Height: 5}, + {Name: []byte("test\x00\x00"), Height: 5}, + {Name: []byte("test\x00b"), Height: 5}, + {Name: []byte("test\x00\xFF"), Height: 5}, + {Name: []byte("testa"), Height: 5}, + } + err = repo.AppendChanges(creation) + r.NoError(err) + + i := 0 + repo.IterateChildren([]byte{}, func(changes []change.Change) bool { + r.Equal(creation[i], changes[0]) + i++ + return true + }) +} diff --git a/claimtrie/node/noderepo/pebble.go b/claimtrie/node/noderepo/pebble.go new file mode 100644 index 00000000..a13dda82 --- /dev/null +++ b/claimtrie/node/noderepo/pebble.go @@ -0,0 +1,171 @@ +package noderepo + +import ( + "bytes" + "sort" + + "github.com/cockroachdb/pebble" + "github.com/lbryio/lbcd/claimtrie/change" + "github.com/pkg/errors" +) + +type Pebble struct { + db *pebble.DB +} + +func NewPebble(path string) (*Pebble, error) { + + db, err := pebble.Open(path, &pebble.Options{Cache: pebble.NewCache(64 << 20), BytesPerSync: 8 << 20, MaxOpenFiles: 2000}) + repo := &Pebble{db: db} + + return repo, errors.Wrapf(err, "unable to open %s", path) +} + +// AppendChanges makes an assumption that anything you pass to it is newer than what was saved before. +func (repo *Pebble) AppendChanges(changes []change.Change) error { + + batch := repo.db.NewBatch() + defer batch.Close() + + buffer := bytes.NewBuffer(nil) + + for _, chg := range changes { + buffer.Reset() + err := chg.Marshal(buffer) + if err != nil { + return errors.Wrap(err, "in marshaller") + } + + err = batch.Merge(chg.Name, buffer.Bytes(), pebble.NoSync) + if err != nil { + return errors.Wrap(err, "in merge") + } + } + return errors.Wrap(batch.Commit(pebble.NoSync), "in commit") +} + +func (repo *Pebble) LoadChanges(name []byte) ([]change.Change, error) { + + data, closer, err := repo.db.Get(name) + if err != nil && err != pebble.ErrNotFound { + return nil, errors.Wrapf(err, "in get %s", name) // does returning a name in an error expose too much? + } + if closer != nil { + defer closer.Close() + } + + return unmarshalChanges(name, data) +} + +func unmarshalChanges(name, data []byte) ([]change.Change, error) { + // data is 84bytes+ per change + changes := make([]change.Change, 0, len(data)/84+1) // average is 5.1 changes + + buffer := bytes.NewBuffer(data) + for buffer.Len() > 0 { + var chg change.Change + err := chg.Unmarshal(buffer) + if err != nil { + return nil, errors.Wrap(err, "in decode") + } + chg.Name = name + changes = append(changes, chg) + } + + // this was required for the normalization stuff: + sort.SliceStable(changes, func(i, j int) bool { + return changes[i].Height < changes[j].Height + }) + + return changes, nil +} + +func (repo *Pebble) DropChanges(name []byte, finalHeight int32) error { + changes, err := repo.LoadChanges(name) + if err != nil { + return errors.Wrapf(err, "in load changes for %s", name) + } + i := 0 + for ; i < len(changes); i++ { // assuming changes are ordered by height + if changes[i].Height > finalHeight { + break + } + if changes[i].VisibleHeight > finalHeight { // created after this height has to be deleted + changes = append(changes[:i], changes[i+1:]...) + i-- + } + } + // making a performance assumption that DropChanges won't happen often: + err = repo.db.Set(name, []byte{}, pebble.NoSync) + if err != nil { + return errors.Wrapf(err, "in set at %s", name) + } + return repo.AppendChanges(changes[:i]) +} + +func (repo *Pebble) IterateChildren(name []byte, f func(changes []change.Change) bool) error { + start := make([]byte, len(name)+1) // zeros that last byte; need a constant len for stack alloc? + copy(start, name) + + end := make([]byte, len(name)) // max name length is 255 + copy(end, name) + validEnd := false + for i := len(name) - 1; i >= 0; i-- { + end[i]++ + if end[i] != 0 { + validEnd = true + break + } + } + if !validEnd { + end = nil // uh, we think this means run to the end of the table + } + + prefixIterOptions := &pebble.IterOptions{ + LowerBound: start, + UpperBound: end, + } + + iter := repo.db.NewIter(prefixIterOptions) + defer iter.Close() + + for iter.First(); iter.Valid(); iter.Next() { + // NOTE! iter.Key() is ephemeral! + changes, err := unmarshalChanges(iter.Key(), iter.Value()) + if err != nil { + return errors.Wrapf(err, "from unmarshaller at %s", iter.Key()) + } + if !f(changes) { + break + } + } + return nil +} + +func (repo *Pebble) IterateAll(predicate func(name []byte) bool) { + iter := repo.db.NewIter(nil) + defer iter.Close() + + for iter.First(); iter.Valid(); iter.Next() { + if !predicate(iter.Key()) { + break + } + } +} + +func (repo *Pebble) Close() error { + + err := repo.db.Flush() + if err != nil { + // if we fail to close are we going to try again later? + return errors.Wrap(err, "on flush") + } + + err = repo.db.Close() + return errors.Wrap(err, "on close") +} + +func (repo *Pebble) Flush() error { + _, err := repo.db.AsyncFlush() + return err +} diff --git a/claimtrie/node/normalizing_manager.go b/claimtrie/node/normalizing_manager.go new file mode 100644 index 00000000..d35403cd --- /dev/null +++ b/claimtrie/node/normalizing_manager.go @@ -0,0 +1,114 @@ +package node + +import ( + "bytes" + + "github.com/lbryio/lbcd/claimtrie/change" + "github.com/lbryio/lbcd/claimtrie/normalization" + "github.com/lbryio/lbcd/claimtrie/param" +) + +type NormalizingManager struct { // implements Manager + Manager + normalizedAt int32 +} + +func NewNormalizingManager(baseManager Manager) Manager { + log.Info(normalization.NormalizeTitle) + return &NormalizingManager{ + Manager: baseManager, + normalizedAt: -1, + } +} + +func (nm *NormalizingManager) AppendChange(chg change.Change) { + chg.Name = normalization.NormalizeIfNecessary(chg.Name, chg.Height) + nm.Manager.AppendChange(chg) +} + +func (nm *NormalizingManager) IncrementHeightTo(height int32) ([][]byte, error) { + nm.addNormalizationForkChangesIfNecessary(height) + return nm.Manager.IncrementHeightTo(height) +} + +func (nm *NormalizingManager) DecrementHeightTo(affectedNames [][]byte, height int32) error { + if nm.normalizedAt > height { + nm.normalizedAt = -1 + } + return nm.Manager.DecrementHeightTo(affectedNames, height) +} + +func (nm *NormalizingManager) addNormalizationForkChangesIfNecessary(height int32) { + + if nm.Manager.Height()+1 != height { + // initialization phase + if height >= param.ActiveParams.NormalizedNameForkHeight { + nm.normalizedAt = param.ActiveParams.NormalizedNameForkHeight // eh, we don't really know that it happened there + } + } + + if nm.normalizedAt >= 0 || height != param.ActiveParams.NormalizedNameForkHeight { + return + } + nm.normalizedAt = height + log.Info("Generating necessary changes for the normalization fork...") + + // the original code had an unfortunate bug where many unnecessary takeovers + // were triggered at the normalization fork + predicate := func(name []byte) bool { + norm := normalization.Normalize(name) + eq := bytes.Equal(name, norm) + if eq { + return true + } + + clone := make([]byte, len(name)) + copy(clone, name) // iteration name buffer is reused on future loops + + // by loading changes for norm here, you can determine if there will be a conflict + + n, err := nm.Manager.NodeAt(nm.Manager.Height(), clone) + if err != nil || n == nil { + return true + } + for _, c := range n.Claims { + nm.Manager.AppendChange(change.Change{ + Type: change.AddClaim, + Name: norm, + Height: c.AcceptedAt, + OutPoint: c.OutPoint, + ClaimID: c.ClaimID, + Amount: c.Amount, + ActiveHeight: c.ActiveAt, // necessary to match the old hash + VisibleHeight: height, // necessary to match the old hash; it would have been much better without + }) + nm.Manager.AppendChange(change.Change{ + Type: change.SpendClaim, + Name: clone, + Height: height, + OutPoint: c.OutPoint, + }) + } + for _, c := range n.Supports { + nm.Manager.AppendChange(change.Change{ + Type: change.AddSupport, + Name: norm, + Height: c.AcceptedAt, + OutPoint: c.OutPoint, + ClaimID: c.ClaimID, + Amount: c.Amount, + ActiveHeight: c.ActiveAt, + VisibleHeight: height, + }) + nm.Manager.AppendChange(change.Change{ + Type: change.SpendSupport, + Name: clone, + Height: height, + OutPoint: c.OutPoint, + }) + } + + return true + } + nm.Manager.IterateNames(predicate) +} diff --git a/claimtrie/node/repo.go b/claimtrie/node/repo.go new file mode 100644 index 00000000..4aaa65e8 --- /dev/null +++ b/claimtrie/node/repo.go @@ -0,0 +1,31 @@ +package node + +import ( + "github.com/lbryio/lbcd/claimtrie/change" +) + +// Repo defines APIs for Node to access persistence layer. +type Repo interface { + // AppendChanges saves changes into the repo. + // The changes can belong to different nodes, but the chronological + // order must be preserved for the same node. + AppendChanges(changes []change.Change) error + + // LoadChanges loads changes of a node up to (includes) the specified height. + // If no changes found, both returned slice and error will be nil. + LoadChanges(name []byte) ([]change.Change, error) + + DropChanges(name []byte, finalHeight int32) error + + // Close closes the repo. + Close() error + + // IterateChildren returns change sets for each of name.+ + // Return false on f to stop the iteration. + IterateChildren(name []byte, f func(changes []change.Change) bool) error + + // IterateAll iterates keys until the predicate function returns false + IterateAll(predicate func(name []byte) bool) + + Flush() error +} diff --git a/claimtrie/normalization/CaseFolding_v11.txt b/claimtrie/normalization/CaseFolding_v11.txt new file mode 100644 index 00000000..cce350f4 --- /dev/null +++ b/claimtrie/normalization/CaseFolding_v11.txt @@ -0,0 +1,1574 @@ +# CaseFolding-11.0.0.txt +# Date: 2018-01-31, 08:20:09 GMT +# © 2018 Unicode®, Inc. +# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. +# For terms of use, see http://www.unicode.org/terms_of_use.html +# +# Unicode Character Database +# For documentation, see http://www.unicode.org/reports/tr44/ +# +# Case Folding Properties +# +# This file is a supplement to the UnicodeData file. +# It provides a case folding mapping generated from the Unicode Character Database. +# If all characters are mapped according to the full mapping below, then +# case differences (according to UnicodeData.txt and SpecialCasing.txt) +# are eliminated. +# +# The data supports both implementations that require simple case foldings +# (where string lengths don't change), and implementations that allow full case folding +# (where string lengths may grow). Note that where they can be supported, the +# full case foldings are superior: for example, they allow "MASSE" and "Maße" to match. +# +# All code points not listed in this file map to themselves. +# +# NOTE: case folding does not preserve normalization formats! +# +# For information on case folding, including how to have case folding +# preserve normalization formats, see Section 3.13 Default Case Algorithms in +# The Unicode Standard. +# +# ================================================================================ +# Format +# ================================================================================ +# The entries in this file are in the following machine-readable format: +# +# ; ; ; # +# +# The status field is: +# C: common case folding, common mappings shared by both simple and full mappings. +# F: full case folding, mappings that cause strings to grow in length. Multiple characters are separated by spaces. +# S: simple case folding, mappings to single characters where different from F. +# T: special case for uppercase I and dotted uppercase I +# - For non-Turkic languages, this mapping is normally not used. +# - For Turkic languages (tr, az), this mapping can be used instead of the normal mapping for these characters. +# Note that the Turkic mappings do not maintain canonical equivalence without additional processing. +# See the discussions of case mapping in the Unicode Standard for more information. +# +# Usage: +# A. To do a simple case folding, use the mappings with status C + S. +# B. To do a full case folding, use the mappings with status C + F. +# +# The mappings with status T can be used or omitted depending on the desired case-folding +# behavior. (The default option is to exclude them.) +# +# ================================================================= + +# Property: Case_Folding + +# All code points not explicitly listed for Case_Folding +# have the value C for the status field, and the code point itself for the mapping field. + +# ================================================================= +0041; C; 0061; # LATIN CAPITAL LETTER A +0042; C; 0062; # LATIN CAPITAL LETTER B +0043; C; 0063; # LATIN CAPITAL LETTER C +0044; C; 0064; # LATIN CAPITAL LETTER D +0045; C; 0065; # LATIN CAPITAL LETTER E +0046; C; 0066; # LATIN CAPITAL LETTER F +0047; C; 0067; # LATIN CAPITAL LETTER G +0048; C; 0068; # LATIN CAPITAL LETTER H +0049; C; 0069; # LATIN CAPITAL LETTER I +0049; T; 0131; # LATIN CAPITAL LETTER I +004A; C; 006A; # LATIN CAPITAL LETTER J +004B; C; 006B; # LATIN CAPITAL LETTER K +004C; C; 006C; # LATIN CAPITAL LETTER L +004D; C; 006D; # LATIN CAPITAL LETTER M +004E; C; 006E; # LATIN CAPITAL LETTER N +004F; C; 006F; # LATIN CAPITAL LETTER O +0050; C; 0070; # LATIN CAPITAL LETTER P +0051; C; 0071; # LATIN CAPITAL LETTER Q +0052; C; 0072; # LATIN CAPITAL LETTER R +0053; C; 0073; # LATIN CAPITAL LETTER S +0054; C; 0074; # LATIN CAPITAL LETTER T +0055; C; 0075; # LATIN CAPITAL LETTER U +0056; C; 0076; # LATIN CAPITAL LETTER V +0057; C; 0077; # LATIN CAPITAL LETTER W +0058; C; 0078; # LATIN CAPITAL LETTER X +0059; C; 0079; # LATIN CAPITAL LETTER Y +005A; C; 007A; # LATIN CAPITAL LETTER Z +00B5; C; 03BC; # MICRO SIGN +00C0; C; 00E0; # LATIN CAPITAL LETTER A WITH GRAVE +00C1; C; 00E1; # LATIN CAPITAL LETTER A WITH ACUTE +00C2; C; 00E2; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX +00C3; C; 00E3; # LATIN CAPITAL LETTER A WITH TILDE +00C4; C; 00E4; # LATIN CAPITAL LETTER A WITH DIAERESIS +00C5; C; 00E5; # LATIN CAPITAL LETTER A WITH RING ABOVE +00C6; C; 00E6; # LATIN CAPITAL LETTER AE +00C7; C; 00E7; # LATIN CAPITAL LETTER C WITH CEDILLA +00C8; C; 00E8; # LATIN CAPITAL LETTER E WITH GRAVE +00C9; C; 00E9; # LATIN CAPITAL LETTER E WITH ACUTE +00CA; C; 00EA; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX +00CB; C; 00EB; # LATIN CAPITAL LETTER E WITH DIAERESIS +00CC; C; 00EC; # LATIN CAPITAL LETTER I WITH GRAVE +00CD; C; 00ED; # LATIN CAPITAL LETTER I WITH ACUTE +00CE; C; 00EE; # LATIN CAPITAL LETTER I WITH CIRCUMFLEX +00CF; C; 00EF; # LATIN CAPITAL LETTER I WITH DIAERESIS +00D0; C; 00F0; # LATIN CAPITAL LETTER ETH +00D1; C; 00F1; # LATIN CAPITAL LETTER N WITH TILDE +00D2; C; 00F2; # LATIN CAPITAL LETTER O WITH GRAVE +00D3; C; 00F3; # LATIN CAPITAL LETTER O WITH ACUTE +00D4; C; 00F4; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX +00D5; C; 00F5; # LATIN CAPITAL LETTER O WITH TILDE +00D6; C; 00F6; # LATIN CAPITAL LETTER O WITH DIAERESIS +00D8; C; 00F8; # LATIN CAPITAL LETTER O WITH STROKE +00D9; C; 00F9; # LATIN CAPITAL LETTER U WITH GRAVE +00DA; C; 00FA; # LATIN CAPITAL LETTER U WITH ACUTE +00DB; C; 00FB; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX +00DC; C; 00FC; # LATIN CAPITAL LETTER U WITH DIAERESIS +00DD; C; 00FD; # LATIN CAPITAL LETTER Y WITH ACUTE +00DE; C; 00FE; # LATIN CAPITAL LETTER THORN +00DF; F; 0073 0073; # LATIN SMALL LETTER SHARP S +0100; C; 0101; # LATIN CAPITAL LETTER A WITH MACRON +0102; C; 0103; # LATIN CAPITAL LETTER A WITH BREVE +0104; C; 0105; # LATIN CAPITAL LETTER A WITH OGONEK +0106; C; 0107; # LATIN CAPITAL LETTER C WITH ACUTE +0108; C; 0109; # LATIN CAPITAL LETTER C WITH CIRCUMFLEX +010A; C; 010B; # LATIN CAPITAL LETTER C WITH DOT ABOVE +010C; C; 010D; # LATIN CAPITAL LETTER C WITH CARON +010E; C; 010F; # LATIN CAPITAL LETTER D WITH CARON +0110; C; 0111; # LATIN CAPITAL LETTER D WITH STROKE +0112; C; 0113; # LATIN CAPITAL LETTER E WITH MACRON +0114; C; 0115; # LATIN CAPITAL LETTER E WITH BREVE +0116; C; 0117; # LATIN CAPITAL LETTER E WITH DOT ABOVE +0118; C; 0119; # LATIN CAPITAL LETTER E WITH OGONEK +011A; C; 011B; # LATIN CAPITAL LETTER E WITH CARON +011C; C; 011D; # LATIN CAPITAL LETTER G WITH CIRCUMFLEX +011E; C; 011F; # LATIN CAPITAL LETTER G WITH BREVE +0120; C; 0121; # LATIN CAPITAL LETTER G WITH DOT ABOVE +0122; C; 0123; # LATIN CAPITAL LETTER G WITH CEDILLA +0124; C; 0125; # LATIN CAPITAL LETTER H WITH CIRCUMFLEX +0126; C; 0127; # LATIN CAPITAL LETTER H WITH STROKE +0128; C; 0129; # LATIN CAPITAL LETTER I WITH TILDE +012A; C; 012B; # LATIN CAPITAL LETTER I WITH MACRON +012C; C; 012D; # LATIN CAPITAL LETTER I WITH BREVE +012E; C; 012F; # LATIN CAPITAL LETTER I WITH OGONEK +0130; F; 0069 0307; # LATIN CAPITAL LETTER I WITH DOT ABOVE +0130; T; 0069; # LATIN CAPITAL LETTER I WITH DOT ABOVE +0132; C; 0133; # LATIN CAPITAL LIGATURE IJ +0134; C; 0135; # LATIN CAPITAL LETTER J WITH CIRCUMFLEX +0136; C; 0137; # LATIN CAPITAL LETTER K WITH CEDILLA +0139; C; 013A; # LATIN CAPITAL LETTER L WITH ACUTE +013B; C; 013C; # LATIN CAPITAL LETTER L WITH CEDILLA +013D; C; 013E; # LATIN CAPITAL LETTER L WITH CARON +013F; C; 0140; # LATIN CAPITAL LETTER L WITH MIDDLE DOT +0141; C; 0142; # LATIN CAPITAL LETTER L WITH STROKE +0143; C; 0144; # LATIN CAPITAL LETTER N WITH ACUTE +0145; C; 0146; # LATIN CAPITAL LETTER N WITH CEDILLA +0147; C; 0148; # LATIN CAPITAL LETTER N WITH CARON +0149; F; 02BC 006E; # LATIN SMALL LETTER N PRECEDED BY APOSTROPHE +014A; C; 014B; # LATIN CAPITAL LETTER ENG +014C; C; 014D; # LATIN CAPITAL LETTER O WITH MACRON +014E; C; 014F; # LATIN CAPITAL LETTER O WITH BREVE +0150; C; 0151; # LATIN CAPITAL LETTER O WITH DOUBLE ACUTE +0152; C; 0153; # LATIN CAPITAL LIGATURE OE +0154; C; 0155; # LATIN CAPITAL LETTER R WITH ACUTE +0156; C; 0157; # LATIN CAPITAL LETTER R WITH CEDILLA +0158; C; 0159; # LATIN CAPITAL LETTER R WITH CARON +015A; C; 015B; # LATIN CAPITAL LETTER S WITH ACUTE +015C; C; 015D; # LATIN CAPITAL LETTER S WITH CIRCUMFLEX +015E; C; 015F; # LATIN CAPITAL LETTER S WITH CEDILLA +0160; C; 0161; # LATIN CAPITAL LETTER S WITH CARON +0162; C; 0163; # LATIN CAPITAL LETTER T WITH CEDILLA +0164; C; 0165; # LATIN CAPITAL LETTER T WITH CARON +0166; C; 0167; # LATIN CAPITAL LETTER T WITH STROKE +0168; C; 0169; # LATIN CAPITAL LETTER U WITH TILDE +016A; C; 016B; # LATIN CAPITAL LETTER U WITH MACRON +016C; C; 016D; # LATIN CAPITAL LETTER U WITH BREVE +016E; C; 016F; # LATIN CAPITAL LETTER U WITH RING ABOVE +0170; C; 0171; # LATIN CAPITAL LETTER U WITH DOUBLE ACUTE +0172; C; 0173; # LATIN CAPITAL LETTER U WITH OGONEK +0174; C; 0175; # LATIN CAPITAL LETTER W WITH CIRCUMFLEX +0176; C; 0177; # LATIN CAPITAL LETTER Y WITH CIRCUMFLEX +0178; C; 00FF; # LATIN CAPITAL LETTER Y WITH DIAERESIS +0179; C; 017A; # LATIN CAPITAL LETTER Z WITH ACUTE +017B; C; 017C; # LATIN CAPITAL LETTER Z WITH DOT ABOVE +017D; C; 017E; # LATIN CAPITAL LETTER Z WITH CARON +017F; C; 0073; # LATIN SMALL LETTER LONG S +0181; C; 0253; # LATIN CAPITAL LETTER B WITH HOOK +0182; C; 0183; # LATIN CAPITAL LETTER B WITH TOPBAR +0184; C; 0185; # LATIN CAPITAL LETTER TONE SIX +0186; C; 0254; # LATIN CAPITAL LETTER OPEN O +0187; C; 0188; # LATIN CAPITAL LETTER C WITH HOOK +0189; C; 0256; # LATIN CAPITAL LETTER AFRICAN D +018A; C; 0257; # LATIN CAPITAL LETTER D WITH HOOK +018B; C; 018C; # LATIN CAPITAL LETTER D WITH TOPBAR +018E; C; 01DD; # LATIN CAPITAL LETTER REVERSED E +018F; C; 0259; # LATIN CAPITAL LETTER SCHWA +0190; C; 025B; # LATIN CAPITAL LETTER OPEN E +0191; C; 0192; # LATIN CAPITAL LETTER F WITH HOOK +0193; C; 0260; # LATIN CAPITAL LETTER G WITH HOOK +0194; C; 0263; # LATIN CAPITAL LETTER GAMMA +0196; C; 0269; # LATIN CAPITAL LETTER IOTA +0197; C; 0268; # LATIN CAPITAL LETTER I WITH STROKE +0198; C; 0199; # LATIN CAPITAL LETTER K WITH HOOK +019C; C; 026F; # LATIN CAPITAL LETTER TURNED M +019D; C; 0272; # LATIN CAPITAL LETTER N WITH LEFT HOOK +019F; C; 0275; # LATIN CAPITAL LETTER O WITH MIDDLE TILDE +01A0; C; 01A1; # LATIN CAPITAL LETTER O WITH HORN +01A2; C; 01A3; # LATIN CAPITAL LETTER OI +01A4; C; 01A5; # LATIN CAPITAL LETTER P WITH HOOK +01A6; C; 0280; # LATIN LETTER YR +01A7; C; 01A8; # LATIN CAPITAL LETTER TONE TWO +01A9; C; 0283; # LATIN CAPITAL LETTER ESH +01AC; C; 01AD; # LATIN CAPITAL LETTER T WITH HOOK +01AE; C; 0288; # LATIN CAPITAL LETTER T WITH RETROFLEX HOOK +01AF; C; 01B0; # LATIN CAPITAL LETTER U WITH HORN +01B1; C; 028A; # LATIN CAPITAL LETTER UPSILON +01B2; C; 028B; # LATIN CAPITAL LETTER V WITH HOOK +01B3; C; 01B4; # LATIN CAPITAL LETTER Y WITH HOOK +01B5; C; 01B6; # LATIN CAPITAL LETTER Z WITH STROKE +01B7; C; 0292; # LATIN CAPITAL LETTER EZH +01B8; C; 01B9; # LATIN CAPITAL LETTER EZH REVERSED +01BC; C; 01BD; # LATIN CAPITAL LETTER TONE FIVE +01C4; C; 01C6; # LATIN CAPITAL LETTER DZ WITH CARON +01C5; C; 01C6; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON +01C7; C; 01C9; # LATIN CAPITAL LETTER LJ +01C8; C; 01C9; # LATIN CAPITAL LETTER L WITH SMALL LETTER J +01CA; C; 01CC; # LATIN CAPITAL LETTER NJ +01CB; C; 01CC; # LATIN CAPITAL LETTER N WITH SMALL LETTER J +01CD; C; 01CE; # LATIN CAPITAL LETTER A WITH CARON +01CF; C; 01D0; # LATIN CAPITAL LETTER I WITH CARON +01D1; C; 01D2; # LATIN CAPITAL LETTER O WITH CARON +01D3; C; 01D4; # LATIN CAPITAL LETTER U WITH CARON +01D5; C; 01D6; # LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON +01D7; C; 01D8; # LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE +01D9; C; 01DA; # LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON +01DB; C; 01DC; # LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE +01DE; C; 01DF; # LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON +01E0; C; 01E1; # LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON +01E2; C; 01E3; # LATIN CAPITAL LETTER AE WITH MACRON +01E4; C; 01E5; # LATIN CAPITAL LETTER G WITH STROKE +01E6; C; 01E7; # LATIN CAPITAL LETTER G WITH CARON +01E8; C; 01E9; # LATIN CAPITAL LETTER K WITH CARON +01EA; C; 01EB; # LATIN CAPITAL LETTER O WITH OGONEK +01EC; C; 01ED; # LATIN CAPITAL LETTER O WITH OGONEK AND MACRON +01EE; C; 01EF; # LATIN CAPITAL LETTER EZH WITH CARON +01F0; F; 006A 030C; # LATIN SMALL LETTER J WITH CARON +01F1; C; 01F3; # LATIN CAPITAL LETTER DZ +01F2; C; 01F3; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z +01F4; C; 01F5; # LATIN CAPITAL LETTER G WITH ACUTE +01F6; C; 0195; # LATIN CAPITAL LETTER HWAIR +01F7; C; 01BF; # LATIN CAPITAL LETTER WYNN +01F8; C; 01F9; # LATIN CAPITAL LETTER N WITH GRAVE +01FA; C; 01FB; # LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE +01FC; C; 01FD; # LATIN CAPITAL LETTER AE WITH ACUTE +01FE; C; 01FF; # LATIN CAPITAL LETTER O WITH STROKE AND ACUTE +0200; C; 0201; # LATIN CAPITAL LETTER A WITH DOUBLE GRAVE +0202; C; 0203; # LATIN CAPITAL LETTER A WITH INVERTED BREVE +0204; C; 0205; # LATIN CAPITAL LETTER E WITH DOUBLE GRAVE +0206; C; 0207; # LATIN CAPITAL LETTER E WITH INVERTED BREVE +0208; C; 0209; # LATIN CAPITAL LETTER I WITH DOUBLE GRAVE +020A; C; 020B; # LATIN CAPITAL LETTER I WITH INVERTED BREVE +020C; C; 020D; # LATIN CAPITAL LETTER O WITH DOUBLE GRAVE +020E; C; 020F; # LATIN CAPITAL LETTER O WITH INVERTED BREVE +0210; C; 0211; # LATIN CAPITAL LETTER R WITH DOUBLE GRAVE +0212; C; 0213; # LATIN CAPITAL LETTER R WITH INVERTED BREVE +0214; C; 0215; # LATIN CAPITAL LETTER U WITH DOUBLE GRAVE +0216; C; 0217; # LATIN CAPITAL LETTER U WITH INVERTED BREVE +0218; C; 0219; # LATIN CAPITAL LETTER S WITH COMMA BELOW +021A; C; 021B; # LATIN CAPITAL LETTER T WITH COMMA BELOW +021C; C; 021D; # LATIN CAPITAL LETTER YOGH +021E; C; 021F; # LATIN CAPITAL LETTER H WITH CARON +0220; C; 019E; # LATIN CAPITAL LETTER N WITH LONG RIGHT LEG +0222; C; 0223; # LATIN CAPITAL LETTER OU +0224; C; 0225; # LATIN CAPITAL LETTER Z WITH HOOK +0226; C; 0227; # LATIN CAPITAL LETTER A WITH DOT ABOVE +0228; C; 0229; # LATIN CAPITAL LETTER E WITH CEDILLA +022A; C; 022B; # LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON +022C; C; 022D; # LATIN CAPITAL LETTER O WITH TILDE AND MACRON +022E; C; 022F; # LATIN CAPITAL LETTER O WITH DOT ABOVE +0230; C; 0231; # LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON +0232; C; 0233; # LATIN CAPITAL LETTER Y WITH MACRON +023A; C; 2C65; # LATIN CAPITAL LETTER A WITH STROKE +023B; C; 023C; # LATIN CAPITAL LETTER C WITH STROKE +023D; C; 019A; # LATIN CAPITAL LETTER L WITH BAR +023E; C; 2C66; # LATIN CAPITAL LETTER T WITH DIAGONAL STROKE +0241; C; 0242; # LATIN CAPITAL LETTER GLOTTAL STOP +0243; C; 0180; # LATIN CAPITAL LETTER B WITH STROKE +0244; C; 0289; # LATIN CAPITAL LETTER U BAR +0245; C; 028C; # LATIN CAPITAL LETTER TURNED V +0246; C; 0247; # LATIN CAPITAL LETTER E WITH STROKE +0248; C; 0249; # LATIN CAPITAL LETTER J WITH STROKE +024A; C; 024B; # LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL +024C; C; 024D; # LATIN CAPITAL LETTER R WITH STROKE +024E; C; 024F; # LATIN CAPITAL LETTER Y WITH STROKE +0345; C; 03B9; # COMBINING GREEK YPOGEGRAMMENI +0370; C; 0371; # GREEK CAPITAL LETTER HETA +0372; C; 0373; # GREEK CAPITAL LETTER ARCHAIC SAMPI +0376; C; 0377; # GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA +037F; C; 03F3; # GREEK CAPITAL LETTER YOT +0386; C; 03AC; # GREEK CAPITAL LETTER ALPHA WITH TONOS +0388; C; 03AD; # GREEK CAPITAL LETTER EPSILON WITH TONOS +0389; C; 03AE; # GREEK CAPITAL LETTER ETA WITH TONOS +038A; C; 03AF; # GREEK CAPITAL LETTER IOTA WITH TONOS +038C; C; 03CC; # GREEK CAPITAL LETTER OMICRON WITH TONOS +038E; C; 03CD; # GREEK CAPITAL LETTER UPSILON WITH TONOS +038F; C; 03CE; # GREEK CAPITAL LETTER OMEGA WITH TONOS +0390; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS +0391; C; 03B1; # GREEK CAPITAL LETTER ALPHA +0392; C; 03B2; # GREEK CAPITAL LETTER BETA +0393; C; 03B3; # GREEK CAPITAL LETTER GAMMA +0394; C; 03B4; # GREEK CAPITAL LETTER DELTA +0395; C; 03B5; # GREEK CAPITAL LETTER EPSILON +0396; C; 03B6; # GREEK CAPITAL LETTER ZETA +0397; C; 03B7; # GREEK CAPITAL LETTER ETA +0398; C; 03B8; # GREEK CAPITAL LETTER THETA +0399; C; 03B9; # GREEK CAPITAL LETTER IOTA +039A; C; 03BA; # GREEK CAPITAL LETTER KAPPA +039B; C; 03BB; # GREEK CAPITAL LETTER LAMDA +039C; C; 03BC; # GREEK CAPITAL LETTER MU +039D; C; 03BD; # GREEK CAPITAL LETTER NU +039E; C; 03BE; # GREEK CAPITAL LETTER XI +039F; C; 03BF; # GREEK CAPITAL LETTER OMICRON +03A0; C; 03C0; # GREEK CAPITAL LETTER PI +03A1; C; 03C1; # GREEK CAPITAL LETTER RHO +03A3; C; 03C3; # GREEK CAPITAL LETTER SIGMA +03A4; C; 03C4; # GREEK CAPITAL LETTER TAU +03A5; C; 03C5; # GREEK CAPITAL LETTER UPSILON +03A6; C; 03C6; # GREEK CAPITAL LETTER PHI +03A7; C; 03C7; # GREEK CAPITAL LETTER CHI +03A8; C; 03C8; # GREEK CAPITAL LETTER PSI +03A9; C; 03C9; # GREEK CAPITAL LETTER OMEGA +03AA; C; 03CA; # GREEK CAPITAL LETTER IOTA WITH DIALYTIKA +03AB; C; 03CB; # GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA +03B0; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS +03C2; C; 03C3; # GREEK SMALL LETTER FINAL SIGMA +03CF; C; 03D7; # GREEK CAPITAL KAI SYMBOL +03D0; C; 03B2; # GREEK BETA SYMBOL +03D1; C; 03B8; # GREEK THETA SYMBOL +03D5; C; 03C6; # GREEK PHI SYMBOL +03D6; C; 03C0; # GREEK PI SYMBOL +03D8; C; 03D9; # GREEK LETTER ARCHAIC KOPPA +03DA; C; 03DB; # GREEK LETTER STIGMA +03DC; C; 03DD; # GREEK LETTER DIGAMMA +03DE; C; 03DF; # GREEK LETTER KOPPA +03E0; C; 03E1; # GREEK LETTER SAMPI +03E2; C; 03E3; # COPTIC CAPITAL LETTER SHEI +03E4; C; 03E5; # COPTIC CAPITAL LETTER FEI +03E6; C; 03E7; # COPTIC CAPITAL LETTER KHEI +03E8; C; 03E9; # COPTIC CAPITAL LETTER HORI +03EA; C; 03EB; # COPTIC CAPITAL LETTER GANGIA +03EC; C; 03ED; # COPTIC CAPITAL LETTER SHIMA +03EE; C; 03EF; # COPTIC CAPITAL LETTER DEI +03F0; C; 03BA; # GREEK KAPPA SYMBOL +03F1; C; 03C1; # GREEK RHO SYMBOL +03F4; C; 03B8; # GREEK CAPITAL THETA SYMBOL +03F5; C; 03B5; # GREEK LUNATE EPSILON SYMBOL +03F7; C; 03F8; # GREEK CAPITAL LETTER SHO +03F9; C; 03F2; # GREEK CAPITAL LUNATE SIGMA SYMBOL +03FA; C; 03FB; # GREEK CAPITAL LETTER SAN +03FD; C; 037B; # GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL +03FE; C; 037C; # GREEK CAPITAL DOTTED LUNATE SIGMA SYMBOL +03FF; C; 037D; # GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL +0400; C; 0450; # CYRILLIC CAPITAL LETTER IE WITH GRAVE +0401; C; 0451; # CYRILLIC CAPITAL LETTER IO +0402; C; 0452; # CYRILLIC CAPITAL LETTER DJE +0403; C; 0453; # CYRILLIC CAPITAL LETTER GJE +0404; C; 0454; # CYRILLIC CAPITAL LETTER UKRAINIAN IE +0405; C; 0455; # CYRILLIC CAPITAL LETTER DZE +0406; C; 0456; # CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I +0407; C; 0457; # CYRILLIC CAPITAL LETTER YI +0408; C; 0458; # CYRILLIC CAPITAL LETTER JE +0409; C; 0459; # CYRILLIC CAPITAL LETTER LJE +040A; C; 045A; # CYRILLIC CAPITAL LETTER NJE +040B; C; 045B; # CYRILLIC CAPITAL LETTER TSHE +040C; C; 045C; # CYRILLIC CAPITAL LETTER KJE +040D; C; 045D; # CYRILLIC CAPITAL LETTER I WITH GRAVE +040E; C; 045E; # CYRILLIC CAPITAL LETTER SHORT U +040F; C; 045F; # CYRILLIC CAPITAL LETTER DZHE +0410; C; 0430; # CYRILLIC CAPITAL LETTER A +0411; C; 0431; # CYRILLIC CAPITAL LETTER BE +0412; C; 0432; # CYRILLIC CAPITAL LETTER VE +0413; C; 0433; # CYRILLIC CAPITAL LETTER GHE +0414; C; 0434; # CYRILLIC CAPITAL LETTER DE +0415; C; 0435; # CYRILLIC CAPITAL LETTER IE +0416; C; 0436; # CYRILLIC CAPITAL LETTER ZHE +0417; C; 0437; # CYRILLIC CAPITAL LETTER ZE +0418; C; 0438; # CYRILLIC CAPITAL LETTER I +0419; C; 0439; # CYRILLIC CAPITAL LETTER SHORT I +041A; C; 043A; # CYRILLIC CAPITAL LETTER KA +041B; C; 043B; # CYRILLIC CAPITAL LETTER EL +041C; C; 043C; # CYRILLIC CAPITAL LETTER EM +041D; C; 043D; # CYRILLIC CAPITAL LETTER EN +041E; C; 043E; # CYRILLIC CAPITAL LETTER O +041F; C; 043F; # CYRILLIC CAPITAL LETTER PE +0420; C; 0440; # CYRILLIC CAPITAL LETTER ER +0421; C; 0441; # CYRILLIC CAPITAL LETTER ES +0422; C; 0442; # CYRILLIC CAPITAL LETTER TE +0423; C; 0443; # CYRILLIC CAPITAL LETTER U +0424; C; 0444; # CYRILLIC CAPITAL LETTER EF +0425; C; 0445; # CYRILLIC CAPITAL LETTER HA +0426; C; 0446; # CYRILLIC CAPITAL LETTER TSE +0427; C; 0447; # CYRILLIC CAPITAL LETTER CHE +0428; C; 0448; # CYRILLIC CAPITAL LETTER SHA +0429; C; 0449; # CYRILLIC CAPITAL LETTER SHCHA +042A; C; 044A; # CYRILLIC CAPITAL LETTER HARD SIGN +042B; C; 044B; # CYRILLIC CAPITAL LETTER YERU +042C; C; 044C; # CYRILLIC CAPITAL LETTER SOFT SIGN +042D; C; 044D; # CYRILLIC CAPITAL LETTER E +042E; C; 044E; # CYRILLIC CAPITAL LETTER YU +042F; C; 044F; # CYRILLIC CAPITAL LETTER YA +0460; C; 0461; # CYRILLIC CAPITAL LETTER OMEGA +0462; C; 0463; # CYRILLIC CAPITAL LETTER YAT +0464; C; 0465; # CYRILLIC CAPITAL LETTER IOTIFIED E +0466; C; 0467; # CYRILLIC CAPITAL LETTER LITTLE YUS +0468; C; 0469; # CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS +046A; C; 046B; # CYRILLIC CAPITAL LETTER BIG YUS +046C; C; 046D; # CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS +046E; C; 046F; # CYRILLIC CAPITAL LETTER KSI +0470; C; 0471; # CYRILLIC CAPITAL LETTER PSI +0472; C; 0473; # CYRILLIC CAPITAL LETTER FITA +0474; C; 0475; # CYRILLIC CAPITAL LETTER IZHITSA +0476; C; 0477; # CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT +0478; C; 0479; # CYRILLIC CAPITAL LETTER UK +047A; C; 047B; # CYRILLIC CAPITAL LETTER ROUND OMEGA +047C; C; 047D; # CYRILLIC CAPITAL LETTER OMEGA WITH TITLO +047E; C; 047F; # CYRILLIC CAPITAL LETTER OT +0480; C; 0481; # CYRILLIC CAPITAL LETTER KOPPA +048A; C; 048B; # CYRILLIC CAPITAL LETTER SHORT I WITH TAIL +048C; C; 048D; # CYRILLIC CAPITAL LETTER SEMISOFT SIGN +048E; C; 048F; # CYRILLIC CAPITAL LETTER ER WITH TICK +0490; C; 0491; # CYRILLIC CAPITAL LETTER GHE WITH UPTURN +0492; C; 0493; # CYRILLIC CAPITAL LETTER GHE WITH STROKE +0494; C; 0495; # CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK +0496; C; 0497; # CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER +0498; C; 0499; # CYRILLIC CAPITAL LETTER ZE WITH DESCENDER +049A; C; 049B; # CYRILLIC CAPITAL LETTER KA WITH DESCENDER +049C; C; 049D; # CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE +049E; C; 049F; # CYRILLIC CAPITAL LETTER KA WITH STROKE +04A0; C; 04A1; # CYRILLIC CAPITAL LETTER BASHKIR KA +04A2; C; 04A3; # CYRILLIC CAPITAL LETTER EN WITH DESCENDER +04A4; C; 04A5; # CYRILLIC CAPITAL LIGATURE EN GHE +04A6; C; 04A7; # CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK +04A8; C; 04A9; # CYRILLIC CAPITAL LETTER ABKHASIAN HA +04AA; C; 04AB; # CYRILLIC CAPITAL LETTER ES WITH DESCENDER +04AC; C; 04AD; # CYRILLIC CAPITAL LETTER TE WITH DESCENDER +04AE; C; 04AF; # CYRILLIC CAPITAL LETTER STRAIGHT U +04B0; C; 04B1; # CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE +04B2; C; 04B3; # CYRILLIC CAPITAL LETTER HA WITH DESCENDER +04B4; C; 04B5; # CYRILLIC CAPITAL LIGATURE TE TSE +04B6; C; 04B7; # CYRILLIC CAPITAL LETTER CHE WITH DESCENDER +04B8; C; 04B9; # CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE +04BA; C; 04BB; # CYRILLIC CAPITAL LETTER SHHA +04BC; C; 04BD; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE +04BE; C; 04BF; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER +04C0; C; 04CF; # CYRILLIC LETTER PALOCHKA +04C1; C; 04C2; # CYRILLIC CAPITAL LETTER ZHE WITH BREVE +04C3; C; 04C4; # CYRILLIC CAPITAL LETTER KA WITH HOOK +04C5; C; 04C6; # CYRILLIC CAPITAL LETTER EL WITH TAIL +04C7; C; 04C8; # CYRILLIC CAPITAL LETTER EN WITH HOOK +04C9; C; 04CA; # CYRILLIC CAPITAL LETTER EN WITH TAIL +04CB; C; 04CC; # CYRILLIC CAPITAL LETTER KHAKASSIAN CHE +04CD; C; 04CE; # CYRILLIC CAPITAL LETTER EM WITH TAIL +04D0; C; 04D1; # CYRILLIC CAPITAL LETTER A WITH BREVE +04D2; C; 04D3; # CYRILLIC CAPITAL LETTER A WITH DIAERESIS +04D4; C; 04D5; # CYRILLIC CAPITAL LIGATURE A IE +04D6; C; 04D7; # CYRILLIC CAPITAL LETTER IE WITH BREVE +04D8; C; 04D9; # CYRILLIC CAPITAL LETTER SCHWA +04DA; C; 04DB; # CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS +04DC; C; 04DD; # CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS +04DE; C; 04DF; # CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS +04E0; C; 04E1; # CYRILLIC CAPITAL LETTER ABKHASIAN DZE +04E2; C; 04E3; # CYRILLIC CAPITAL LETTER I WITH MACRON +04E4; C; 04E5; # CYRILLIC CAPITAL LETTER I WITH DIAERESIS +04E6; C; 04E7; # CYRILLIC CAPITAL LETTER O WITH DIAERESIS +04E8; C; 04E9; # CYRILLIC CAPITAL LETTER BARRED O +04EA; C; 04EB; # CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS +04EC; C; 04ED; # CYRILLIC CAPITAL LETTER E WITH DIAERESIS +04EE; C; 04EF; # CYRILLIC CAPITAL LETTER U WITH MACRON +04F0; C; 04F1; # CYRILLIC CAPITAL LETTER U WITH DIAERESIS +04F2; C; 04F3; # CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE +04F4; C; 04F5; # CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS +04F6; C; 04F7; # CYRILLIC CAPITAL LETTER GHE WITH DESCENDER +04F8; C; 04F9; # CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS +04FA; C; 04FB; # CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK +04FC; C; 04FD; # CYRILLIC CAPITAL LETTER HA WITH HOOK +04FE; C; 04FF; # CYRILLIC CAPITAL LETTER HA WITH STROKE +0500; C; 0501; # CYRILLIC CAPITAL LETTER KOMI DE +0502; C; 0503; # CYRILLIC CAPITAL LETTER KOMI DJE +0504; C; 0505; # CYRILLIC CAPITAL LETTER KOMI ZJE +0506; C; 0507; # CYRILLIC CAPITAL LETTER KOMI DZJE +0508; C; 0509; # CYRILLIC CAPITAL LETTER KOMI LJE +050A; C; 050B; # CYRILLIC CAPITAL LETTER KOMI NJE +050C; C; 050D; # CYRILLIC CAPITAL LETTER KOMI SJE +050E; C; 050F; # CYRILLIC CAPITAL LETTER KOMI TJE +0510; C; 0511; # CYRILLIC CAPITAL LETTER REVERSED ZE +0512; C; 0513; # CYRILLIC CAPITAL LETTER EL WITH HOOK +0514; C; 0515; # CYRILLIC CAPITAL LETTER LHA +0516; C; 0517; # CYRILLIC CAPITAL LETTER RHA +0518; C; 0519; # CYRILLIC CAPITAL LETTER YAE +051A; C; 051B; # CYRILLIC CAPITAL LETTER QA +051C; C; 051D; # CYRILLIC CAPITAL LETTER WE +051E; C; 051F; # CYRILLIC CAPITAL LETTER ALEUT KA +0520; C; 0521; # CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK +0522; C; 0523; # CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK +0524; C; 0525; # CYRILLIC CAPITAL LETTER PE WITH DESCENDER +0526; C; 0527; # CYRILLIC CAPITAL LETTER SHHA WITH DESCENDER +0528; C; 0529; # CYRILLIC CAPITAL LETTER EN WITH LEFT HOOK +052A; C; 052B; # CYRILLIC CAPITAL LETTER DZZHE +052C; C; 052D; # CYRILLIC CAPITAL LETTER DCHE +052E; C; 052F; # CYRILLIC CAPITAL LETTER EL WITH DESCENDER +0531; C; 0561; # ARMENIAN CAPITAL LETTER AYB +0532; C; 0562; # ARMENIAN CAPITAL LETTER BEN +0533; C; 0563; # ARMENIAN CAPITAL LETTER GIM +0534; C; 0564; # ARMENIAN CAPITAL LETTER DA +0535; C; 0565; # ARMENIAN CAPITAL LETTER ECH +0536; C; 0566; # ARMENIAN CAPITAL LETTER ZA +0537; C; 0567; # ARMENIAN CAPITAL LETTER EH +0538; C; 0568; # ARMENIAN CAPITAL LETTER ET +0539; C; 0569; # ARMENIAN CAPITAL LETTER TO +053A; C; 056A; # ARMENIAN CAPITAL LETTER ZHE +053B; C; 056B; # ARMENIAN CAPITAL LETTER INI +053C; C; 056C; # ARMENIAN CAPITAL LETTER LIWN +053D; C; 056D; # ARMENIAN CAPITAL LETTER XEH +053E; C; 056E; # ARMENIAN CAPITAL LETTER CA +053F; C; 056F; # ARMENIAN CAPITAL LETTER KEN +0540; C; 0570; # ARMENIAN CAPITAL LETTER HO +0541; C; 0571; # ARMENIAN CAPITAL LETTER JA +0542; C; 0572; # ARMENIAN CAPITAL LETTER GHAD +0543; C; 0573; # ARMENIAN CAPITAL LETTER CHEH +0544; C; 0574; # ARMENIAN CAPITAL LETTER MEN +0545; C; 0575; # ARMENIAN CAPITAL LETTER YI +0546; C; 0576; # ARMENIAN CAPITAL LETTER NOW +0547; C; 0577; # ARMENIAN CAPITAL LETTER SHA +0548; C; 0578; # ARMENIAN CAPITAL LETTER VO +0549; C; 0579; # ARMENIAN CAPITAL LETTER CHA +054A; C; 057A; # ARMENIAN CAPITAL LETTER PEH +054B; C; 057B; # ARMENIAN CAPITAL LETTER JHEH +054C; C; 057C; # ARMENIAN CAPITAL LETTER RA +054D; C; 057D; # ARMENIAN CAPITAL LETTER SEH +054E; C; 057E; # ARMENIAN CAPITAL LETTER VEW +054F; C; 057F; # ARMENIAN CAPITAL LETTER TIWN +0550; C; 0580; # ARMENIAN CAPITAL LETTER REH +0551; C; 0581; # ARMENIAN CAPITAL LETTER CO +0552; C; 0582; # ARMENIAN CAPITAL LETTER YIWN +0553; C; 0583; # ARMENIAN CAPITAL LETTER PIWR +0554; C; 0584; # ARMENIAN CAPITAL LETTER KEH +0555; C; 0585; # ARMENIAN CAPITAL LETTER OH +0556; C; 0586; # ARMENIAN CAPITAL LETTER FEH +0587; F; 0565 0582; # ARMENIAN SMALL LIGATURE ECH YIWN +10A0; C; 2D00; # GEORGIAN CAPITAL LETTER AN +10A1; C; 2D01; # GEORGIAN CAPITAL LETTER BAN +10A2; C; 2D02; # GEORGIAN CAPITAL LETTER GAN +10A3; C; 2D03; # GEORGIAN CAPITAL LETTER DON +10A4; C; 2D04; # GEORGIAN CAPITAL LETTER EN +10A5; C; 2D05; # GEORGIAN CAPITAL LETTER VIN +10A6; C; 2D06; # GEORGIAN CAPITAL LETTER ZEN +10A7; C; 2D07; # GEORGIAN CAPITAL LETTER TAN +10A8; C; 2D08; # GEORGIAN CAPITAL LETTER IN +10A9; C; 2D09; # GEORGIAN CAPITAL LETTER KAN +10AA; C; 2D0A; # GEORGIAN CAPITAL LETTER LAS +10AB; C; 2D0B; # GEORGIAN CAPITAL LETTER MAN +10AC; C; 2D0C; # GEORGIAN CAPITAL LETTER NAR +10AD; C; 2D0D; # GEORGIAN CAPITAL LETTER ON +10AE; C; 2D0E; # GEORGIAN CAPITAL LETTER PAR +10AF; C; 2D0F; # GEORGIAN CAPITAL LETTER ZHAR +10B0; C; 2D10; # GEORGIAN CAPITAL LETTER RAE +10B1; C; 2D11; # GEORGIAN CAPITAL LETTER SAN +10B2; C; 2D12; # GEORGIAN CAPITAL LETTER TAR +10B3; C; 2D13; # GEORGIAN CAPITAL LETTER UN +10B4; C; 2D14; # GEORGIAN CAPITAL LETTER PHAR +10B5; C; 2D15; # GEORGIAN CAPITAL LETTER KHAR +10B6; C; 2D16; # GEORGIAN CAPITAL LETTER GHAN +10B7; C; 2D17; # GEORGIAN CAPITAL LETTER QAR +10B8; C; 2D18; # GEORGIAN CAPITAL LETTER SHIN +10B9; C; 2D19; # GEORGIAN CAPITAL LETTER CHIN +10BA; C; 2D1A; # GEORGIAN CAPITAL LETTER CAN +10BB; C; 2D1B; # GEORGIAN CAPITAL LETTER JIL +10BC; C; 2D1C; # GEORGIAN CAPITAL LETTER CIL +10BD; C; 2D1D; # GEORGIAN CAPITAL LETTER CHAR +10BE; C; 2D1E; # GEORGIAN CAPITAL LETTER XAN +10BF; C; 2D1F; # GEORGIAN CAPITAL LETTER JHAN +10C0; C; 2D20; # GEORGIAN CAPITAL LETTER HAE +10C1; C; 2D21; # GEORGIAN CAPITAL LETTER HE +10C2; C; 2D22; # GEORGIAN CAPITAL LETTER HIE +10C3; C; 2D23; # GEORGIAN CAPITAL LETTER WE +10C4; C; 2D24; # GEORGIAN CAPITAL LETTER HAR +10C5; C; 2D25; # GEORGIAN CAPITAL LETTER HOE +10C7; C; 2D27; # GEORGIAN CAPITAL LETTER YN +10CD; C; 2D2D; # GEORGIAN CAPITAL LETTER AEN +13F8; C; 13F0; # CHEROKEE SMALL LETTER YE +13F9; C; 13F1; # CHEROKEE SMALL LETTER YI +13FA; C; 13F2; # CHEROKEE SMALL LETTER YO +13FB; C; 13F3; # CHEROKEE SMALL LETTER YU +13FC; C; 13F4; # CHEROKEE SMALL LETTER YV +13FD; C; 13F5; # CHEROKEE SMALL LETTER MV +1C80; C; 0432; # CYRILLIC SMALL LETTER ROUNDED VE +1C81; C; 0434; # CYRILLIC SMALL LETTER LONG-LEGGED DE +1C82; C; 043E; # CYRILLIC SMALL LETTER NARROW O +1C83; C; 0441; # CYRILLIC SMALL LETTER WIDE ES +1C84; C; 0442; # CYRILLIC SMALL LETTER TALL TE +1C85; C; 0442; # CYRILLIC SMALL LETTER THREE-LEGGED TE +1C86; C; 044A; # CYRILLIC SMALL LETTER TALL HARD SIGN +1C87; C; 0463; # CYRILLIC SMALL LETTER TALL YAT +1C88; C; A64B; # CYRILLIC SMALL LETTER UNBLENDED UK +1C90; C; 10D0; # GEORGIAN MTAVRULI CAPITAL LETTER AN +1C91; C; 10D1; # GEORGIAN MTAVRULI CAPITAL LETTER BAN +1C92; C; 10D2; # GEORGIAN MTAVRULI CAPITAL LETTER GAN +1C93; C; 10D3; # GEORGIAN MTAVRULI CAPITAL LETTER DON +1C94; C; 10D4; # GEORGIAN MTAVRULI CAPITAL LETTER EN +1C95; C; 10D5; # GEORGIAN MTAVRULI CAPITAL LETTER VIN +1C96; C; 10D6; # GEORGIAN MTAVRULI CAPITAL LETTER ZEN +1C97; C; 10D7; # GEORGIAN MTAVRULI CAPITAL LETTER TAN +1C98; C; 10D8; # GEORGIAN MTAVRULI CAPITAL LETTER IN +1C99; C; 10D9; # GEORGIAN MTAVRULI CAPITAL LETTER KAN +1C9A; C; 10DA; # GEORGIAN MTAVRULI CAPITAL LETTER LAS +1C9B; C; 10DB; # GEORGIAN MTAVRULI CAPITAL LETTER MAN +1C9C; C; 10DC; # GEORGIAN MTAVRULI CAPITAL LETTER NAR +1C9D; C; 10DD; # GEORGIAN MTAVRULI CAPITAL LETTER ON +1C9E; C; 10DE; # GEORGIAN MTAVRULI CAPITAL LETTER PAR +1C9F; C; 10DF; # GEORGIAN MTAVRULI CAPITAL LETTER ZHAR +1CA0; C; 10E0; # GEORGIAN MTAVRULI CAPITAL LETTER RAE +1CA1; C; 10E1; # GEORGIAN MTAVRULI CAPITAL LETTER SAN +1CA2; C; 10E2; # GEORGIAN MTAVRULI CAPITAL LETTER TAR +1CA3; C; 10E3; # GEORGIAN MTAVRULI CAPITAL LETTER UN +1CA4; C; 10E4; # GEORGIAN MTAVRULI CAPITAL LETTER PHAR +1CA5; C; 10E5; # GEORGIAN MTAVRULI CAPITAL LETTER KHAR +1CA6; C; 10E6; # GEORGIAN MTAVRULI CAPITAL LETTER GHAN +1CA7; C; 10E7; # GEORGIAN MTAVRULI CAPITAL LETTER QAR +1CA8; C; 10E8; # GEORGIAN MTAVRULI CAPITAL LETTER SHIN +1CA9; C; 10E9; # GEORGIAN MTAVRULI CAPITAL LETTER CHIN +1CAA; C; 10EA; # GEORGIAN MTAVRULI CAPITAL LETTER CAN +1CAB; C; 10EB; # GEORGIAN MTAVRULI CAPITAL LETTER JIL +1CAC; C; 10EC; # GEORGIAN MTAVRULI CAPITAL LETTER CIL +1CAD; C; 10ED; # GEORGIAN MTAVRULI CAPITAL LETTER CHAR +1CAE; C; 10EE; # GEORGIAN MTAVRULI CAPITAL LETTER XAN +1CAF; C; 10EF; # GEORGIAN MTAVRULI CAPITAL LETTER JHAN +1CB0; C; 10F0; # GEORGIAN MTAVRULI CAPITAL LETTER HAE +1CB1; C; 10F1; # GEORGIAN MTAVRULI CAPITAL LETTER HE +1CB2; C; 10F2; # GEORGIAN MTAVRULI CAPITAL LETTER HIE +1CB3; C; 10F3; # GEORGIAN MTAVRULI CAPITAL LETTER WE +1CB4; C; 10F4; # GEORGIAN MTAVRULI CAPITAL LETTER HAR +1CB5; C; 10F5; # GEORGIAN MTAVRULI CAPITAL LETTER HOE +1CB6; C; 10F6; # GEORGIAN MTAVRULI CAPITAL LETTER FI +1CB7; C; 10F7; # GEORGIAN MTAVRULI CAPITAL LETTER YN +1CB8; C; 10F8; # GEORGIAN MTAVRULI CAPITAL LETTER ELIFI +1CB9; C; 10F9; # GEORGIAN MTAVRULI CAPITAL LETTER TURNED GAN +1CBA; C; 10FA; # GEORGIAN MTAVRULI CAPITAL LETTER AIN +1CBD; C; 10FD; # GEORGIAN MTAVRULI CAPITAL LETTER AEN +1CBE; C; 10FE; # GEORGIAN MTAVRULI CAPITAL LETTER HARD SIGN +1CBF; C; 10FF; # GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN +1E00; C; 1E01; # LATIN CAPITAL LETTER A WITH RING BELOW +1E02; C; 1E03; # LATIN CAPITAL LETTER B WITH DOT ABOVE +1E04; C; 1E05; # LATIN CAPITAL LETTER B WITH DOT BELOW +1E06; C; 1E07; # LATIN CAPITAL LETTER B WITH LINE BELOW +1E08; C; 1E09; # LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE +1E0A; C; 1E0B; # LATIN CAPITAL LETTER D WITH DOT ABOVE +1E0C; C; 1E0D; # LATIN CAPITAL LETTER D WITH DOT BELOW +1E0E; C; 1E0F; # LATIN CAPITAL LETTER D WITH LINE BELOW +1E10; C; 1E11; # LATIN CAPITAL LETTER D WITH CEDILLA +1E12; C; 1E13; # LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW +1E14; C; 1E15; # LATIN CAPITAL LETTER E WITH MACRON AND GRAVE +1E16; C; 1E17; # LATIN CAPITAL LETTER E WITH MACRON AND ACUTE +1E18; C; 1E19; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW +1E1A; C; 1E1B; # LATIN CAPITAL LETTER E WITH TILDE BELOW +1E1C; C; 1E1D; # LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE +1E1E; C; 1E1F; # LATIN CAPITAL LETTER F WITH DOT ABOVE +1E20; C; 1E21; # LATIN CAPITAL LETTER G WITH MACRON +1E22; C; 1E23; # LATIN CAPITAL LETTER H WITH DOT ABOVE +1E24; C; 1E25; # LATIN CAPITAL LETTER H WITH DOT BELOW +1E26; C; 1E27; # LATIN CAPITAL LETTER H WITH DIAERESIS +1E28; C; 1E29; # LATIN CAPITAL LETTER H WITH CEDILLA +1E2A; C; 1E2B; # LATIN CAPITAL LETTER H WITH BREVE BELOW +1E2C; C; 1E2D; # LATIN CAPITAL LETTER I WITH TILDE BELOW +1E2E; C; 1E2F; # LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE +1E30; C; 1E31; # LATIN CAPITAL LETTER K WITH ACUTE +1E32; C; 1E33; # LATIN CAPITAL LETTER K WITH DOT BELOW +1E34; C; 1E35; # LATIN CAPITAL LETTER K WITH LINE BELOW +1E36; C; 1E37; # LATIN CAPITAL LETTER L WITH DOT BELOW +1E38; C; 1E39; # LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON +1E3A; C; 1E3B; # LATIN CAPITAL LETTER L WITH LINE BELOW +1E3C; C; 1E3D; # LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW +1E3E; C; 1E3F; # LATIN CAPITAL LETTER M WITH ACUTE +1E40; C; 1E41; # LATIN CAPITAL LETTER M WITH DOT ABOVE +1E42; C; 1E43; # LATIN CAPITAL LETTER M WITH DOT BELOW +1E44; C; 1E45; # LATIN CAPITAL LETTER N WITH DOT ABOVE +1E46; C; 1E47; # LATIN CAPITAL LETTER N WITH DOT BELOW +1E48; C; 1E49; # LATIN CAPITAL LETTER N WITH LINE BELOW +1E4A; C; 1E4B; # LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW +1E4C; C; 1E4D; # LATIN CAPITAL LETTER O WITH TILDE AND ACUTE +1E4E; C; 1E4F; # LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS +1E50; C; 1E51; # LATIN CAPITAL LETTER O WITH MACRON AND GRAVE +1E52; C; 1E53; # LATIN CAPITAL LETTER O WITH MACRON AND ACUTE +1E54; C; 1E55; # LATIN CAPITAL LETTER P WITH ACUTE +1E56; C; 1E57; # LATIN CAPITAL LETTER P WITH DOT ABOVE +1E58; C; 1E59; # LATIN CAPITAL LETTER R WITH DOT ABOVE +1E5A; C; 1E5B; # LATIN CAPITAL LETTER R WITH DOT BELOW +1E5C; C; 1E5D; # LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON +1E5E; C; 1E5F; # LATIN CAPITAL LETTER R WITH LINE BELOW +1E60; C; 1E61; # LATIN CAPITAL LETTER S WITH DOT ABOVE +1E62; C; 1E63; # LATIN CAPITAL LETTER S WITH DOT BELOW +1E64; C; 1E65; # LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE +1E66; C; 1E67; # LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE +1E68; C; 1E69; # LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE +1E6A; C; 1E6B; # LATIN CAPITAL LETTER T WITH DOT ABOVE +1E6C; C; 1E6D; # LATIN CAPITAL LETTER T WITH DOT BELOW +1E6E; C; 1E6F; # LATIN CAPITAL LETTER T WITH LINE BELOW +1E70; C; 1E71; # LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW +1E72; C; 1E73; # LATIN CAPITAL LETTER U WITH DIAERESIS BELOW +1E74; C; 1E75; # LATIN CAPITAL LETTER U WITH TILDE BELOW +1E76; C; 1E77; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW +1E78; C; 1E79; # LATIN CAPITAL LETTER U WITH TILDE AND ACUTE +1E7A; C; 1E7B; # LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS +1E7C; C; 1E7D; # LATIN CAPITAL LETTER V WITH TILDE +1E7E; C; 1E7F; # LATIN CAPITAL LETTER V WITH DOT BELOW +1E80; C; 1E81; # LATIN CAPITAL LETTER W WITH GRAVE +1E82; C; 1E83; # LATIN CAPITAL LETTER W WITH ACUTE +1E84; C; 1E85; # LATIN CAPITAL LETTER W WITH DIAERESIS +1E86; C; 1E87; # LATIN CAPITAL LETTER W WITH DOT ABOVE +1E88; C; 1E89; # LATIN CAPITAL LETTER W WITH DOT BELOW +1E8A; C; 1E8B; # LATIN CAPITAL LETTER X WITH DOT ABOVE +1E8C; C; 1E8D; # LATIN CAPITAL LETTER X WITH DIAERESIS +1E8E; C; 1E8F; # LATIN CAPITAL LETTER Y WITH DOT ABOVE +1E90; C; 1E91; # LATIN CAPITAL LETTER Z WITH CIRCUMFLEX +1E92; C; 1E93; # LATIN CAPITAL LETTER Z WITH DOT BELOW +1E94; C; 1E95; # LATIN CAPITAL LETTER Z WITH LINE BELOW +1E96; F; 0068 0331; # LATIN SMALL LETTER H WITH LINE BELOW +1E97; F; 0074 0308; # LATIN SMALL LETTER T WITH DIAERESIS +1E98; F; 0077 030A; # LATIN SMALL LETTER W WITH RING ABOVE +1E99; F; 0079 030A; # LATIN SMALL LETTER Y WITH RING ABOVE +1E9A; F; 0061 02BE; # LATIN SMALL LETTER A WITH RIGHT HALF RING +1E9B; C; 1E61; # LATIN SMALL LETTER LONG S WITH DOT ABOVE +1E9E; F; 0073 0073; # LATIN CAPITAL LETTER SHARP S +1E9E; S; 00DF; # LATIN CAPITAL LETTER SHARP S +1EA0; C; 1EA1; # LATIN CAPITAL LETTER A WITH DOT BELOW +1EA2; C; 1EA3; # LATIN CAPITAL LETTER A WITH HOOK ABOVE +1EA4; C; 1EA5; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE +1EA6; C; 1EA7; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE +1EA8; C; 1EA9; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE +1EAA; C; 1EAB; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE +1EAC; C; 1EAD; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW +1EAE; C; 1EAF; # LATIN CAPITAL LETTER A WITH BREVE AND ACUTE +1EB0; C; 1EB1; # LATIN CAPITAL LETTER A WITH BREVE AND GRAVE +1EB2; C; 1EB3; # LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE +1EB4; C; 1EB5; # LATIN CAPITAL LETTER A WITH BREVE AND TILDE +1EB6; C; 1EB7; # LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW +1EB8; C; 1EB9; # LATIN CAPITAL LETTER E WITH DOT BELOW +1EBA; C; 1EBB; # LATIN CAPITAL LETTER E WITH HOOK ABOVE +1EBC; C; 1EBD; # LATIN CAPITAL LETTER E WITH TILDE +1EBE; C; 1EBF; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE +1EC0; C; 1EC1; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE +1EC2; C; 1EC3; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE +1EC4; C; 1EC5; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE +1EC6; C; 1EC7; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW +1EC8; C; 1EC9; # LATIN CAPITAL LETTER I WITH HOOK ABOVE +1ECA; C; 1ECB; # LATIN CAPITAL LETTER I WITH DOT BELOW +1ECC; C; 1ECD; # LATIN CAPITAL LETTER O WITH DOT BELOW +1ECE; C; 1ECF; # LATIN CAPITAL LETTER O WITH HOOK ABOVE +1ED0; C; 1ED1; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE +1ED2; C; 1ED3; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE +1ED4; C; 1ED5; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE +1ED6; C; 1ED7; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE +1ED8; C; 1ED9; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW +1EDA; C; 1EDB; # LATIN CAPITAL LETTER O WITH HORN AND ACUTE +1EDC; C; 1EDD; # LATIN CAPITAL LETTER O WITH HORN AND GRAVE +1EDE; C; 1EDF; # LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE +1EE0; C; 1EE1; # LATIN CAPITAL LETTER O WITH HORN AND TILDE +1EE2; C; 1EE3; # LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW +1EE4; C; 1EE5; # LATIN CAPITAL LETTER U WITH DOT BELOW +1EE6; C; 1EE7; # LATIN CAPITAL LETTER U WITH HOOK ABOVE +1EE8; C; 1EE9; # LATIN CAPITAL LETTER U WITH HORN AND ACUTE +1EEA; C; 1EEB; # LATIN CAPITAL LETTER U WITH HORN AND GRAVE +1EEC; C; 1EED; # LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE +1EEE; C; 1EEF; # LATIN CAPITAL LETTER U WITH HORN AND TILDE +1EF0; C; 1EF1; # LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW +1EF2; C; 1EF3; # LATIN CAPITAL LETTER Y WITH GRAVE +1EF4; C; 1EF5; # LATIN CAPITAL LETTER Y WITH DOT BELOW +1EF6; C; 1EF7; # LATIN CAPITAL LETTER Y WITH HOOK ABOVE +1EF8; C; 1EF9; # LATIN CAPITAL LETTER Y WITH TILDE +1EFA; C; 1EFB; # LATIN CAPITAL LETTER MIDDLE-WELSH LL +1EFC; C; 1EFD; # LATIN CAPITAL LETTER MIDDLE-WELSH V +1EFE; C; 1EFF; # LATIN CAPITAL LETTER Y WITH LOOP +1F08; C; 1F00; # GREEK CAPITAL LETTER ALPHA WITH PSILI +1F09; C; 1F01; # GREEK CAPITAL LETTER ALPHA WITH DASIA +1F0A; C; 1F02; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA +1F0B; C; 1F03; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA +1F0C; C; 1F04; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA +1F0D; C; 1F05; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA +1F0E; C; 1F06; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI +1F0F; C; 1F07; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI +1F18; C; 1F10; # GREEK CAPITAL LETTER EPSILON WITH PSILI +1F19; C; 1F11; # GREEK CAPITAL LETTER EPSILON WITH DASIA +1F1A; C; 1F12; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA +1F1B; C; 1F13; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA +1F1C; C; 1F14; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA +1F1D; C; 1F15; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F28; C; 1F20; # GREEK CAPITAL LETTER ETA WITH PSILI +1F29; C; 1F21; # GREEK CAPITAL LETTER ETA WITH DASIA +1F2A; C; 1F22; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA +1F2B; C; 1F23; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA +1F2C; C; 1F24; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA +1F2D; C; 1F25; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA +1F2E; C; 1F26; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI +1F2F; C; 1F27; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI +1F38; C; 1F30; # GREEK CAPITAL LETTER IOTA WITH PSILI +1F39; C; 1F31; # GREEK CAPITAL LETTER IOTA WITH DASIA +1F3A; C; 1F32; # GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA +1F3B; C; 1F33; # GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA +1F3C; C; 1F34; # GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA +1F3D; C; 1F35; # GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA +1F3E; C; 1F36; # GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI +1F3F; C; 1F37; # GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI +1F48; C; 1F40; # GREEK CAPITAL LETTER OMICRON WITH PSILI +1F49; C; 1F41; # GREEK CAPITAL LETTER OMICRON WITH DASIA +1F4A; C; 1F42; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA +1F4B; C; 1F43; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA +1F4C; C; 1F44; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA +1F4D; C; 1F45; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F50; F; 03C5 0313; # GREEK SMALL LETTER UPSILON WITH PSILI +1F52; F; 03C5 0313 0300; # GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA +1F54; F; 03C5 0313 0301; # GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA +1F56; F; 03C5 0313 0342; # GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI +1F59; C; 1F51; # GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B; C; 1F53; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D; C; 1F55; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F; C; 1F57; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F68; C; 1F60; # GREEK CAPITAL LETTER OMEGA WITH PSILI +1F69; C; 1F61; # GREEK CAPITAL LETTER OMEGA WITH DASIA +1F6A; C; 1F62; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA +1F6B; C; 1F63; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA +1F6C; C; 1F64; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA +1F6D; C; 1F65; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA +1F6E; C; 1F66; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI +1F6F; C; 1F67; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI +1F80; F; 1F00 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI +1F81; F; 1F01 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI +1F82; F; 1F02 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI +1F83; F; 1F03 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI +1F84; F; 1F04 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI +1F85; F; 1F05 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI +1F86; F; 1F06 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI +1F87; F; 1F07 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1F88; F; 1F00 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI +1F88; S; 1F80; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI +1F89; F; 1F01 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI +1F89; S; 1F81; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI +1F8A; F; 1F02 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F8A; S; 1F82; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F8B; F; 1F03 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F8B; S; 1F83; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F8C; F; 1F04 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F8C; S; 1F84; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F8D; F; 1F05 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F8D; S; 1F85; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F8E; F; 1F06 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F8E; S; 1F86; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F8F; F; 1F07 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1F8F; S; 1F87; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1F90; F; 1F20 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI +1F91; F; 1F21 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI +1F92; F; 1F22 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI +1F93; F; 1F23 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI +1F94; F; 1F24 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI +1F95; F; 1F25 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI +1F96; F; 1F26 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI +1F97; F; 1F27 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1F98; F; 1F20 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI +1F98; S; 1F90; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI +1F99; F; 1F21 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI +1F99; S; 1F91; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI +1F9A; F; 1F22 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F9A; S; 1F92; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F9B; F; 1F23 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F9B; S; 1F93; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F9C; F; 1F24 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F9C; S; 1F94; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F9D; F; 1F25 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F9D; S; 1F95; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F9E; F; 1F26 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F9E; S; 1F96; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F9F; F; 1F27 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1F9F; S; 1F97; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FA0; F; 1F60 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI +1FA1; F; 1F61 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI +1FA2; F; 1F62 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI +1FA3; F; 1F63 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI +1FA4; F; 1F64 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI +1FA5; F; 1F65 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI +1FA6; F; 1F66 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI +1FA7; F; 1F67 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1FA8; F; 1F60 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI +1FA8; S; 1FA0; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI +1FA9; F; 1F61 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI +1FA9; S; 1FA1; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI +1FAA; F; 1F62 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1FAA; S; 1FA2; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1FAB; F; 1F63 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1FAB; S; 1FA3; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1FAC; F; 1F64 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1FAC; S; 1FA4; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1FAD; F; 1F65 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1FAD; S; 1FA5; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1FAE; F; 1F66 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1FAE; S; 1FA6; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1FAF; F; 1F67 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FAF; S; 1FA7; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FB2; F; 1F70 03B9; # GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI +1FB3; F; 03B1 03B9; # GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI +1FB4; F; 03AC 03B9; # GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6; F; 03B1 0342; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI +1FB7; F; 03B1 0342 03B9; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI +1FB8; C; 1FB0; # GREEK CAPITAL LETTER ALPHA WITH VRACHY +1FB9; C; 1FB1; # GREEK CAPITAL LETTER ALPHA WITH MACRON +1FBA; C; 1F70; # GREEK CAPITAL LETTER ALPHA WITH VARIA +1FBB; C; 1F71; # GREEK CAPITAL LETTER ALPHA WITH OXIA +1FBC; F; 03B1 03B9; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBC; S; 1FB3; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBE; C; 03B9; # GREEK PROSGEGRAMMENI +1FC2; F; 1F74 03B9; # GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI +1FC3; F; 03B7 03B9; # GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI +1FC4; F; 03AE 03B9; # GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6; F; 03B7 0342; # GREEK SMALL LETTER ETA WITH PERISPOMENI +1FC7; F; 03B7 0342 03B9; # GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI +1FC8; C; 1F72; # GREEK CAPITAL LETTER EPSILON WITH VARIA +1FC9; C; 1F73; # GREEK CAPITAL LETTER EPSILON WITH OXIA +1FCA; C; 1F74; # GREEK CAPITAL LETTER ETA WITH VARIA +1FCB; C; 1F75; # GREEK CAPITAL LETTER ETA WITH OXIA +1FCC; F; 03B7 03B9; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FCC; S; 1FC3; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FD2; F; 03B9 0308 0300; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA +1FD3; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6; F; 03B9 0342; # GREEK SMALL LETTER IOTA WITH PERISPOMENI +1FD7; F; 03B9 0308 0342; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI +1FD8; C; 1FD0; # GREEK CAPITAL LETTER IOTA WITH VRACHY +1FD9; C; 1FD1; # GREEK CAPITAL LETTER IOTA WITH MACRON +1FDA; C; 1F76; # GREEK CAPITAL LETTER IOTA WITH VARIA +1FDB; C; 1F77; # GREEK CAPITAL LETTER IOTA WITH OXIA +1FE2; F; 03C5 0308 0300; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA +1FE3; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA +1FE4; F; 03C1 0313; # GREEK SMALL LETTER RHO WITH PSILI +1FE6; F; 03C5 0342; # GREEK SMALL LETTER UPSILON WITH PERISPOMENI +1FE7; F; 03C5 0308 0342; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI +1FE8; C; 1FE0; # GREEK CAPITAL LETTER UPSILON WITH VRACHY +1FE9; C; 1FE1; # GREEK CAPITAL LETTER UPSILON WITH MACRON +1FEA; C; 1F7A; # GREEK CAPITAL LETTER UPSILON WITH VARIA +1FEB; C; 1F7B; # GREEK CAPITAL LETTER UPSILON WITH OXIA +1FEC; C; 1FE5; # GREEK CAPITAL LETTER RHO WITH DASIA +1FF2; F; 1F7C 03B9; # GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI +1FF3; F; 03C9 03B9; # GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI +1FF4; F; 03CE 03B9; # GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6; F; 03C9 0342; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI +1FF7; F; 03C9 0342 03B9; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI +1FF8; C; 1F78; # GREEK CAPITAL LETTER OMICRON WITH VARIA +1FF9; C; 1F79; # GREEK CAPITAL LETTER OMICRON WITH OXIA +1FFA; C; 1F7C; # GREEK CAPITAL LETTER OMEGA WITH VARIA +1FFB; C; 1F7D; # GREEK CAPITAL LETTER OMEGA WITH OXIA +1FFC; F; 03C9 03B9; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +1FFC; S; 1FF3; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +2126; C; 03C9; # OHM SIGN +212A; C; 006B; # KELVIN SIGN +212B; C; 00E5; # ANGSTROM SIGN +2132; C; 214E; # TURNED CAPITAL F +2160; C; 2170; # ROMAN NUMERAL ONE +2161; C; 2171; # ROMAN NUMERAL TWO +2162; C; 2172; # ROMAN NUMERAL THREE +2163; C; 2173; # ROMAN NUMERAL FOUR +2164; C; 2174; # ROMAN NUMERAL FIVE +2165; C; 2175; # ROMAN NUMERAL SIX +2166; C; 2176; # ROMAN NUMERAL SEVEN +2167; C; 2177; # ROMAN NUMERAL EIGHT +2168; C; 2178; # ROMAN NUMERAL NINE +2169; C; 2179; # ROMAN NUMERAL TEN +216A; C; 217A; # ROMAN NUMERAL ELEVEN +216B; C; 217B; # ROMAN NUMERAL TWELVE +216C; C; 217C; # ROMAN NUMERAL FIFTY +216D; C; 217D; # ROMAN NUMERAL ONE HUNDRED +216E; C; 217E; # ROMAN NUMERAL FIVE HUNDRED +216F; C; 217F; # ROMAN NUMERAL ONE THOUSAND +2183; C; 2184; # ROMAN NUMERAL REVERSED ONE HUNDRED +24B6; C; 24D0; # CIRCLED LATIN CAPITAL LETTER A +24B7; C; 24D1; # CIRCLED LATIN CAPITAL LETTER B +24B8; C; 24D2; # CIRCLED LATIN CAPITAL LETTER C +24B9; C; 24D3; # CIRCLED LATIN CAPITAL LETTER D +24BA; C; 24D4; # CIRCLED LATIN CAPITAL LETTER E +24BB; C; 24D5; # CIRCLED LATIN CAPITAL LETTER F +24BC; C; 24D6; # CIRCLED LATIN CAPITAL LETTER G +24BD; C; 24D7; # CIRCLED LATIN CAPITAL LETTER H +24BE; C; 24D8; # CIRCLED LATIN CAPITAL LETTER I +24BF; C; 24D9; # CIRCLED LATIN CAPITAL LETTER J +24C0; C; 24DA; # CIRCLED LATIN CAPITAL LETTER K +24C1; C; 24DB; # CIRCLED LATIN CAPITAL LETTER L +24C2; C; 24DC; # CIRCLED LATIN CAPITAL LETTER M +24C3; C; 24DD; # CIRCLED LATIN CAPITAL LETTER N +24C4; C; 24DE; # CIRCLED LATIN CAPITAL LETTER O +24C5; C; 24DF; # CIRCLED LATIN CAPITAL LETTER P +24C6; C; 24E0; # CIRCLED LATIN CAPITAL LETTER Q +24C7; C; 24E1; # CIRCLED LATIN CAPITAL LETTER R +24C8; C; 24E2; # CIRCLED LATIN CAPITAL LETTER S +24C9; C; 24E3; # CIRCLED LATIN CAPITAL LETTER T +24CA; C; 24E4; # CIRCLED LATIN CAPITAL LETTER U +24CB; C; 24E5; # CIRCLED LATIN CAPITAL LETTER V +24CC; C; 24E6; # CIRCLED LATIN CAPITAL LETTER W +24CD; C; 24E7; # CIRCLED LATIN CAPITAL LETTER X +24CE; C; 24E8; # CIRCLED LATIN CAPITAL LETTER Y +24CF; C; 24E9; # CIRCLED LATIN CAPITAL LETTER Z +2C00; C; 2C30; # GLAGOLITIC CAPITAL LETTER AZU +2C01; C; 2C31; # GLAGOLITIC CAPITAL LETTER BUKY +2C02; C; 2C32; # GLAGOLITIC CAPITAL LETTER VEDE +2C03; C; 2C33; # GLAGOLITIC CAPITAL LETTER GLAGOLI +2C04; C; 2C34; # GLAGOLITIC CAPITAL LETTER DOBRO +2C05; C; 2C35; # GLAGOLITIC CAPITAL LETTER YESTU +2C06; C; 2C36; # GLAGOLITIC CAPITAL LETTER ZHIVETE +2C07; C; 2C37; # GLAGOLITIC CAPITAL LETTER DZELO +2C08; C; 2C38; # GLAGOLITIC CAPITAL LETTER ZEMLJA +2C09; C; 2C39; # GLAGOLITIC CAPITAL LETTER IZHE +2C0A; C; 2C3A; # GLAGOLITIC CAPITAL LETTER INITIAL IZHE +2C0B; C; 2C3B; # GLAGOLITIC CAPITAL LETTER I +2C0C; C; 2C3C; # GLAGOLITIC CAPITAL LETTER DJERVI +2C0D; C; 2C3D; # GLAGOLITIC CAPITAL LETTER KAKO +2C0E; C; 2C3E; # GLAGOLITIC CAPITAL LETTER LJUDIJE +2C0F; C; 2C3F; # GLAGOLITIC CAPITAL LETTER MYSLITE +2C10; C; 2C40; # GLAGOLITIC CAPITAL LETTER NASHI +2C11; C; 2C41; # GLAGOLITIC CAPITAL LETTER ONU +2C12; C; 2C42; # GLAGOLITIC CAPITAL LETTER POKOJI +2C13; C; 2C43; # GLAGOLITIC CAPITAL LETTER RITSI +2C14; C; 2C44; # GLAGOLITIC CAPITAL LETTER SLOVO +2C15; C; 2C45; # GLAGOLITIC CAPITAL LETTER TVRIDO +2C16; C; 2C46; # GLAGOLITIC CAPITAL LETTER UKU +2C17; C; 2C47; # GLAGOLITIC CAPITAL LETTER FRITU +2C18; C; 2C48; # GLAGOLITIC CAPITAL LETTER HERU +2C19; C; 2C49; # GLAGOLITIC CAPITAL LETTER OTU +2C1A; C; 2C4A; # GLAGOLITIC CAPITAL LETTER PE +2C1B; C; 2C4B; # GLAGOLITIC CAPITAL LETTER SHTA +2C1C; C; 2C4C; # GLAGOLITIC CAPITAL LETTER TSI +2C1D; C; 2C4D; # GLAGOLITIC CAPITAL LETTER CHRIVI +2C1E; C; 2C4E; # GLAGOLITIC CAPITAL LETTER SHA +2C1F; C; 2C4F; # GLAGOLITIC CAPITAL LETTER YERU +2C20; C; 2C50; # GLAGOLITIC CAPITAL LETTER YERI +2C21; C; 2C51; # GLAGOLITIC CAPITAL LETTER YATI +2C22; C; 2C52; # GLAGOLITIC CAPITAL LETTER SPIDERY HA +2C23; C; 2C53; # GLAGOLITIC CAPITAL LETTER YU +2C24; C; 2C54; # GLAGOLITIC CAPITAL LETTER SMALL YUS +2C25; C; 2C55; # GLAGOLITIC CAPITAL LETTER SMALL YUS WITH TAIL +2C26; C; 2C56; # GLAGOLITIC CAPITAL LETTER YO +2C27; C; 2C57; # GLAGOLITIC CAPITAL LETTER IOTATED SMALL YUS +2C28; C; 2C58; # GLAGOLITIC CAPITAL LETTER BIG YUS +2C29; C; 2C59; # GLAGOLITIC CAPITAL LETTER IOTATED BIG YUS +2C2A; C; 2C5A; # GLAGOLITIC CAPITAL LETTER FITA +2C2B; C; 2C5B; # GLAGOLITIC CAPITAL LETTER IZHITSA +2C2C; C; 2C5C; # GLAGOLITIC CAPITAL LETTER SHTAPIC +2C2D; C; 2C5D; # GLAGOLITIC CAPITAL LETTER TROKUTASTI A +2C2E; C; 2C5E; # GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE +2C60; C; 2C61; # LATIN CAPITAL LETTER L WITH DOUBLE BAR +2C62; C; 026B; # LATIN CAPITAL LETTER L WITH MIDDLE TILDE +2C63; C; 1D7D; # LATIN CAPITAL LETTER P WITH STROKE +2C64; C; 027D; # LATIN CAPITAL LETTER R WITH TAIL +2C67; C; 2C68; # LATIN CAPITAL LETTER H WITH DESCENDER +2C69; C; 2C6A; # LATIN CAPITAL LETTER K WITH DESCENDER +2C6B; C; 2C6C; # LATIN CAPITAL LETTER Z WITH DESCENDER +2C6D; C; 0251; # LATIN CAPITAL LETTER ALPHA +2C6E; C; 0271; # LATIN CAPITAL LETTER M WITH HOOK +2C6F; C; 0250; # LATIN CAPITAL LETTER TURNED A +2C70; C; 0252; # LATIN CAPITAL LETTER TURNED ALPHA +2C72; C; 2C73; # LATIN CAPITAL LETTER W WITH HOOK +2C75; C; 2C76; # LATIN CAPITAL LETTER HALF H +2C7E; C; 023F; # LATIN CAPITAL LETTER S WITH SWASH TAIL +2C7F; C; 0240; # LATIN CAPITAL LETTER Z WITH SWASH TAIL +2C80; C; 2C81; # COPTIC CAPITAL LETTER ALFA +2C82; C; 2C83; # COPTIC CAPITAL LETTER VIDA +2C84; C; 2C85; # COPTIC CAPITAL LETTER GAMMA +2C86; C; 2C87; # COPTIC CAPITAL LETTER DALDA +2C88; C; 2C89; # COPTIC CAPITAL LETTER EIE +2C8A; C; 2C8B; # COPTIC CAPITAL LETTER SOU +2C8C; C; 2C8D; # COPTIC CAPITAL LETTER ZATA +2C8E; C; 2C8F; # COPTIC CAPITAL LETTER HATE +2C90; C; 2C91; # COPTIC CAPITAL LETTER THETHE +2C92; C; 2C93; # COPTIC CAPITAL LETTER IAUDA +2C94; C; 2C95; # COPTIC CAPITAL LETTER KAPA +2C96; C; 2C97; # COPTIC CAPITAL LETTER LAULA +2C98; C; 2C99; # COPTIC CAPITAL LETTER MI +2C9A; C; 2C9B; # COPTIC CAPITAL LETTER NI +2C9C; C; 2C9D; # COPTIC CAPITAL LETTER KSI +2C9E; C; 2C9F; # COPTIC CAPITAL LETTER O +2CA0; C; 2CA1; # COPTIC CAPITAL LETTER PI +2CA2; C; 2CA3; # COPTIC CAPITAL LETTER RO +2CA4; C; 2CA5; # COPTIC CAPITAL LETTER SIMA +2CA6; C; 2CA7; # COPTIC CAPITAL LETTER TAU +2CA8; C; 2CA9; # COPTIC CAPITAL LETTER UA +2CAA; C; 2CAB; # COPTIC CAPITAL LETTER FI +2CAC; C; 2CAD; # COPTIC CAPITAL LETTER KHI +2CAE; C; 2CAF; # COPTIC CAPITAL LETTER PSI +2CB0; C; 2CB1; # COPTIC CAPITAL LETTER OOU +2CB2; C; 2CB3; # COPTIC CAPITAL LETTER DIALECT-P ALEF +2CB4; C; 2CB5; # COPTIC CAPITAL LETTER OLD COPTIC AIN +2CB6; C; 2CB7; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC EIE +2CB8; C; 2CB9; # COPTIC CAPITAL LETTER DIALECT-P KAPA +2CBA; C; 2CBB; # COPTIC CAPITAL LETTER DIALECT-P NI +2CBC; C; 2CBD; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC NI +2CBE; C; 2CBF; # COPTIC CAPITAL LETTER OLD COPTIC OOU +2CC0; C; 2CC1; # COPTIC CAPITAL LETTER SAMPI +2CC2; C; 2CC3; # COPTIC CAPITAL LETTER CROSSED SHEI +2CC4; C; 2CC5; # COPTIC CAPITAL LETTER OLD COPTIC SHEI +2CC6; C; 2CC7; # COPTIC CAPITAL LETTER OLD COPTIC ESH +2CC8; C; 2CC9; # COPTIC CAPITAL LETTER AKHMIMIC KHEI +2CCA; C; 2CCB; # COPTIC CAPITAL LETTER DIALECT-P HORI +2CCC; C; 2CCD; # COPTIC CAPITAL LETTER OLD COPTIC HORI +2CCE; C; 2CCF; # COPTIC CAPITAL LETTER OLD COPTIC HA +2CD0; C; 2CD1; # COPTIC CAPITAL LETTER L-SHAPED HA +2CD2; C; 2CD3; # COPTIC CAPITAL LETTER OLD COPTIC HEI +2CD4; C; 2CD5; # COPTIC CAPITAL LETTER OLD COPTIC HAT +2CD6; C; 2CD7; # COPTIC CAPITAL LETTER OLD COPTIC GANGIA +2CD8; C; 2CD9; # COPTIC CAPITAL LETTER OLD COPTIC DJA +2CDA; C; 2CDB; # COPTIC CAPITAL LETTER OLD COPTIC SHIMA +2CDC; C; 2CDD; # COPTIC CAPITAL LETTER OLD NUBIAN SHIMA +2CDE; C; 2CDF; # COPTIC CAPITAL LETTER OLD NUBIAN NGI +2CE0; C; 2CE1; # COPTIC CAPITAL LETTER OLD NUBIAN NYI +2CE2; C; 2CE3; # COPTIC CAPITAL LETTER OLD NUBIAN WAU +2CEB; C; 2CEC; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI +2CED; C; 2CEE; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC GANGIA +2CF2; C; 2CF3; # COPTIC CAPITAL LETTER BOHAIRIC KHEI +A640; C; A641; # CYRILLIC CAPITAL LETTER ZEMLYA +A642; C; A643; # CYRILLIC CAPITAL LETTER DZELO +A644; C; A645; # CYRILLIC CAPITAL LETTER REVERSED DZE +A646; C; A647; # CYRILLIC CAPITAL LETTER IOTA +A648; C; A649; # CYRILLIC CAPITAL LETTER DJERV +A64A; C; A64B; # CYRILLIC CAPITAL LETTER MONOGRAPH UK +A64C; C; A64D; # CYRILLIC CAPITAL LETTER BROAD OMEGA +A64E; C; A64F; # CYRILLIC CAPITAL LETTER NEUTRAL YER +A650; C; A651; # CYRILLIC CAPITAL LETTER YERU WITH BACK YER +A652; C; A653; # CYRILLIC CAPITAL LETTER IOTIFIED YAT +A654; C; A655; # CYRILLIC CAPITAL LETTER REVERSED YU +A656; C; A657; # CYRILLIC CAPITAL LETTER IOTIFIED A +A658; C; A659; # CYRILLIC CAPITAL LETTER CLOSED LITTLE YUS +A65A; C; A65B; # CYRILLIC CAPITAL LETTER BLENDED YUS +A65C; C; A65D; # CYRILLIC CAPITAL LETTER IOTIFIED CLOSED LITTLE YUS +A65E; C; A65F; # CYRILLIC CAPITAL LETTER YN +A660; C; A661; # CYRILLIC CAPITAL LETTER REVERSED TSE +A662; C; A663; # CYRILLIC CAPITAL LETTER SOFT DE +A664; C; A665; # CYRILLIC CAPITAL LETTER SOFT EL +A666; C; A667; # CYRILLIC CAPITAL LETTER SOFT EM +A668; C; A669; # CYRILLIC CAPITAL LETTER MONOCULAR O +A66A; C; A66B; # CYRILLIC CAPITAL LETTER BINOCULAR O +A66C; C; A66D; # CYRILLIC CAPITAL LETTER DOUBLE MONOCULAR O +A680; C; A681; # CYRILLIC CAPITAL LETTER DWE +A682; C; A683; # CYRILLIC CAPITAL LETTER DZWE +A684; C; A685; # CYRILLIC CAPITAL LETTER ZHWE +A686; C; A687; # CYRILLIC CAPITAL LETTER CCHE +A688; C; A689; # CYRILLIC CAPITAL LETTER DZZE +A68A; C; A68B; # CYRILLIC CAPITAL LETTER TE WITH MIDDLE HOOK +A68C; C; A68D; # CYRILLIC CAPITAL LETTER TWE +A68E; C; A68F; # CYRILLIC CAPITAL LETTER TSWE +A690; C; A691; # CYRILLIC CAPITAL LETTER TSSE +A692; C; A693; # CYRILLIC CAPITAL LETTER TCHE +A694; C; A695; # CYRILLIC CAPITAL LETTER HWE +A696; C; A697; # CYRILLIC CAPITAL LETTER SHWE +A698; C; A699; # CYRILLIC CAPITAL LETTER DOUBLE O +A69A; C; A69B; # CYRILLIC CAPITAL LETTER CROSSED O +A722; C; A723; # LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF +A724; C; A725; # LATIN CAPITAL LETTER EGYPTOLOGICAL AIN +A726; C; A727; # LATIN CAPITAL LETTER HENG +A728; C; A729; # LATIN CAPITAL LETTER TZ +A72A; C; A72B; # LATIN CAPITAL LETTER TRESILLO +A72C; C; A72D; # LATIN CAPITAL LETTER CUATRILLO +A72E; C; A72F; # LATIN CAPITAL LETTER CUATRILLO WITH COMMA +A732; C; A733; # LATIN CAPITAL LETTER AA +A734; C; A735; # LATIN CAPITAL LETTER AO +A736; C; A737; # LATIN CAPITAL LETTER AU +A738; C; A739; # LATIN CAPITAL LETTER AV +A73A; C; A73B; # LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR +A73C; C; A73D; # LATIN CAPITAL LETTER AY +A73E; C; A73F; # LATIN CAPITAL LETTER REVERSED C WITH DOT +A740; C; A741; # LATIN CAPITAL LETTER K WITH STROKE +A742; C; A743; # LATIN CAPITAL LETTER K WITH DIAGONAL STROKE +A744; C; A745; # LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE +A746; C; A747; # LATIN CAPITAL LETTER BROKEN L +A748; C; A749; # LATIN CAPITAL LETTER L WITH HIGH STROKE +A74A; C; A74B; # LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY +A74C; C; A74D; # LATIN CAPITAL LETTER O WITH LOOP +A74E; C; A74F; # LATIN CAPITAL LETTER OO +A750; C; A751; # LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER +A752; C; A753; # LATIN CAPITAL LETTER P WITH FLOURISH +A754; C; A755; # LATIN CAPITAL LETTER P WITH SQUIRREL TAIL +A756; C; A757; # LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER +A758; C; A759; # LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE +A75A; C; A75B; # LATIN CAPITAL LETTER R ROTUNDA +A75C; C; A75D; # LATIN CAPITAL LETTER RUM ROTUNDA +A75E; C; A75F; # LATIN CAPITAL LETTER V WITH DIAGONAL STROKE +A760; C; A761; # LATIN CAPITAL LETTER VY +A762; C; A763; # LATIN CAPITAL LETTER VISIGOTHIC Z +A764; C; A765; # LATIN CAPITAL LETTER THORN WITH STROKE +A766; C; A767; # LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER +A768; C; A769; # LATIN CAPITAL LETTER VEND +A76A; C; A76B; # LATIN CAPITAL LETTER ET +A76C; C; A76D; # LATIN CAPITAL LETTER IS +A76E; C; A76F; # LATIN CAPITAL LETTER CON +A779; C; A77A; # LATIN CAPITAL LETTER INSULAR D +A77B; C; A77C; # LATIN CAPITAL LETTER INSULAR F +A77D; C; 1D79; # LATIN CAPITAL LETTER INSULAR G +A77E; C; A77F; # LATIN CAPITAL LETTER TURNED INSULAR G +A780; C; A781; # LATIN CAPITAL LETTER TURNED L +A782; C; A783; # LATIN CAPITAL LETTER INSULAR R +A784; C; A785; # LATIN CAPITAL LETTER INSULAR S +A786; C; A787; # LATIN CAPITAL LETTER INSULAR T +A78B; C; A78C; # LATIN CAPITAL LETTER SALTILLO +A78D; C; 0265; # LATIN CAPITAL LETTER TURNED H +A790; C; A791; # LATIN CAPITAL LETTER N WITH DESCENDER +A792; C; A793; # LATIN CAPITAL LETTER C WITH BAR +A796; C; A797; # LATIN CAPITAL LETTER B WITH FLOURISH +A798; C; A799; # LATIN CAPITAL LETTER F WITH STROKE +A79A; C; A79B; # LATIN CAPITAL LETTER VOLAPUK AE +A79C; C; A79D; # LATIN CAPITAL LETTER VOLAPUK OE +A79E; C; A79F; # LATIN CAPITAL LETTER VOLAPUK UE +A7A0; C; A7A1; # LATIN CAPITAL LETTER G WITH OBLIQUE STROKE +A7A2; C; A7A3; # LATIN CAPITAL LETTER K WITH OBLIQUE STROKE +A7A4; C; A7A5; # LATIN CAPITAL LETTER N WITH OBLIQUE STROKE +A7A6; C; A7A7; # LATIN CAPITAL LETTER R WITH OBLIQUE STROKE +A7A8; C; A7A9; # LATIN CAPITAL LETTER S WITH OBLIQUE STROKE +A7AA; C; 0266; # LATIN CAPITAL LETTER H WITH HOOK +A7AB; C; 025C; # LATIN CAPITAL LETTER REVERSED OPEN E +A7AC; C; 0261; # LATIN CAPITAL LETTER SCRIPT G +A7AD; C; 026C; # LATIN CAPITAL LETTER L WITH BELT +A7AE; C; 026A; # LATIN CAPITAL LETTER SMALL CAPITAL I +A7B0; C; 029E; # LATIN CAPITAL LETTER TURNED K +A7B1; C; 0287; # LATIN CAPITAL LETTER TURNED T +A7B2; C; 029D; # LATIN CAPITAL LETTER J WITH CROSSED-TAIL +A7B3; C; AB53; # LATIN CAPITAL LETTER CHI +A7B4; C; A7B5; # LATIN CAPITAL LETTER BETA +A7B6; C; A7B7; # LATIN CAPITAL LETTER OMEGA +A7B8; C; A7B9; # LATIN CAPITAL LETTER U WITH STROKE +AB70; C; 13A0; # CHEROKEE SMALL LETTER A +AB71; C; 13A1; # CHEROKEE SMALL LETTER E +AB72; C; 13A2; # CHEROKEE SMALL LETTER I +AB73; C; 13A3; # CHEROKEE SMALL LETTER O +AB74; C; 13A4; # CHEROKEE SMALL LETTER U +AB75; C; 13A5; # CHEROKEE SMALL LETTER V +AB76; C; 13A6; # CHEROKEE SMALL LETTER GA +AB77; C; 13A7; # CHEROKEE SMALL LETTER KA +AB78; C; 13A8; # CHEROKEE SMALL LETTER GE +AB79; C; 13A9; # CHEROKEE SMALL LETTER GI +AB7A; C; 13AA; # CHEROKEE SMALL LETTER GO +AB7B; C; 13AB; # CHEROKEE SMALL LETTER GU +AB7C; C; 13AC; # CHEROKEE SMALL LETTER GV +AB7D; C; 13AD; # CHEROKEE SMALL LETTER HA +AB7E; C; 13AE; # CHEROKEE SMALL LETTER HE +AB7F; C; 13AF; # CHEROKEE SMALL LETTER HI +AB80; C; 13B0; # CHEROKEE SMALL LETTER HO +AB81; C; 13B1; # CHEROKEE SMALL LETTER HU +AB82; C; 13B2; # CHEROKEE SMALL LETTER HV +AB83; C; 13B3; # CHEROKEE SMALL LETTER LA +AB84; C; 13B4; # CHEROKEE SMALL LETTER LE +AB85; C; 13B5; # CHEROKEE SMALL LETTER LI +AB86; C; 13B6; # CHEROKEE SMALL LETTER LO +AB87; C; 13B7; # CHEROKEE SMALL LETTER LU +AB88; C; 13B8; # CHEROKEE SMALL LETTER LV +AB89; C; 13B9; # CHEROKEE SMALL LETTER MA +AB8A; C; 13BA; # CHEROKEE SMALL LETTER ME +AB8B; C; 13BB; # CHEROKEE SMALL LETTER MI +AB8C; C; 13BC; # CHEROKEE SMALL LETTER MO +AB8D; C; 13BD; # CHEROKEE SMALL LETTER MU +AB8E; C; 13BE; # CHEROKEE SMALL LETTER NA +AB8F; C; 13BF; # CHEROKEE SMALL LETTER HNA +AB90; C; 13C0; # CHEROKEE SMALL LETTER NAH +AB91; C; 13C1; # CHEROKEE SMALL LETTER NE +AB92; C; 13C2; # CHEROKEE SMALL LETTER NI +AB93; C; 13C3; # CHEROKEE SMALL LETTER NO +AB94; C; 13C4; # CHEROKEE SMALL LETTER NU +AB95; C; 13C5; # CHEROKEE SMALL LETTER NV +AB96; C; 13C6; # CHEROKEE SMALL LETTER QUA +AB97; C; 13C7; # CHEROKEE SMALL LETTER QUE +AB98; C; 13C8; # CHEROKEE SMALL LETTER QUI +AB99; C; 13C9; # CHEROKEE SMALL LETTER QUO +AB9A; C; 13CA; # CHEROKEE SMALL LETTER QUU +AB9B; C; 13CB; # CHEROKEE SMALL LETTER QUV +AB9C; C; 13CC; # CHEROKEE SMALL LETTER SA +AB9D; C; 13CD; # CHEROKEE SMALL LETTER S +AB9E; C; 13CE; # CHEROKEE SMALL LETTER SE +AB9F; C; 13CF; # CHEROKEE SMALL LETTER SI +ABA0; C; 13D0; # CHEROKEE SMALL LETTER SO +ABA1; C; 13D1; # CHEROKEE SMALL LETTER SU +ABA2; C; 13D2; # CHEROKEE SMALL LETTER SV +ABA3; C; 13D3; # CHEROKEE SMALL LETTER DA +ABA4; C; 13D4; # CHEROKEE SMALL LETTER TA +ABA5; C; 13D5; # CHEROKEE SMALL LETTER DE +ABA6; C; 13D6; # CHEROKEE SMALL LETTER TE +ABA7; C; 13D7; # CHEROKEE SMALL LETTER DI +ABA8; C; 13D8; # CHEROKEE SMALL LETTER TI +ABA9; C; 13D9; # CHEROKEE SMALL LETTER DO +ABAA; C; 13DA; # CHEROKEE SMALL LETTER DU +ABAB; C; 13DB; # CHEROKEE SMALL LETTER DV +ABAC; C; 13DC; # CHEROKEE SMALL LETTER DLA +ABAD; C; 13DD; # CHEROKEE SMALL LETTER TLA +ABAE; C; 13DE; # CHEROKEE SMALL LETTER TLE +ABAF; C; 13DF; # CHEROKEE SMALL LETTER TLI +ABB0; C; 13E0; # CHEROKEE SMALL LETTER TLO +ABB1; C; 13E1; # CHEROKEE SMALL LETTER TLU +ABB2; C; 13E2; # CHEROKEE SMALL LETTER TLV +ABB3; C; 13E3; # CHEROKEE SMALL LETTER TSA +ABB4; C; 13E4; # CHEROKEE SMALL LETTER TSE +ABB5; C; 13E5; # CHEROKEE SMALL LETTER TSI +ABB6; C; 13E6; # CHEROKEE SMALL LETTER TSO +ABB7; C; 13E7; # CHEROKEE SMALL LETTER TSU +ABB8; C; 13E8; # CHEROKEE SMALL LETTER TSV +ABB9; C; 13E9; # CHEROKEE SMALL LETTER WA +ABBA; C; 13EA; # CHEROKEE SMALL LETTER WE +ABBB; C; 13EB; # CHEROKEE SMALL LETTER WI +ABBC; C; 13EC; # CHEROKEE SMALL LETTER WO +ABBD; C; 13ED; # CHEROKEE SMALL LETTER WU +ABBE; C; 13EE; # CHEROKEE SMALL LETTER WV +ABBF; C; 13EF; # CHEROKEE SMALL LETTER YA +FB00; F; 0066 0066; # LATIN SMALL LIGATURE FF +FB01; F; 0066 0069; # LATIN SMALL LIGATURE FI +FB02; F; 0066 006C; # LATIN SMALL LIGATURE FL +FB03; F; 0066 0066 0069; # LATIN SMALL LIGATURE FFI +FB04; F; 0066 0066 006C; # LATIN SMALL LIGATURE FFL +FB05; F; 0073 0074; # LATIN SMALL LIGATURE LONG S T +FB06; F; 0073 0074; # LATIN SMALL LIGATURE ST +FB13; F; 0574 0576; # ARMENIAN SMALL LIGATURE MEN NOW +FB14; F; 0574 0565; # ARMENIAN SMALL LIGATURE MEN ECH +FB15; F; 0574 056B; # ARMENIAN SMALL LIGATURE MEN INI +FB16; F; 057E 0576; # ARMENIAN SMALL LIGATURE VEW NOW +FB17; F; 0574 056D; # ARMENIAN SMALL LIGATURE MEN XEH +FF21; C; FF41; # FULLWIDTH LATIN CAPITAL LETTER A +FF22; C; FF42; # FULLWIDTH LATIN CAPITAL LETTER B +FF23; C; FF43; # FULLWIDTH LATIN CAPITAL LETTER C +FF24; C; FF44; # FULLWIDTH LATIN CAPITAL LETTER D +FF25; C; FF45; # FULLWIDTH LATIN CAPITAL LETTER E +FF26; C; FF46; # FULLWIDTH LATIN CAPITAL LETTER F +FF27; C; FF47; # FULLWIDTH LATIN CAPITAL LETTER G +FF28; C; FF48; # FULLWIDTH LATIN CAPITAL LETTER H +FF29; C; FF49; # FULLWIDTH LATIN CAPITAL LETTER I +FF2A; C; FF4A; # FULLWIDTH LATIN CAPITAL LETTER J +FF2B; C; FF4B; # FULLWIDTH LATIN CAPITAL LETTER K +FF2C; C; FF4C; # FULLWIDTH LATIN CAPITAL LETTER L +FF2D; C; FF4D; # FULLWIDTH LATIN CAPITAL LETTER M +FF2E; C; FF4E; # FULLWIDTH LATIN CAPITAL LETTER N +FF2F; C; FF4F; # FULLWIDTH LATIN CAPITAL LETTER O +FF30; C; FF50; # FULLWIDTH LATIN CAPITAL LETTER P +FF31; C; FF51; # FULLWIDTH LATIN CAPITAL LETTER Q +FF32; C; FF52; # FULLWIDTH LATIN CAPITAL LETTER R +FF33; C; FF53; # FULLWIDTH LATIN CAPITAL LETTER S +FF34; C; FF54; # FULLWIDTH LATIN CAPITAL LETTER T +FF35; C; FF55; # FULLWIDTH LATIN CAPITAL LETTER U +FF36; C; FF56; # FULLWIDTH LATIN CAPITAL LETTER V +FF37; C; FF57; # FULLWIDTH LATIN CAPITAL LETTER W +FF38; C; FF58; # FULLWIDTH LATIN CAPITAL LETTER X +FF39; C; FF59; # FULLWIDTH LATIN CAPITAL LETTER Y +FF3A; C; FF5A; # FULLWIDTH LATIN CAPITAL LETTER Z +10400; C; 10428; # DESERET CAPITAL LETTER LONG I +10401; C; 10429; # DESERET CAPITAL LETTER LONG E +10402; C; 1042A; # DESERET CAPITAL LETTER LONG A +10403; C; 1042B; # DESERET CAPITAL LETTER LONG AH +10404; C; 1042C; # DESERET CAPITAL LETTER LONG O +10405; C; 1042D; # DESERET CAPITAL LETTER LONG OO +10406; C; 1042E; # DESERET CAPITAL LETTER SHORT I +10407; C; 1042F; # DESERET CAPITAL LETTER SHORT E +10408; C; 10430; # DESERET CAPITAL LETTER SHORT A +10409; C; 10431; # DESERET CAPITAL LETTER SHORT AH +1040A; C; 10432; # DESERET CAPITAL LETTER SHORT O +1040B; C; 10433; # DESERET CAPITAL LETTER SHORT OO +1040C; C; 10434; # DESERET CAPITAL LETTER AY +1040D; C; 10435; # DESERET CAPITAL LETTER OW +1040E; C; 10436; # DESERET CAPITAL LETTER WU +1040F; C; 10437; # DESERET CAPITAL LETTER YEE +10410; C; 10438; # DESERET CAPITAL LETTER H +10411; C; 10439; # DESERET CAPITAL LETTER PEE +10412; C; 1043A; # DESERET CAPITAL LETTER BEE +10413; C; 1043B; # DESERET CAPITAL LETTER TEE +10414; C; 1043C; # DESERET CAPITAL LETTER DEE +10415; C; 1043D; # DESERET CAPITAL LETTER CHEE +10416; C; 1043E; # DESERET CAPITAL LETTER JEE +10417; C; 1043F; # DESERET CAPITAL LETTER KAY +10418; C; 10440; # DESERET CAPITAL LETTER GAY +10419; C; 10441; # DESERET CAPITAL LETTER EF +1041A; C; 10442; # DESERET CAPITAL LETTER VEE +1041B; C; 10443; # DESERET CAPITAL LETTER ETH +1041C; C; 10444; # DESERET CAPITAL LETTER THEE +1041D; C; 10445; # DESERET CAPITAL LETTER ES +1041E; C; 10446; # DESERET CAPITAL LETTER ZEE +1041F; C; 10447; # DESERET CAPITAL LETTER ESH +10420; C; 10448; # DESERET CAPITAL LETTER ZHEE +10421; C; 10449; # DESERET CAPITAL LETTER ER +10422; C; 1044A; # DESERET CAPITAL LETTER EL +10423; C; 1044B; # DESERET CAPITAL LETTER EM +10424; C; 1044C; # DESERET CAPITAL LETTER EN +10425; C; 1044D; # DESERET CAPITAL LETTER ENG +10426; C; 1044E; # DESERET CAPITAL LETTER OI +10427; C; 1044F; # DESERET CAPITAL LETTER EW +104B0; C; 104D8; # OSAGE CAPITAL LETTER A +104B1; C; 104D9; # OSAGE CAPITAL LETTER AI +104B2; C; 104DA; # OSAGE CAPITAL LETTER AIN +104B3; C; 104DB; # OSAGE CAPITAL LETTER AH +104B4; C; 104DC; # OSAGE CAPITAL LETTER BRA +104B5; C; 104DD; # OSAGE CAPITAL LETTER CHA +104B6; C; 104DE; # OSAGE CAPITAL LETTER EHCHA +104B7; C; 104DF; # OSAGE CAPITAL LETTER E +104B8; C; 104E0; # OSAGE CAPITAL LETTER EIN +104B9; C; 104E1; # OSAGE CAPITAL LETTER HA +104BA; C; 104E2; # OSAGE CAPITAL LETTER HYA +104BB; C; 104E3; # OSAGE CAPITAL LETTER I +104BC; C; 104E4; # OSAGE CAPITAL LETTER KA +104BD; C; 104E5; # OSAGE CAPITAL LETTER EHKA +104BE; C; 104E6; # OSAGE CAPITAL LETTER KYA +104BF; C; 104E7; # OSAGE CAPITAL LETTER LA +104C0; C; 104E8; # OSAGE CAPITAL LETTER MA +104C1; C; 104E9; # OSAGE CAPITAL LETTER NA +104C2; C; 104EA; # OSAGE CAPITAL LETTER O +104C3; C; 104EB; # OSAGE CAPITAL LETTER OIN +104C4; C; 104EC; # OSAGE CAPITAL LETTER PA +104C5; C; 104ED; # OSAGE CAPITAL LETTER EHPA +104C6; C; 104EE; # OSAGE CAPITAL LETTER SA +104C7; C; 104EF; # OSAGE CAPITAL LETTER SHA +104C8; C; 104F0; # OSAGE CAPITAL LETTER TA +104C9; C; 104F1; # OSAGE CAPITAL LETTER EHTA +104CA; C; 104F2; # OSAGE CAPITAL LETTER TSA +104CB; C; 104F3; # OSAGE CAPITAL LETTER EHTSA +104CC; C; 104F4; # OSAGE CAPITAL LETTER TSHA +104CD; C; 104F5; # OSAGE CAPITAL LETTER DHA +104CE; C; 104F6; # OSAGE CAPITAL LETTER U +104CF; C; 104F7; # OSAGE CAPITAL LETTER WA +104D0; C; 104F8; # OSAGE CAPITAL LETTER KHA +104D1; C; 104F9; # OSAGE CAPITAL LETTER GHA +104D2; C; 104FA; # OSAGE CAPITAL LETTER ZA +104D3; C; 104FB; # OSAGE CAPITAL LETTER ZHA +10C80; C; 10CC0; # OLD HUNGARIAN CAPITAL LETTER A +10C81; C; 10CC1; # OLD HUNGARIAN CAPITAL LETTER AA +10C82; C; 10CC2; # OLD HUNGARIAN CAPITAL LETTER EB +10C83; C; 10CC3; # OLD HUNGARIAN CAPITAL LETTER AMB +10C84; C; 10CC4; # OLD HUNGARIAN CAPITAL LETTER EC +10C85; C; 10CC5; # OLD HUNGARIAN CAPITAL LETTER ENC +10C86; C; 10CC6; # OLD HUNGARIAN CAPITAL LETTER ECS +10C87; C; 10CC7; # OLD HUNGARIAN CAPITAL LETTER ED +10C88; C; 10CC8; # OLD HUNGARIAN CAPITAL LETTER AND +10C89; C; 10CC9; # OLD HUNGARIAN CAPITAL LETTER E +10C8A; C; 10CCA; # OLD HUNGARIAN CAPITAL LETTER CLOSE E +10C8B; C; 10CCB; # OLD HUNGARIAN CAPITAL LETTER EE +10C8C; C; 10CCC; # OLD HUNGARIAN CAPITAL LETTER EF +10C8D; C; 10CCD; # OLD HUNGARIAN CAPITAL LETTER EG +10C8E; C; 10CCE; # OLD HUNGARIAN CAPITAL LETTER EGY +10C8F; C; 10CCF; # OLD HUNGARIAN CAPITAL LETTER EH +10C90; C; 10CD0; # OLD HUNGARIAN CAPITAL LETTER I +10C91; C; 10CD1; # OLD HUNGARIAN CAPITAL LETTER II +10C92; C; 10CD2; # OLD HUNGARIAN CAPITAL LETTER EJ +10C93; C; 10CD3; # OLD HUNGARIAN CAPITAL LETTER EK +10C94; C; 10CD4; # OLD HUNGARIAN CAPITAL LETTER AK +10C95; C; 10CD5; # OLD HUNGARIAN CAPITAL LETTER UNK +10C96; C; 10CD6; # OLD HUNGARIAN CAPITAL LETTER EL +10C97; C; 10CD7; # OLD HUNGARIAN CAPITAL LETTER ELY +10C98; C; 10CD8; # OLD HUNGARIAN CAPITAL LETTER EM +10C99; C; 10CD9; # OLD HUNGARIAN CAPITAL LETTER EN +10C9A; C; 10CDA; # OLD HUNGARIAN CAPITAL LETTER ENY +10C9B; C; 10CDB; # OLD HUNGARIAN CAPITAL LETTER O +10C9C; C; 10CDC; # OLD HUNGARIAN CAPITAL LETTER OO +10C9D; C; 10CDD; # OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG OE +10C9E; C; 10CDE; # OLD HUNGARIAN CAPITAL LETTER RUDIMENTA OE +10C9F; C; 10CDF; # OLD HUNGARIAN CAPITAL LETTER OEE +10CA0; C; 10CE0; # OLD HUNGARIAN CAPITAL LETTER EP +10CA1; C; 10CE1; # OLD HUNGARIAN CAPITAL LETTER EMP +10CA2; C; 10CE2; # OLD HUNGARIAN CAPITAL LETTER ER +10CA3; C; 10CE3; # OLD HUNGARIAN CAPITAL LETTER SHORT ER +10CA4; C; 10CE4; # OLD HUNGARIAN CAPITAL LETTER ES +10CA5; C; 10CE5; # OLD HUNGARIAN CAPITAL LETTER ESZ +10CA6; C; 10CE6; # OLD HUNGARIAN CAPITAL LETTER ET +10CA7; C; 10CE7; # OLD HUNGARIAN CAPITAL LETTER ENT +10CA8; C; 10CE8; # OLD HUNGARIAN CAPITAL LETTER ETY +10CA9; C; 10CE9; # OLD HUNGARIAN CAPITAL LETTER ECH +10CAA; C; 10CEA; # OLD HUNGARIAN CAPITAL LETTER U +10CAB; C; 10CEB; # OLD HUNGARIAN CAPITAL LETTER UU +10CAC; C; 10CEC; # OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG UE +10CAD; C; 10CED; # OLD HUNGARIAN CAPITAL LETTER RUDIMENTA UE +10CAE; C; 10CEE; # OLD HUNGARIAN CAPITAL LETTER EV +10CAF; C; 10CEF; # OLD HUNGARIAN CAPITAL LETTER EZ +10CB0; C; 10CF0; # OLD HUNGARIAN CAPITAL LETTER EZS +10CB1; C; 10CF1; # OLD HUNGARIAN CAPITAL LETTER ENT-SHAPED SIGN +10CB2; C; 10CF2; # OLD HUNGARIAN CAPITAL LETTER US +118A0; C; 118C0; # WARANG CITI CAPITAL LETTER NGAA +118A1; C; 118C1; # WARANG CITI CAPITAL LETTER A +118A2; C; 118C2; # WARANG CITI CAPITAL LETTER WI +118A3; C; 118C3; # WARANG CITI CAPITAL LETTER YU +118A4; C; 118C4; # WARANG CITI CAPITAL LETTER YA +118A5; C; 118C5; # WARANG CITI CAPITAL LETTER YO +118A6; C; 118C6; # WARANG CITI CAPITAL LETTER II +118A7; C; 118C7; # WARANG CITI CAPITAL LETTER UU +118A8; C; 118C8; # WARANG CITI CAPITAL LETTER E +118A9; C; 118C9; # WARANG CITI CAPITAL LETTER O +118AA; C; 118CA; # WARANG CITI CAPITAL LETTER ANG +118AB; C; 118CB; # WARANG CITI CAPITAL LETTER GA +118AC; C; 118CC; # WARANG CITI CAPITAL LETTER KO +118AD; C; 118CD; # WARANG CITI CAPITAL LETTER ENY +118AE; C; 118CE; # WARANG CITI CAPITAL LETTER YUJ +118AF; C; 118CF; # WARANG CITI CAPITAL LETTER UC +118B0; C; 118D0; # WARANG CITI CAPITAL LETTER ENN +118B1; C; 118D1; # WARANG CITI CAPITAL LETTER ODD +118B2; C; 118D2; # WARANG CITI CAPITAL LETTER TTE +118B3; C; 118D3; # WARANG CITI CAPITAL LETTER NUNG +118B4; C; 118D4; # WARANG CITI CAPITAL LETTER DA +118B5; C; 118D5; # WARANG CITI CAPITAL LETTER AT +118B6; C; 118D6; # WARANG CITI CAPITAL LETTER AM +118B7; C; 118D7; # WARANG CITI CAPITAL LETTER BU +118B8; C; 118D8; # WARANG CITI CAPITAL LETTER PU +118B9; C; 118D9; # WARANG CITI CAPITAL LETTER HIYO +118BA; C; 118DA; # WARANG CITI CAPITAL LETTER HOLO +118BB; C; 118DB; # WARANG CITI CAPITAL LETTER HORR +118BC; C; 118DC; # WARANG CITI CAPITAL LETTER HAR +118BD; C; 118DD; # WARANG CITI CAPITAL LETTER SSUU +118BE; C; 118DE; # WARANG CITI CAPITAL LETTER SII +118BF; C; 118DF; # WARANG CITI CAPITAL LETTER VIYO +16E40; C; 16E60; # MEDEFAIDRIN CAPITAL LETTER M +16E41; C; 16E61; # MEDEFAIDRIN CAPITAL LETTER S +16E42; C; 16E62; # MEDEFAIDRIN CAPITAL LETTER V +16E43; C; 16E63; # MEDEFAIDRIN CAPITAL LETTER W +16E44; C; 16E64; # MEDEFAIDRIN CAPITAL LETTER ATIU +16E45; C; 16E65; # MEDEFAIDRIN CAPITAL LETTER Z +16E46; C; 16E66; # MEDEFAIDRIN CAPITAL LETTER KP +16E47; C; 16E67; # MEDEFAIDRIN CAPITAL LETTER P +16E48; C; 16E68; # MEDEFAIDRIN CAPITAL LETTER T +16E49; C; 16E69; # MEDEFAIDRIN CAPITAL LETTER G +16E4A; C; 16E6A; # MEDEFAIDRIN CAPITAL LETTER F +16E4B; C; 16E6B; # MEDEFAIDRIN CAPITAL LETTER I +16E4C; C; 16E6C; # MEDEFAIDRIN CAPITAL LETTER K +16E4D; C; 16E6D; # MEDEFAIDRIN CAPITAL LETTER A +16E4E; C; 16E6E; # MEDEFAIDRIN CAPITAL LETTER J +16E4F; C; 16E6F; # MEDEFAIDRIN CAPITAL LETTER E +16E50; C; 16E70; # MEDEFAIDRIN CAPITAL LETTER B +16E51; C; 16E71; # MEDEFAIDRIN CAPITAL LETTER C +16E52; C; 16E72; # MEDEFAIDRIN CAPITAL LETTER U +16E53; C; 16E73; # MEDEFAIDRIN CAPITAL LETTER YU +16E54; C; 16E74; # MEDEFAIDRIN CAPITAL LETTER L +16E55; C; 16E75; # MEDEFAIDRIN CAPITAL LETTER Q +16E56; C; 16E76; # MEDEFAIDRIN CAPITAL LETTER HP +16E57; C; 16E77; # MEDEFAIDRIN CAPITAL LETTER NY +16E58; C; 16E78; # MEDEFAIDRIN CAPITAL LETTER X +16E59; C; 16E79; # MEDEFAIDRIN CAPITAL LETTER D +16E5A; C; 16E7A; # MEDEFAIDRIN CAPITAL LETTER OE +16E5B; C; 16E7B; # MEDEFAIDRIN CAPITAL LETTER N +16E5C; C; 16E7C; # MEDEFAIDRIN CAPITAL LETTER R +16E5D; C; 16E7D; # MEDEFAIDRIN CAPITAL LETTER O +16E5E; C; 16E7E; # MEDEFAIDRIN CAPITAL LETTER AI +16E5F; C; 16E7F; # MEDEFAIDRIN CAPITAL LETTER Y +1E900; C; 1E922; # ADLAM CAPITAL LETTER ALIF +1E901; C; 1E923; # ADLAM CAPITAL LETTER DAALI +1E902; C; 1E924; # ADLAM CAPITAL LETTER LAAM +1E903; C; 1E925; # ADLAM CAPITAL LETTER MIIM +1E904; C; 1E926; # ADLAM CAPITAL LETTER BA +1E905; C; 1E927; # ADLAM CAPITAL LETTER SINNYIIYHE +1E906; C; 1E928; # ADLAM CAPITAL LETTER PE +1E907; C; 1E929; # ADLAM CAPITAL LETTER BHE +1E908; C; 1E92A; # ADLAM CAPITAL LETTER RA +1E909; C; 1E92B; # ADLAM CAPITAL LETTER E +1E90A; C; 1E92C; # ADLAM CAPITAL LETTER FA +1E90B; C; 1E92D; # ADLAM CAPITAL LETTER I +1E90C; C; 1E92E; # ADLAM CAPITAL LETTER O +1E90D; C; 1E92F; # ADLAM CAPITAL LETTER DHA +1E90E; C; 1E930; # ADLAM CAPITAL LETTER YHE +1E90F; C; 1E931; # ADLAM CAPITAL LETTER WAW +1E910; C; 1E932; # ADLAM CAPITAL LETTER NUN +1E911; C; 1E933; # ADLAM CAPITAL LETTER KAF +1E912; C; 1E934; # ADLAM CAPITAL LETTER YA +1E913; C; 1E935; # ADLAM CAPITAL LETTER U +1E914; C; 1E936; # ADLAM CAPITAL LETTER JIIM +1E915; C; 1E937; # ADLAM CAPITAL LETTER CHI +1E916; C; 1E938; # ADLAM CAPITAL LETTER HA +1E917; C; 1E939; # ADLAM CAPITAL LETTER QAAF +1E918; C; 1E93A; # ADLAM CAPITAL LETTER GA +1E919; C; 1E93B; # ADLAM CAPITAL LETTER NYA +1E91A; C; 1E93C; # ADLAM CAPITAL LETTER TU +1E91B; C; 1E93D; # ADLAM CAPITAL LETTER NHA +1E91C; C; 1E93E; # ADLAM CAPITAL LETTER VA +1E91D; C; 1E93F; # ADLAM CAPITAL LETTER KHA +1E91E; C; 1E940; # ADLAM CAPITAL LETTER GBE +1E91F; C; 1E941; # ADLAM CAPITAL LETTER ZAL +1E920; C; 1E942; # ADLAM CAPITAL LETTER KPO +1E921; C; 1E943; # ADLAM CAPITAL LETTER SHA +# +# EOF diff --git a/claimtrie/normalization/CaseFolding_v13.txt b/claimtrie/normalization/CaseFolding_v13.txt new file mode 100644 index 00000000..033788b2 --- /dev/null +++ b/claimtrie/normalization/CaseFolding_v13.txt @@ -0,0 +1,1584 @@ +# CaseFolding-13.0.0.txt +# Date: 2019-09-08, 23:30:59 GMT +# © 2019 Unicode®, Inc. +# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. +# For terms of use, see http://www.unicode.org/terms_of_use.html +# +# Unicode Character Database +# For documentation, see http://www.unicode.org/reports/tr44/ +# +# Case Folding Properties +# +# This file is a supplement to the UnicodeData file. +# It provides a case folding mapping generated from the Unicode Character Database. +# If all characters are mapped according to the full mapping below, then +# case differences (according to UnicodeData.txt and SpecialCasing.txt) +# are eliminated. +# +# The data supports both implementations that require simple case foldings +# (where string lengths don't change), and implementations that allow full case folding +# (where string lengths may grow). Note that where they can be supported, the +# full case foldings are superior: for example, they allow "MASSE" and "Maße" to match. +# +# All code points not listed in this file map to themselves. +# +# NOTE: case folding does not preserve normalization formats! +# +# For information on case folding, including how to have case folding +# preserve normalization formats, see Section 3.13 Default Case Algorithms in +# The Unicode Standard. +# +# ================================================================================ +# Format +# ================================================================================ +# The entries in this file are in the following machine-readable format: +# +# ; ; ; # +# +# The status field is: +# C: common case folding, common mappings shared by both simple and full mappings. +# F: full case folding, mappings that cause strings to grow in length. Multiple characters are separated by spaces. +# S: simple case folding, mappings to single characters where different from F. +# T: special case for uppercase I and dotted uppercase I +# - For non-Turkic languages, this mapping is normally not used. +# - For Turkic languages (tr, az), this mapping can be used instead of the normal mapping for these characters. +# Note that the Turkic mappings do not maintain canonical equivalence without additional processing. +# See the discussions of case mapping in the Unicode Standard for more information. +# +# Usage: +# A. To do a simple case folding, use the mappings with status C + S. +# B. To do a full case folding, use the mappings with status C + F. +# +# The mappings with status T can be used or omitted depending on the desired case-folding +# behavior. (The default option is to exclude them.) +# +# ================================================================= + +# Property: Case_Folding + +# All code points not explicitly listed for Case_Folding +# have the value C for the status field, and the code point itself for the mapping field. + +# ================================================================= +0041; C; 0061; # LATIN CAPITAL LETTER A +0042; C; 0062; # LATIN CAPITAL LETTER B +0043; C; 0063; # LATIN CAPITAL LETTER C +0044; C; 0064; # LATIN CAPITAL LETTER D +0045; C; 0065; # LATIN CAPITAL LETTER E +0046; C; 0066; # LATIN CAPITAL LETTER F +0047; C; 0067; # LATIN CAPITAL LETTER G +0048; C; 0068; # LATIN CAPITAL LETTER H +0049; C; 0069; # LATIN CAPITAL LETTER I +0049; T; 0131; # LATIN CAPITAL LETTER I +004A; C; 006A; # LATIN CAPITAL LETTER J +004B; C; 006B; # LATIN CAPITAL LETTER K +004C; C; 006C; # LATIN CAPITAL LETTER L +004D; C; 006D; # LATIN CAPITAL LETTER M +004E; C; 006E; # LATIN CAPITAL LETTER N +004F; C; 006F; # LATIN CAPITAL LETTER O +0050; C; 0070; # LATIN CAPITAL LETTER P +0051; C; 0071; # LATIN CAPITAL LETTER Q +0052; C; 0072; # LATIN CAPITAL LETTER R +0053; C; 0073; # LATIN CAPITAL LETTER S +0054; C; 0074; # LATIN CAPITAL LETTER T +0055; C; 0075; # LATIN CAPITAL LETTER U +0056; C; 0076; # LATIN CAPITAL LETTER V +0057; C; 0077; # LATIN CAPITAL LETTER W +0058; C; 0078; # LATIN CAPITAL LETTER X +0059; C; 0079; # LATIN CAPITAL LETTER Y +005A; C; 007A; # LATIN CAPITAL LETTER Z +00B5; C; 03BC; # MICRO SIGN +00C0; C; 00E0; # LATIN CAPITAL LETTER A WITH GRAVE +00C1; C; 00E1; # LATIN CAPITAL LETTER A WITH ACUTE +00C2; C; 00E2; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX +00C3; C; 00E3; # LATIN CAPITAL LETTER A WITH TILDE +00C4; C; 00E4; # LATIN CAPITAL LETTER A WITH DIAERESIS +00C5; C; 00E5; # LATIN CAPITAL LETTER A WITH RING ABOVE +00C6; C; 00E6; # LATIN CAPITAL LETTER AE +00C7; C; 00E7; # LATIN CAPITAL LETTER C WITH CEDILLA +00C8; C; 00E8; # LATIN CAPITAL LETTER E WITH GRAVE +00C9; C; 00E9; # LATIN CAPITAL LETTER E WITH ACUTE +00CA; C; 00EA; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX +00CB; C; 00EB; # LATIN CAPITAL LETTER E WITH DIAERESIS +00CC; C; 00EC; # LATIN CAPITAL LETTER I WITH GRAVE +00CD; C; 00ED; # LATIN CAPITAL LETTER I WITH ACUTE +00CE; C; 00EE; # LATIN CAPITAL LETTER I WITH CIRCUMFLEX +00CF; C; 00EF; # LATIN CAPITAL LETTER I WITH DIAERESIS +00D0; C; 00F0; # LATIN CAPITAL LETTER ETH +00D1; C; 00F1; # LATIN CAPITAL LETTER N WITH TILDE +00D2; C; 00F2; # LATIN CAPITAL LETTER O WITH GRAVE +00D3; C; 00F3; # LATIN CAPITAL LETTER O WITH ACUTE +00D4; C; 00F4; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX +00D5; C; 00F5; # LATIN CAPITAL LETTER O WITH TILDE +00D6; C; 00F6; # LATIN CAPITAL LETTER O WITH DIAERESIS +00D8; C; 00F8; # LATIN CAPITAL LETTER O WITH STROKE +00D9; C; 00F9; # LATIN CAPITAL LETTER U WITH GRAVE +00DA; C; 00FA; # LATIN CAPITAL LETTER U WITH ACUTE +00DB; C; 00FB; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX +00DC; C; 00FC; # LATIN CAPITAL LETTER U WITH DIAERESIS +00DD; C; 00FD; # LATIN CAPITAL LETTER Y WITH ACUTE +00DE; C; 00FE; # LATIN CAPITAL LETTER THORN +00DF; F; 0073 0073; # LATIN SMALL LETTER SHARP S +0100; C; 0101; # LATIN CAPITAL LETTER A WITH MACRON +0102; C; 0103; # LATIN CAPITAL LETTER A WITH BREVE +0104; C; 0105; # LATIN CAPITAL LETTER A WITH OGONEK +0106; C; 0107; # LATIN CAPITAL LETTER C WITH ACUTE +0108; C; 0109; # LATIN CAPITAL LETTER C WITH CIRCUMFLEX +010A; C; 010B; # LATIN CAPITAL LETTER C WITH DOT ABOVE +010C; C; 010D; # LATIN CAPITAL LETTER C WITH CARON +010E; C; 010F; # LATIN CAPITAL LETTER D WITH CARON +0110; C; 0111; # LATIN CAPITAL LETTER D WITH STROKE +0112; C; 0113; # LATIN CAPITAL LETTER E WITH MACRON +0114; C; 0115; # LATIN CAPITAL LETTER E WITH BREVE +0116; C; 0117; # LATIN CAPITAL LETTER E WITH DOT ABOVE +0118; C; 0119; # LATIN CAPITAL LETTER E WITH OGONEK +011A; C; 011B; # LATIN CAPITAL LETTER E WITH CARON +011C; C; 011D; # LATIN CAPITAL LETTER G WITH CIRCUMFLEX +011E; C; 011F; # LATIN CAPITAL LETTER G WITH BREVE +0120; C; 0121; # LATIN CAPITAL LETTER G WITH DOT ABOVE +0122; C; 0123; # LATIN CAPITAL LETTER G WITH CEDILLA +0124; C; 0125; # LATIN CAPITAL LETTER H WITH CIRCUMFLEX +0126; C; 0127; # LATIN CAPITAL LETTER H WITH STROKE +0128; C; 0129; # LATIN CAPITAL LETTER I WITH TILDE +012A; C; 012B; # LATIN CAPITAL LETTER I WITH MACRON +012C; C; 012D; # LATIN CAPITAL LETTER I WITH BREVE +012E; C; 012F; # LATIN CAPITAL LETTER I WITH OGONEK +0130; F; 0069 0307; # LATIN CAPITAL LETTER I WITH DOT ABOVE +0130; T; 0069; # LATIN CAPITAL LETTER I WITH DOT ABOVE +0132; C; 0133; # LATIN CAPITAL LIGATURE IJ +0134; C; 0135; # LATIN CAPITAL LETTER J WITH CIRCUMFLEX +0136; C; 0137; # LATIN CAPITAL LETTER K WITH CEDILLA +0139; C; 013A; # LATIN CAPITAL LETTER L WITH ACUTE +013B; C; 013C; # LATIN CAPITAL LETTER L WITH CEDILLA +013D; C; 013E; # LATIN CAPITAL LETTER L WITH CARON +013F; C; 0140; # LATIN CAPITAL LETTER L WITH MIDDLE DOT +0141; C; 0142; # LATIN CAPITAL LETTER L WITH STROKE +0143; C; 0144; # LATIN CAPITAL LETTER N WITH ACUTE +0145; C; 0146; # LATIN CAPITAL LETTER N WITH CEDILLA +0147; C; 0148; # LATIN CAPITAL LETTER N WITH CARON +0149; F; 02BC 006E; # LATIN SMALL LETTER N PRECEDED BY APOSTROPHE +014A; C; 014B; # LATIN CAPITAL LETTER ENG +014C; C; 014D; # LATIN CAPITAL LETTER O WITH MACRON +014E; C; 014F; # LATIN CAPITAL LETTER O WITH BREVE +0150; C; 0151; # LATIN CAPITAL LETTER O WITH DOUBLE ACUTE +0152; C; 0153; # LATIN CAPITAL LIGATURE OE +0154; C; 0155; # LATIN CAPITAL LETTER R WITH ACUTE +0156; C; 0157; # LATIN CAPITAL LETTER R WITH CEDILLA +0158; C; 0159; # LATIN CAPITAL LETTER R WITH CARON +015A; C; 015B; # LATIN CAPITAL LETTER S WITH ACUTE +015C; C; 015D; # LATIN CAPITAL LETTER S WITH CIRCUMFLEX +015E; C; 015F; # LATIN CAPITAL LETTER S WITH CEDILLA +0160; C; 0161; # LATIN CAPITAL LETTER S WITH CARON +0162; C; 0163; # LATIN CAPITAL LETTER T WITH CEDILLA +0164; C; 0165; # LATIN CAPITAL LETTER T WITH CARON +0166; C; 0167; # LATIN CAPITAL LETTER T WITH STROKE +0168; C; 0169; # LATIN CAPITAL LETTER U WITH TILDE +016A; C; 016B; # LATIN CAPITAL LETTER U WITH MACRON +016C; C; 016D; # LATIN CAPITAL LETTER U WITH BREVE +016E; C; 016F; # LATIN CAPITAL LETTER U WITH RING ABOVE +0170; C; 0171; # LATIN CAPITAL LETTER U WITH DOUBLE ACUTE +0172; C; 0173; # LATIN CAPITAL LETTER U WITH OGONEK +0174; C; 0175; # LATIN CAPITAL LETTER W WITH CIRCUMFLEX +0176; C; 0177; # LATIN CAPITAL LETTER Y WITH CIRCUMFLEX +0178; C; 00FF; # LATIN CAPITAL LETTER Y WITH DIAERESIS +0179; C; 017A; # LATIN CAPITAL LETTER Z WITH ACUTE +017B; C; 017C; # LATIN CAPITAL LETTER Z WITH DOT ABOVE +017D; C; 017E; # LATIN CAPITAL LETTER Z WITH CARON +017F; C; 0073; # LATIN SMALL LETTER LONG S +0181; C; 0253; # LATIN CAPITAL LETTER B WITH HOOK +0182; C; 0183; # LATIN CAPITAL LETTER B WITH TOPBAR +0184; C; 0185; # LATIN CAPITAL LETTER TONE SIX +0186; C; 0254; # LATIN CAPITAL LETTER OPEN O +0187; C; 0188; # LATIN CAPITAL LETTER C WITH HOOK +0189; C; 0256; # LATIN CAPITAL LETTER AFRICAN D +018A; C; 0257; # LATIN CAPITAL LETTER D WITH HOOK +018B; C; 018C; # LATIN CAPITAL LETTER D WITH TOPBAR +018E; C; 01DD; # LATIN CAPITAL LETTER REVERSED E +018F; C; 0259; # LATIN CAPITAL LETTER SCHWA +0190; C; 025B; # LATIN CAPITAL LETTER OPEN E +0191; C; 0192; # LATIN CAPITAL LETTER F WITH HOOK +0193; C; 0260; # LATIN CAPITAL LETTER G WITH HOOK +0194; C; 0263; # LATIN CAPITAL LETTER GAMMA +0196; C; 0269; # LATIN CAPITAL LETTER IOTA +0197; C; 0268; # LATIN CAPITAL LETTER I WITH STROKE +0198; C; 0199; # LATIN CAPITAL LETTER K WITH HOOK +019C; C; 026F; # LATIN CAPITAL LETTER TURNED M +019D; C; 0272; # LATIN CAPITAL LETTER N WITH LEFT HOOK +019F; C; 0275; # LATIN CAPITAL LETTER O WITH MIDDLE TILDE +01A0; C; 01A1; # LATIN CAPITAL LETTER O WITH HORN +01A2; C; 01A3; # LATIN CAPITAL LETTER OI +01A4; C; 01A5; # LATIN CAPITAL LETTER P WITH HOOK +01A6; C; 0280; # LATIN LETTER YR +01A7; C; 01A8; # LATIN CAPITAL LETTER TONE TWO +01A9; C; 0283; # LATIN CAPITAL LETTER ESH +01AC; C; 01AD; # LATIN CAPITAL LETTER T WITH HOOK +01AE; C; 0288; # LATIN CAPITAL LETTER T WITH RETROFLEX HOOK +01AF; C; 01B0; # LATIN CAPITAL LETTER U WITH HORN +01B1; C; 028A; # LATIN CAPITAL LETTER UPSILON +01B2; C; 028B; # LATIN CAPITAL LETTER V WITH HOOK +01B3; C; 01B4; # LATIN CAPITAL LETTER Y WITH HOOK +01B5; C; 01B6; # LATIN CAPITAL LETTER Z WITH STROKE +01B7; C; 0292; # LATIN CAPITAL LETTER EZH +01B8; C; 01B9; # LATIN CAPITAL LETTER EZH REVERSED +01BC; C; 01BD; # LATIN CAPITAL LETTER TONE FIVE +01C4; C; 01C6; # LATIN CAPITAL LETTER DZ WITH CARON +01C5; C; 01C6; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON +01C7; C; 01C9; # LATIN CAPITAL LETTER LJ +01C8; C; 01C9; # LATIN CAPITAL LETTER L WITH SMALL LETTER J +01CA; C; 01CC; # LATIN CAPITAL LETTER NJ +01CB; C; 01CC; # LATIN CAPITAL LETTER N WITH SMALL LETTER J +01CD; C; 01CE; # LATIN CAPITAL LETTER A WITH CARON +01CF; C; 01D0; # LATIN CAPITAL LETTER I WITH CARON +01D1; C; 01D2; # LATIN CAPITAL LETTER O WITH CARON +01D3; C; 01D4; # LATIN CAPITAL LETTER U WITH CARON +01D5; C; 01D6; # LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON +01D7; C; 01D8; # LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE +01D9; C; 01DA; # LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON +01DB; C; 01DC; # LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE +01DE; C; 01DF; # LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON +01E0; C; 01E1; # LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON +01E2; C; 01E3; # LATIN CAPITAL LETTER AE WITH MACRON +01E4; C; 01E5; # LATIN CAPITAL LETTER G WITH STROKE +01E6; C; 01E7; # LATIN CAPITAL LETTER G WITH CARON +01E8; C; 01E9; # LATIN CAPITAL LETTER K WITH CARON +01EA; C; 01EB; # LATIN CAPITAL LETTER O WITH OGONEK +01EC; C; 01ED; # LATIN CAPITAL LETTER O WITH OGONEK AND MACRON +01EE; C; 01EF; # LATIN CAPITAL LETTER EZH WITH CARON +01F0; F; 006A 030C; # LATIN SMALL LETTER J WITH CARON +01F1; C; 01F3; # LATIN CAPITAL LETTER DZ +01F2; C; 01F3; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z +01F4; C; 01F5; # LATIN CAPITAL LETTER G WITH ACUTE +01F6; C; 0195; # LATIN CAPITAL LETTER HWAIR +01F7; C; 01BF; # LATIN CAPITAL LETTER WYNN +01F8; C; 01F9; # LATIN CAPITAL LETTER N WITH GRAVE +01FA; C; 01FB; # LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE +01FC; C; 01FD; # LATIN CAPITAL LETTER AE WITH ACUTE +01FE; C; 01FF; # LATIN CAPITAL LETTER O WITH STROKE AND ACUTE +0200; C; 0201; # LATIN CAPITAL LETTER A WITH DOUBLE GRAVE +0202; C; 0203; # LATIN CAPITAL LETTER A WITH INVERTED BREVE +0204; C; 0205; # LATIN CAPITAL LETTER E WITH DOUBLE GRAVE +0206; C; 0207; # LATIN CAPITAL LETTER E WITH INVERTED BREVE +0208; C; 0209; # LATIN CAPITAL LETTER I WITH DOUBLE GRAVE +020A; C; 020B; # LATIN CAPITAL LETTER I WITH INVERTED BREVE +020C; C; 020D; # LATIN CAPITAL LETTER O WITH DOUBLE GRAVE +020E; C; 020F; # LATIN CAPITAL LETTER O WITH INVERTED BREVE +0210; C; 0211; # LATIN CAPITAL LETTER R WITH DOUBLE GRAVE +0212; C; 0213; # LATIN CAPITAL LETTER R WITH INVERTED BREVE +0214; C; 0215; # LATIN CAPITAL LETTER U WITH DOUBLE GRAVE +0216; C; 0217; # LATIN CAPITAL LETTER U WITH INVERTED BREVE +0218; C; 0219; # LATIN CAPITAL LETTER S WITH COMMA BELOW +021A; C; 021B; # LATIN CAPITAL LETTER T WITH COMMA BELOW +021C; C; 021D; # LATIN CAPITAL LETTER YOGH +021E; C; 021F; # LATIN CAPITAL LETTER H WITH CARON +0220; C; 019E; # LATIN CAPITAL LETTER N WITH LONG RIGHT LEG +0222; C; 0223; # LATIN CAPITAL LETTER OU +0224; C; 0225; # LATIN CAPITAL LETTER Z WITH HOOK +0226; C; 0227; # LATIN CAPITAL LETTER A WITH DOT ABOVE +0228; C; 0229; # LATIN CAPITAL LETTER E WITH CEDILLA +022A; C; 022B; # LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON +022C; C; 022D; # LATIN CAPITAL LETTER O WITH TILDE AND MACRON +022E; C; 022F; # LATIN CAPITAL LETTER O WITH DOT ABOVE +0230; C; 0231; # LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON +0232; C; 0233; # LATIN CAPITAL LETTER Y WITH MACRON +023A; C; 2C65; # LATIN CAPITAL LETTER A WITH STROKE +023B; C; 023C; # LATIN CAPITAL LETTER C WITH STROKE +023D; C; 019A; # LATIN CAPITAL LETTER L WITH BAR +023E; C; 2C66; # LATIN CAPITAL LETTER T WITH DIAGONAL STROKE +0241; C; 0242; # LATIN CAPITAL LETTER GLOTTAL STOP +0243; C; 0180; # LATIN CAPITAL LETTER B WITH STROKE +0244; C; 0289; # LATIN CAPITAL LETTER U BAR +0245; C; 028C; # LATIN CAPITAL LETTER TURNED V +0246; C; 0247; # LATIN CAPITAL LETTER E WITH STROKE +0248; C; 0249; # LATIN CAPITAL LETTER J WITH STROKE +024A; C; 024B; # LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL +024C; C; 024D; # LATIN CAPITAL LETTER R WITH STROKE +024E; C; 024F; # LATIN CAPITAL LETTER Y WITH STROKE +0345; C; 03B9; # COMBINING GREEK YPOGEGRAMMENI +0370; C; 0371; # GREEK CAPITAL LETTER HETA +0372; C; 0373; # GREEK CAPITAL LETTER ARCHAIC SAMPI +0376; C; 0377; # GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA +037F; C; 03F3; # GREEK CAPITAL LETTER YOT +0386; C; 03AC; # GREEK CAPITAL LETTER ALPHA WITH TONOS +0388; C; 03AD; # GREEK CAPITAL LETTER EPSILON WITH TONOS +0389; C; 03AE; # GREEK CAPITAL LETTER ETA WITH TONOS +038A; C; 03AF; # GREEK CAPITAL LETTER IOTA WITH TONOS +038C; C; 03CC; # GREEK CAPITAL LETTER OMICRON WITH TONOS +038E; C; 03CD; # GREEK CAPITAL LETTER UPSILON WITH TONOS +038F; C; 03CE; # GREEK CAPITAL LETTER OMEGA WITH TONOS +0390; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS +0391; C; 03B1; # GREEK CAPITAL LETTER ALPHA +0392; C; 03B2; # GREEK CAPITAL LETTER BETA +0393; C; 03B3; # GREEK CAPITAL LETTER GAMMA +0394; C; 03B4; # GREEK CAPITAL LETTER DELTA +0395; C; 03B5; # GREEK CAPITAL LETTER EPSILON +0396; C; 03B6; # GREEK CAPITAL LETTER ZETA +0397; C; 03B7; # GREEK CAPITAL LETTER ETA +0398; C; 03B8; # GREEK CAPITAL LETTER THETA +0399; C; 03B9; # GREEK CAPITAL LETTER IOTA +039A; C; 03BA; # GREEK CAPITAL LETTER KAPPA +039B; C; 03BB; # GREEK CAPITAL LETTER LAMDA +039C; C; 03BC; # GREEK CAPITAL LETTER MU +039D; C; 03BD; # GREEK CAPITAL LETTER NU +039E; C; 03BE; # GREEK CAPITAL LETTER XI +039F; C; 03BF; # GREEK CAPITAL LETTER OMICRON +03A0; C; 03C0; # GREEK CAPITAL LETTER PI +03A1; C; 03C1; # GREEK CAPITAL LETTER RHO +03A3; C; 03C3; # GREEK CAPITAL LETTER SIGMA +03A4; C; 03C4; # GREEK CAPITAL LETTER TAU +03A5; C; 03C5; # GREEK CAPITAL LETTER UPSILON +03A6; C; 03C6; # GREEK CAPITAL LETTER PHI +03A7; C; 03C7; # GREEK CAPITAL LETTER CHI +03A8; C; 03C8; # GREEK CAPITAL LETTER PSI +03A9; C; 03C9; # GREEK CAPITAL LETTER OMEGA +03AA; C; 03CA; # GREEK CAPITAL LETTER IOTA WITH DIALYTIKA +03AB; C; 03CB; # GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA +03B0; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS +03C2; C; 03C3; # GREEK SMALL LETTER FINAL SIGMA +03CF; C; 03D7; # GREEK CAPITAL KAI SYMBOL +03D0; C; 03B2; # GREEK BETA SYMBOL +03D1; C; 03B8; # GREEK THETA SYMBOL +03D5; C; 03C6; # GREEK PHI SYMBOL +03D6; C; 03C0; # GREEK PI SYMBOL +03D8; C; 03D9; # GREEK LETTER ARCHAIC KOPPA +03DA; C; 03DB; # GREEK LETTER STIGMA +03DC; C; 03DD; # GREEK LETTER DIGAMMA +03DE; C; 03DF; # GREEK LETTER KOPPA +03E0; C; 03E1; # GREEK LETTER SAMPI +03E2; C; 03E3; # COPTIC CAPITAL LETTER SHEI +03E4; C; 03E5; # COPTIC CAPITAL LETTER FEI +03E6; C; 03E7; # COPTIC CAPITAL LETTER KHEI +03E8; C; 03E9; # COPTIC CAPITAL LETTER HORI +03EA; C; 03EB; # COPTIC CAPITAL LETTER GANGIA +03EC; C; 03ED; # COPTIC CAPITAL LETTER SHIMA +03EE; C; 03EF; # COPTIC CAPITAL LETTER DEI +03F0; C; 03BA; # GREEK KAPPA SYMBOL +03F1; C; 03C1; # GREEK RHO SYMBOL +03F4; C; 03B8; # GREEK CAPITAL THETA SYMBOL +03F5; C; 03B5; # GREEK LUNATE EPSILON SYMBOL +03F7; C; 03F8; # GREEK CAPITAL LETTER SHO +03F9; C; 03F2; # GREEK CAPITAL LUNATE SIGMA SYMBOL +03FA; C; 03FB; # GREEK CAPITAL LETTER SAN +03FD; C; 037B; # GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL +03FE; C; 037C; # GREEK CAPITAL DOTTED LUNATE SIGMA SYMBOL +03FF; C; 037D; # GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL +0400; C; 0450; # CYRILLIC CAPITAL LETTER IE WITH GRAVE +0401; C; 0451; # CYRILLIC CAPITAL LETTER IO +0402; C; 0452; # CYRILLIC CAPITAL LETTER DJE +0403; C; 0453; # CYRILLIC CAPITAL LETTER GJE +0404; C; 0454; # CYRILLIC CAPITAL LETTER UKRAINIAN IE +0405; C; 0455; # CYRILLIC CAPITAL LETTER DZE +0406; C; 0456; # CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I +0407; C; 0457; # CYRILLIC CAPITAL LETTER YI +0408; C; 0458; # CYRILLIC CAPITAL LETTER JE +0409; C; 0459; # CYRILLIC CAPITAL LETTER LJE +040A; C; 045A; # CYRILLIC CAPITAL LETTER NJE +040B; C; 045B; # CYRILLIC CAPITAL LETTER TSHE +040C; C; 045C; # CYRILLIC CAPITAL LETTER KJE +040D; C; 045D; # CYRILLIC CAPITAL LETTER I WITH GRAVE +040E; C; 045E; # CYRILLIC CAPITAL LETTER SHORT U +040F; C; 045F; # CYRILLIC CAPITAL LETTER DZHE +0410; C; 0430; # CYRILLIC CAPITAL LETTER A +0411; C; 0431; # CYRILLIC CAPITAL LETTER BE +0412; C; 0432; # CYRILLIC CAPITAL LETTER VE +0413; C; 0433; # CYRILLIC CAPITAL LETTER GHE +0414; C; 0434; # CYRILLIC CAPITAL LETTER DE +0415; C; 0435; # CYRILLIC CAPITAL LETTER IE +0416; C; 0436; # CYRILLIC CAPITAL LETTER ZHE +0417; C; 0437; # CYRILLIC CAPITAL LETTER ZE +0418; C; 0438; # CYRILLIC CAPITAL LETTER I +0419; C; 0439; # CYRILLIC CAPITAL LETTER SHORT I +041A; C; 043A; # CYRILLIC CAPITAL LETTER KA +041B; C; 043B; # CYRILLIC CAPITAL LETTER EL +041C; C; 043C; # CYRILLIC CAPITAL LETTER EM +041D; C; 043D; # CYRILLIC CAPITAL LETTER EN +041E; C; 043E; # CYRILLIC CAPITAL LETTER O +041F; C; 043F; # CYRILLIC CAPITAL LETTER PE +0420; C; 0440; # CYRILLIC CAPITAL LETTER ER +0421; C; 0441; # CYRILLIC CAPITAL LETTER ES +0422; C; 0442; # CYRILLIC CAPITAL LETTER TE +0423; C; 0443; # CYRILLIC CAPITAL LETTER U +0424; C; 0444; # CYRILLIC CAPITAL LETTER EF +0425; C; 0445; # CYRILLIC CAPITAL LETTER HA +0426; C; 0446; # CYRILLIC CAPITAL LETTER TSE +0427; C; 0447; # CYRILLIC CAPITAL LETTER CHE +0428; C; 0448; # CYRILLIC CAPITAL LETTER SHA +0429; C; 0449; # CYRILLIC CAPITAL LETTER SHCHA +042A; C; 044A; # CYRILLIC CAPITAL LETTER HARD SIGN +042B; C; 044B; # CYRILLIC CAPITAL LETTER YERU +042C; C; 044C; # CYRILLIC CAPITAL LETTER SOFT SIGN +042D; C; 044D; # CYRILLIC CAPITAL LETTER E +042E; C; 044E; # CYRILLIC CAPITAL LETTER YU +042F; C; 044F; # CYRILLIC CAPITAL LETTER YA +0460; C; 0461; # CYRILLIC CAPITAL LETTER OMEGA +0462; C; 0463; # CYRILLIC CAPITAL LETTER YAT +0464; C; 0465; # CYRILLIC CAPITAL LETTER IOTIFIED E +0466; C; 0467; # CYRILLIC CAPITAL LETTER LITTLE YUS +0468; C; 0469; # CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS +046A; C; 046B; # CYRILLIC CAPITAL LETTER BIG YUS +046C; C; 046D; # CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS +046E; C; 046F; # CYRILLIC CAPITAL LETTER KSI +0470; C; 0471; # CYRILLIC CAPITAL LETTER PSI +0472; C; 0473; # CYRILLIC CAPITAL LETTER FITA +0474; C; 0475; # CYRILLIC CAPITAL LETTER IZHITSA +0476; C; 0477; # CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT +0478; C; 0479; # CYRILLIC CAPITAL LETTER UK +047A; C; 047B; # CYRILLIC CAPITAL LETTER ROUND OMEGA +047C; C; 047D; # CYRILLIC CAPITAL LETTER OMEGA WITH TITLO +047E; C; 047F; # CYRILLIC CAPITAL LETTER OT +0480; C; 0481; # CYRILLIC CAPITAL LETTER KOPPA +048A; C; 048B; # CYRILLIC CAPITAL LETTER SHORT I WITH TAIL +048C; C; 048D; # CYRILLIC CAPITAL LETTER SEMISOFT SIGN +048E; C; 048F; # CYRILLIC CAPITAL LETTER ER WITH TICK +0490; C; 0491; # CYRILLIC CAPITAL LETTER GHE WITH UPTURN +0492; C; 0493; # CYRILLIC CAPITAL LETTER GHE WITH STROKE +0494; C; 0495; # CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK +0496; C; 0497; # CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER +0498; C; 0499; # CYRILLIC CAPITAL LETTER ZE WITH DESCENDER +049A; C; 049B; # CYRILLIC CAPITAL LETTER KA WITH DESCENDER +049C; C; 049D; # CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE +049E; C; 049F; # CYRILLIC CAPITAL LETTER KA WITH STROKE +04A0; C; 04A1; # CYRILLIC CAPITAL LETTER BASHKIR KA +04A2; C; 04A3; # CYRILLIC CAPITAL LETTER EN WITH DESCENDER +04A4; C; 04A5; # CYRILLIC CAPITAL LIGATURE EN GHE +04A6; C; 04A7; # CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK +04A8; C; 04A9; # CYRILLIC CAPITAL LETTER ABKHASIAN HA +04AA; C; 04AB; # CYRILLIC CAPITAL LETTER ES WITH DESCENDER +04AC; C; 04AD; # CYRILLIC CAPITAL LETTER TE WITH DESCENDER +04AE; C; 04AF; # CYRILLIC CAPITAL LETTER STRAIGHT U +04B0; C; 04B1; # CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE +04B2; C; 04B3; # CYRILLIC CAPITAL LETTER HA WITH DESCENDER +04B4; C; 04B5; # CYRILLIC CAPITAL LIGATURE TE TSE +04B6; C; 04B7; # CYRILLIC CAPITAL LETTER CHE WITH DESCENDER +04B8; C; 04B9; # CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE +04BA; C; 04BB; # CYRILLIC CAPITAL LETTER SHHA +04BC; C; 04BD; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE +04BE; C; 04BF; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER +04C0; C; 04CF; # CYRILLIC LETTER PALOCHKA +04C1; C; 04C2; # CYRILLIC CAPITAL LETTER ZHE WITH BREVE +04C3; C; 04C4; # CYRILLIC CAPITAL LETTER KA WITH HOOK +04C5; C; 04C6; # CYRILLIC CAPITAL LETTER EL WITH TAIL +04C7; C; 04C8; # CYRILLIC CAPITAL LETTER EN WITH HOOK +04C9; C; 04CA; # CYRILLIC CAPITAL LETTER EN WITH TAIL +04CB; C; 04CC; # CYRILLIC CAPITAL LETTER KHAKASSIAN CHE +04CD; C; 04CE; # CYRILLIC CAPITAL LETTER EM WITH TAIL +04D0; C; 04D1; # CYRILLIC CAPITAL LETTER A WITH BREVE +04D2; C; 04D3; # CYRILLIC CAPITAL LETTER A WITH DIAERESIS +04D4; C; 04D5; # CYRILLIC CAPITAL LIGATURE A IE +04D6; C; 04D7; # CYRILLIC CAPITAL LETTER IE WITH BREVE +04D8; C; 04D9; # CYRILLIC CAPITAL LETTER SCHWA +04DA; C; 04DB; # CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS +04DC; C; 04DD; # CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS +04DE; C; 04DF; # CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS +04E0; C; 04E1; # CYRILLIC CAPITAL LETTER ABKHASIAN DZE +04E2; C; 04E3; # CYRILLIC CAPITAL LETTER I WITH MACRON +04E4; C; 04E5; # CYRILLIC CAPITAL LETTER I WITH DIAERESIS +04E6; C; 04E7; # CYRILLIC CAPITAL LETTER O WITH DIAERESIS +04E8; C; 04E9; # CYRILLIC CAPITAL LETTER BARRED O +04EA; C; 04EB; # CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS +04EC; C; 04ED; # CYRILLIC CAPITAL LETTER E WITH DIAERESIS +04EE; C; 04EF; # CYRILLIC CAPITAL LETTER U WITH MACRON +04F0; C; 04F1; # CYRILLIC CAPITAL LETTER U WITH DIAERESIS +04F2; C; 04F3; # CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE +04F4; C; 04F5; # CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS +04F6; C; 04F7; # CYRILLIC CAPITAL LETTER GHE WITH DESCENDER +04F8; C; 04F9; # CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS +04FA; C; 04FB; # CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK +04FC; C; 04FD; # CYRILLIC CAPITAL LETTER HA WITH HOOK +04FE; C; 04FF; # CYRILLIC CAPITAL LETTER HA WITH STROKE +0500; C; 0501; # CYRILLIC CAPITAL LETTER KOMI DE +0502; C; 0503; # CYRILLIC CAPITAL LETTER KOMI DJE +0504; C; 0505; # CYRILLIC CAPITAL LETTER KOMI ZJE +0506; C; 0507; # CYRILLIC CAPITAL LETTER KOMI DZJE +0508; C; 0509; # CYRILLIC CAPITAL LETTER KOMI LJE +050A; C; 050B; # CYRILLIC CAPITAL LETTER KOMI NJE +050C; C; 050D; # CYRILLIC CAPITAL LETTER KOMI SJE +050E; C; 050F; # CYRILLIC CAPITAL LETTER KOMI TJE +0510; C; 0511; # CYRILLIC CAPITAL LETTER REVERSED ZE +0512; C; 0513; # CYRILLIC CAPITAL LETTER EL WITH HOOK +0514; C; 0515; # CYRILLIC CAPITAL LETTER LHA +0516; C; 0517; # CYRILLIC CAPITAL LETTER RHA +0518; C; 0519; # CYRILLIC CAPITAL LETTER YAE +051A; C; 051B; # CYRILLIC CAPITAL LETTER QA +051C; C; 051D; # CYRILLIC CAPITAL LETTER WE +051E; C; 051F; # CYRILLIC CAPITAL LETTER ALEUT KA +0520; C; 0521; # CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK +0522; C; 0523; # CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK +0524; C; 0525; # CYRILLIC CAPITAL LETTER PE WITH DESCENDER +0526; C; 0527; # CYRILLIC CAPITAL LETTER SHHA WITH DESCENDER +0528; C; 0529; # CYRILLIC CAPITAL LETTER EN WITH LEFT HOOK +052A; C; 052B; # CYRILLIC CAPITAL LETTER DZZHE +052C; C; 052D; # CYRILLIC CAPITAL LETTER DCHE +052E; C; 052F; # CYRILLIC CAPITAL LETTER EL WITH DESCENDER +0531; C; 0561; # ARMENIAN CAPITAL LETTER AYB +0532; C; 0562; # ARMENIAN CAPITAL LETTER BEN +0533; C; 0563; # ARMENIAN CAPITAL LETTER GIM +0534; C; 0564; # ARMENIAN CAPITAL LETTER DA +0535; C; 0565; # ARMENIAN CAPITAL LETTER ECH +0536; C; 0566; # ARMENIAN CAPITAL LETTER ZA +0537; C; 0567; # ARMENIAN CAPITAL LETTER EH +0538; C; 0568; # ARMENIAN CAPITAL LETTER ET +0539; C; 0569; # ARMENIAN CAPITAL LETTER TO +053A; C; 056A; # ARMENIAN CAPITAL LETTER ZHE +053B; C; 056B; # ARMENIAN CAPITAL LETTER INI +053C; C; 056C; # ARMENIAN CAPITAL LETTER LIWN +053D; C; 056D; # ARMENIAN CAPITAL LETTER XEH +053E; C; 056E; # ARMENIAN CAPITAL LETTER CA +053F; C; 056F; # ARMENIAN CAPITAL LETTER KEN +0540; C; 0570; # ARMENIAN CAPITAL LETTER HO +0541; C; 0571; # ARMENIAN CAPITAL LETTER JA +0542; C; 0572; # ARMENIAN CAPITAL LETTER GHAD +0543; C; 0573; # ARMENIAN CAPITAL LETTER CHEH +0544; C; 0574; # ARMENIAN CAPITAL LETTER MEN +0545; C; 0575; # ARMENIAN CAPITAL LETTER YI +0546; C; 0576; # ARMENIAN CAPITAL LETTER NOW +0547; C; 0577; # ARMENIAN CAPITAL LETTER SHA +0548; C; 0578; # ARMENIAN CAPITAL LETTER VO +0549; C; 0579; # ARMENIAN CAPITAL LETTER CHA +054A; C; 057A; # ARMENIAN CAPITAL LETTER PEH +054B; C; 057B; # ARMENIAN CAPITAL LETTER JHEH +054C; C; 057C; # ARMENIAN CAPITAL LETTER RA +054D; C; 057D; # ARMENIAN CAPITAL LETTER SEH +054E; C; 057E; # ARMENIAN CAPITAL LETTER VEW +054F; C; 057F; # ARMENIAN CAPITAL LETTER TIWN +0550; C; 0580; # ARMENIAN CAPITAL LETTER REH +0551; C; 0581; # ARMENIAN CAPITAL LETTER CO +0552; C; 0582; # ARMENIAN CAPITAL LETTER YIWN +0553; C; 0583; # ARMENIAN CAPITAL LETTER PIWR +0554; C; 0584; # ARMENIAN CAPITAL LETTER KEH +0555; C; 0585; # ARMENIAN CAPITAL LETTER OH +0556; C; 0586; # ARMENIAN CAPITAL LETTER FEH +0587; F; 0565 0582; # ARMENIAN SMALL LIGATURE ECH YIWN +10A0; C; 2D00; # GEORGIAN CAPITAL LETTER AN +10A1; C; 2D01; # GEORGIAN CAPITAL LETTER BAN +10A2; C; 2D02; # GEORGIAN CAPITAL LETTER GAN +10A3; C; 2D03; # GEORGIAN CAPITAL LETTER DON +10A4; C; 2D04; # GEORGIAN CAPITAL LETTER EN +10A5; C; 2D05; # GEORGIAN CAPITAL LETTER VIN +10A6; C; 2D06; # GEORGIAN CAPITAL LETTER ZEN +10A7; C; 2D07; # GEORGIAN CAPITAL LETTER TAN +10A8; C; 2D08; # GEORGIAN CAPITAL LETTER IN +10A9; C; 2D09; # GEORGIAN CAPITAL LETTER KAN +10AA; C; 2D0A; # GEORGIAN CAPITAL LETTER LAS +10AB; C; 2D0B; # GEORGIAN CAPITAL LETTER MAN +10AC; C; 2D0C; # GEORGIAN CAPITAL LETTER NAR +10AD; C; 2D0D; # GEORGIAN CAPITAL LETTER ON +10AE; C; 2D0E; # GEORGIAN CAPITAL LETTER PAR +10AF; C; 2D0F; # GEORGIAN CAPITAL LETTER ZHAR +10B0; C; 2D10; # GEORGIAN CAPITAL LETTER RAE +10B1; C; 2D11; # GEORGIAN CAPITAL LETTER SAN +10B2; C; 2D12; # GEORGIAN CAPITAL LETTER TAR +10B3; C; 2D13; # GEORGIAN CAPITAL LETTER UN +10B4; C; 2D14; # GEORGIAN CAPITAL LETTER PHAR +10B5; C; 2D15; # GEORGIAN CAPITAL LETTER KHAR +10B6; C; 2D16; # GEORGIAN CAPITAL LETTER GHAN +10B7; C; 2D17; # GEORGIAN CAPITAL LETTER QAR +10B8; C; 2D18; # GEORGIAN CAPITAL LETTER SHIN +10B9; C; 2D19; # GEORGIAN CAPITAL LETTER CHIN +10BA; C; 2D1A; # GEORGIAN CAPITAL LETTER CAN +10BB; C; 2D1B; # GEORGIAN CAPITAL LETTER JIL +10BC; C; 2D1C; # GEORGIAN CAPITAL LETTER CIL +10BD; C; 2D1D; # GEORGIAN CAPITAL LETTER CHAR +10BE; C; 2D1E; # GEORGIAN CAPITAL LETTER XAN +10BF; C; 2D1F; # GEORGIAN CAPITAL LETTER JHAN +10C0; C; 2D20; # GEORGIAN CAPITAL LETTER HAE +10C1; C; 2D21; # GEORGIAN CAPITAL LETTER HE +10C2; C; 2D22; # GEORGIAN CAPITAL LETTER HIE +10C3; C; 2D23; # GEORGIAN CAPITAL LETTER WE +10C4; C; 2D24; # GEORGIAN CAPITAL LETTER HAR +10C5; C; 2D25; # GEORGIAN CAPITAL LETTER HOE +10C7; C; 2D27; # GEORGIAN CAPITAL LETTER YN +10CD; C; 2D2D; # GEORGIAN CAPITAL LETTER AEN +13F8; C; 13F0; # CHEROKEE SMALL LETTER YE +13F9; C; 13F1; # CHEROKEE SMALL LETTER YI +13FA; C; 13F2; # CHEROKEE SMALL LETTER YO +13FB; C; 13F3; # CHEROKEE SMALL LETTER YU +13FC; C; 13F4; # CHEROKEE SMALL LETTER YV +13FD; C; 13F5; # CHEROKEE SMALL LETTER MV +1C80; C; 0432; # CYRILLIC SMALL LETTER ROUNDED VE +1C81; C; 0434; # CYRILLIC SMALL LETTER LONG-LEGGED DE +1C82; C; 043E; # CYRILLIC SMALL LETTER NARROW O +1C83; C; 0441; # CYRILLIC SMALL LETTER WIDE ES +1C84; C; 0442; # CYRILLIC SMALL LETTER TALL TE +1C85; C; 0442; # CYRILLIC SMALL LETTER THREE-LEGGED TE +1C86; C; 044A; # CYRILLIC SMALL LETTER TALL HARD SIGN +1C87; C; 0463; # CYRILLIC SMALL LETTER TALL YAT +1C88; C; A64B; # CYRILLIC SMALL LETTER UNBLENDED UK +1C90; C; 10D0; # GEORGIAN MTAVRULI CAPITAL LETTER AN +1C91; C; 10D1; # GEORGIAN MTAVRULI CAPITAL LETTER BAN +1C92; C; 10D2; # GEORGIAN MTAVRULI CAPITAL LETTER GAN +1C93; C; 10D3; # GEORGIAN MTAVRULI CAPITAL LETTER DON +1C94; C; 10D4; # GEORGIAN MTAVRULI CAPITAL LETTER EN +1C95; C; 10D5; # GEORGIAN MTAVRULI CAPITAL LETTER VIN +1C96; C; 10D6; # GEORGIAN MTAVRULI CAPITAL LETTER ZEN +1C97; C; 10D7; # GEORGIAN MTAVRULI CAPITAL LETTER TAN +1C98; C; 10D8; # GEORGIAN MTAVRULI CAPITAL LETTER IN +1C99; C; 10D9; # GEORGIAN MTAVRULI CAPITAL LETTER KAN +1C9A; C; 10DA; # GEORGIAN MTAVRULI CAPITAL LETTER LAS +1C9B; C; 10DB; # GEORGIAN MTAVRULI CAPITAL LETTER MAN +1C9C; C; 10DC; # GEORGIAN MTAVRULI CAPITAL LETTER NAR +1C9D; C; 10DD; # GEORGIAN MTAVRULI CAPITAL LETTER ON +1C9E; C; 10DE; # GEORGIAN MTAVRULI CAPITAL LETTER PAR +1C9F; C; 10DF; # GEORGIAN MTAVRULI CAPITAL LETTER ZHAR +1CA0; C; 10E0; # GEORGIAN MTAVRULI CAPITAL LETTER RAE +1CA1; C; 10E1; # GEORGIAN MTAVRULI CAPITAL LETTER SAN +1CA2; C; 10E2; # GEORGIAN MTAVRULI CAPITAL LETTER TAR +1CA3; C; 10E3; # GEORGIAN MTAVRULI CAPITAL LETTER UN +1CA4; C; 10E4; # GEORGIAN MTAVRULI CAPITAL LETTER PHAR +1CA5; C; 10E5; # GEORGIAN MTAVRULI CAPITAL LETTER KHAR +1CA6; C; 10E6; # GEORGIAN MTAVRULI CAPITAL LETTER GHAN +1CA7; C; 10E7; # GEORGIAN MTAVRULI CAPITAL LETTER QAR +1CA8; C; 10E8; # GEORGIAN MTAVRULI CAPITAL LETTER SHIN +1CA9; C; 10E9; # GEORGIAN MTAVRULI CAPITAL LETTER CHIN +1CAA; C; 10EA; # GEORGIAN MTAVRULI CAPITAL LETTER CAN +1CAB; C; 10EB; # GEORGIAN MTAVRULI CAPITAL LETTER JIL +1CAC; C; 10EC; # GEORGIAN MTAVRULI CAPITAL LETTER CIL +1CAD; C; 10ED; # GEORGIAN MTAVRULI CAPITAL LETTER CHAR +1CAE; C; 10EE; # GEORGIAN MTAVRULI CAPITAL LETTER XAN +1CAF; C; 10EF; # GEORGIAN MTAVRULI CAPITAL LETTER JHAN +1CB0; C; 10F0; # GEORGIAN MTAVRULI CAPITAL LETTER HAE +1CB1; C; 10F1; # GEORGIAN MTAVRULI CAPITAL LETTER HE +1CB2; C; 10F2; # GEORGIAN MTAVRULI CAPITAL LETTER HIE +1CB3; C; 10F3; # GEORGIAN MTAVRULI CAPITAL LETTER WE +1CB4; C; 10F4; # GEORGIAN MTAVRULI CAPITAL LETTER HAR +1CB5; C; 10F5; # GEORGIAN MTAVRULI CAPITAL LETTER HOE +1CB6; C; 10F6; # GEORGIAN MTAVRULI CAPITAL LETTER FI +1CB7; C; 10F7; # GEORGIAN MTAVRULI CAPITAL LETTER YN +1CB8; C; 10F8; # GEORGIAN MTAVRULI CAPITAL LETTER ELIFI +1CB9; C; 10F9; # GEORGIAN MTAVRULI CAPITAL LETTER TURNED GAN +1CBA; C; 10FA; # GEORGIAN MTAVRULI CAPITAL LETTER AIN +1CBD; C; 10FD; # GEORGIAN MTAVRULI CAPITAL LETTER AEN +1CBE; C; 10FE; # GEORGIAN MTAVRULI CAPITAL LETTER HARD SIGN +1CBF; C; 10FF; # GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN +1E00; C; 1E01; # LATIN CAPITAL LETTER A WITH RING BELOW +1E02; C; 1E03; # LATIN CAPITAL LETTER B WITH DOT ABOVE +1E04; C; 1E05; # LATIN CAPITAL LETTER B WITH DOT BELOW +1E06; C; 1E07; # LATIN CAPITAL LETTER B WITH LINE BELOW +1E08; C; 1E09; # LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE +1E0A; C; 1E0B; # LATIN CAPITAL LETTER D WITH DOT ABOVE +1E0C; C; 1E0D; # LATIN CAPITAL LETTER D WITH DOT BELOW +1E0E; C; 1E0F; # LATIN CAPITAL LETTER D WITH LINE BELOW +1E10; C; 1E11; # LATIN CAPITAL LETTER D WITH CEDILLA +1E12; C; 1E13; # LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW +1E14; C; 1E15; # LATIN CAPITAL LETTER E WITH MACRON AND GRAVE +1E16; C; 1E17; # LATIN CAPITAL LETTER E WITH MACRON AND ACUTE +1E18; C; 1E19; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW +1E1A; C; 1E1B; # LATIN CAPITAL LETTER E WITH TILDE BELOW +1E1C; C; 1E1D; # LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE +1E1E; C; 1E1F; # LATIN CAPITAL LETTER F WITH DOT ABOVE +1E20; C; 1E21; # LATIN CAPITAL LETTER G WITH MACRON +1E22; C; 1E23; # LATIN CAPITAL LETTER H WITH DOT ABOVE +1E24; C; 1E25; # LATIN CAPITAL LETTER H WITH DOT BELOW +1E26; C; 1E27; # LATIN CAPITAL LETTER H WITH DIAERESIS +1E28; C; 1E29; # LATIN CAPITAL LETTER H WITH CEDILLA +1E2A; C; 1E2B; # LATIN CAPITAL LETTER H WITH BREVE BELOW +1E2C; C; 1E2D; # LATIN CAPITAL LETTER I WITH TILDE BELOW +1E2E; C; 1E2F; # LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE +1E30; C; 1E31; # LATIN CAPITAL LETTER K WITH ACUTE +1E32; C; 1E33; # LATIN CAPITAL LETTER K WITH DOT BELOW +1E34; C; 1E35; # LATIN CAPITAL LETTER K WITH LINE BELOW +1E36; C; 1E37; # LATIN CAPITAL LETTER L WITH DOT BELOW +1E38; C; 1E39; # LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON +1E3A; C; 1E3B; # LATIN CAPITAL LETTER L WITH LINE BELOW +1E3C; C; 1E3D; # LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW +1E3E; C; 1E3F; # LATIN CAPITAL LETTER M WITH ACUTE +1E40; C; 1E41; # LATIN CAPITAL LETTER M WITH DOT ABOVE +1E42; C; 1E43; # LATIN CAPITAL LETTER M WITH DOT BELOW +1E44; C; 1E45; # LATIN CAPITAL LETTER N WITH DOT ABOVE +1E46; C; 1E47; # LATIN CAPITAL LETTER N WITH DOT BELOW +1E48; C; 1E49; # LATIN CAPITAL LETTER N WITH LINE BELOW +1E4A; C; 1E4B; # LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW +1E4C; C; 1E4D; # LATIN CAPITAL LETTER O WITH TILDE AND ACUTE +1E4E; C; 1E4F; # LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS +1E50; C; 1E51; # LATIN CAPITAL LETTER O WITH MACRON AND GRAVE +1E52; C; 1E53; # LATIN CAPITAL LETTER O WITH MACRON AND ACUTE +1E54; C; 1E55; # LATIN CAPITAL LETTER P WITH ACUTE +1E56; C; 1E57; # LATIN CAPITAL LETTER P WITH DOT ABOVE +1E58; C; 1E59; # LATIN CAPITAL LETTER R WITH DOT ABOVE +1E5A; C; 1E5B; # LATIN CAPITAL LETTER R WITH DOT BELOW +1E5C; C; 1E5D; # LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON +1E5E; C; 1E5F; # LATIN CAPITAL LETTER R WITH LINE BELOW +1E60; C; 1E61; # LATIN CAPITAL LETTER S WITH DOT ABOVE +1E62; C; 1E63; # LATIN CAPITAL LETTER S WITH DOT BELOW +1E64; C; 1E65; # LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE +1E66; C; 1E67; # LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE +1E68; C; 1E69; # LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE +1E6A; C; 1E6B; # LATIN CAPITAL LETTER T WITH DOT ABOVE +1E6C; C; 1E6D; # LATIN CAPITAL LETTER T WITH DOT BELOW +1E6E; C; 1E6F; # LATIN CAPITAL LETTER T WITH LINE BELOW +1E70; C; 1E71; # LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW +1E72; C; 1E73; # LATIN CAPITAL LETTER U WITH DIAERESIS BELOW +1E74; C; 1E75; # LATIN CAPITAL LETTER U WITH TILDE BELOW +1E76; C; 1E77; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW +1E78; C; 1E79; # LATIN CAPITAL LETTER U WITH TILDE AND ACUTE +1E7A; C; 1E7B; # LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS +1E7C; C; 1E7D; # LATIN CAPITAL LETTER V WITH TILDE +1E7E; C; 1E7F; # LATIN CAPITAL LETTER V WITH DOT BELOW +1E80; C; 1E81; # LATIN CAPITAL LETTER W WITH GRAVE +1E82; C; 1E83; # LATIN CAPITAL LETTER W WITH ACUTE +1E84; C; 1E85; # LATIN CAPITAL LETTER W WITH DIAERESIS +1E86; C; 1E87; # LATIN CAPITAL LETTER W WITH DOT ABOVE +1E88; C; 1E89; # LATIN CAPITAL LETTER W WITH DOT BELOW +1E8A; C; 1E8B; # LATIN CAPITAL LETTER X WITH DOT ABOVE +1E8C; C; 1E8D; # LATIN CAPITAL LETTER X WITH DIAERESIS +1E8E; C; 1E8F; # LATIN CAPITAL LETTER Y WITH DOT ABOVE +1E90; C; 1E91; # LATIN CAPITAL LETTER Z WITH CIRCUMFLEX +1E92; C; 1E93; # LATIN CAPITAL LETTER Z WITH DOT BELOW +1E94; C; 1E95; # LATIN CAPITAL LETTER Z WITH LINE BELOW +1E96; F; 0068 0331; # LATIN SMALL LETTER H WITH LINE BELOW +1E97; F; 0074 0308; # LATIN SMALL LETTER T WITH DIAERESIS +1E98; F; 0077 030A; # LATIN SMALL LETTER W WITH RING ABOVE +1E99; F; 0079 030A; # LATIN SMALL LETTER Y WITH RING ABOVE +1E9A; F; 0061 02BE; # LATIN SMALL LETTER A WITH RIGHT HALF RING +1E9B; C; 1E61; # LATIN SMALL LETTER LONG S WITH DOT ABOVE +1E9E; F; 0073 0073; # LATIN CAPITAL LETTER SHARP S +1E9E; S; 00DF; # LATIN CAPITAL LETTER SHARP S +1EA0; C; 1EA1; # LATIN CAPITAL LETTER A WITH DOT BELOW +1EA2; C; 1EA3; # LATIN CAPITAL LETTER A WITH HOOK ABOVE +1EA4; C; 1EA5; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE +1EA6; C; 1EA7; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE +1EA8; C; 1EA9; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE +1EAA; C; 1EAB; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE +1EAC; C; 1EAD; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW +1EAE; C; 1EAF; # LATIN CAPITAL LETTER A WITH BREVE AND ACUTE +1EB0; C; 1EB1; # LATIN CAPITAL LETTER A WITH BREVE AND GRAVE +1EB2; C; 1EB3; # LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE +1EB4; C; 1EB5; # LATIN CAPITAL LETTER A WITH BREVE AND TILDE +1EB6; C; 1EB7; # LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW +1EB8; C; 1EB9; # LATIN CAPITAL LETTER E WITH DOT BELOW +1EBA; C; 1EBB; # LATIN CAPITAL LETTER E WITH HOOK ABOVE +1EBC; C; 1EBD; # LATIN CAPITAL LETTER E WITH TILDE +1EBE; C; 1EBF; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE +1EC0; C; 1EC1; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE +1EC2; C; 1EC3; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE +1EC4; C; 1EC5; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE +1EC6; C; 1EC7; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW +1EC8; C; 1EC9; # LATIN CAPITAL LETTER I WITH HOOK ABOVE +1ECA; C; 1ECB; # LATIN CAPITAL LETTER I WITH DOT BELOW +1ECC; C; 1ECD; # LATIN CAPITAL LETTER O WITH DOT BELOW +1ECE; C; 1ECF; # LATIN CAPITAL LETTER O WITH HOOK ABOVE +1ED0; C; 1ED1; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE +1ED2; C; 1ED3; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE +1ED4; C; 1ED5; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE +1ED6; C; 1ED7; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE +1ED8; C; 1ED9; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW +1EDA; C; 1EDB; # LATIN CAPITAL LETTER O WITH HORN AND ACUTE +1EDC; C; 1EDD; # LATIN CAPITAL LETTER O WITH HORN AND GRAVE +1EDE; C; 1EDF; # LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE +1EE0; C; 1EE1; # LATIN CAPITAL LETTER O WITH HORN AND TILDE +1EE2; C; 1EE3; # LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW +1EE4; C; 1EE5; # LATIN CAPITAL LETTER U WITH DOT BELOW +1EE6; C; 1EE7; # LATIN CAPITAL LETTER U WITH HOOK ABOVE +1EE8; C; 1EE9; # LATIN CAPITAL LETTER U WITH HORN AND ACUTE +1EEA; C; 1EEB; # LATIN CAPITAL LETTER U WITH HORN AND GRAVE +1EEC; C; 1EED; # LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE +1EEE; C; 1EEF; # LATIN CAPITAL LETTER U WITH HORN AND TILDE +1EF0; C; 1EF1; # LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW +1EF2; C; 1EF3; # LATIN CAPITAL LETTER Y WITH GRAVE +1EF4; C; 1EF5; # LATIN CAPITAL LETTER Y WITH DOT BELOW +1EF6; C; 1EF7; # LATIN CAPITAL LETTER Y WITH HOOK ABOVE +1EF8; C; 1EF9; # LATIN CAPITAL LETTER Y WITH TILDE +1EFA; C; 1EFB; # LATIN CAPITAL LETTER MIDDLE-WELSH LL +1EFC; C; 1EFD; # LATIN CAPITAL LETTER MIDDLE-WELSH V +1EFE; C; 1EFF; # LATIN CAPITAL LETTER Y WITH LOOP +1F08; C; 1F00; # GREEK CAPITAL LETTER ALPHA WITH PSILI +1F09; C; 1F01; # GREEK CAPITAL LETTER ALPHA WITH DASIA +1F0A; C; 1F02; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA +1F0B; C; 1F03; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA +1F0C; C; 1F04; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA +1F0D; C; 1F05; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA +1F0E; C; 1F06; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI +1F0F; C; 1F07; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI +1F18; C; 1F10; # GREEK CAPITAL LETTER EPSILON WITH PSILI +1F19; C; 1F11; # GREEK CAPITAL LETTER EPSILON WITH DASIA +1F1A; C; 1F12; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA +1F1B; C; 1F13; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA +1F1C; C; 1F14; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA +1F1D; C; 1F15; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F28; C; 1F20; # GREEK CAPITAL LETTER ETA WITH PSILI +1F29; C; 1F21; # GREEK CAPITAL LETTER ETA WITH DASIA +1F2A; C; 1F22; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA +1F2B; C; 1F23; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA +1F2C; C; 1F24; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA +1F2D; C; 1F25; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA +1F2E; C; 1F26; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI +1F2F; C; 1F27; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI +1F38; C; 1F30; # GREEK CAPITAL LETTER IOTA WITH PSILI +1F39; C; 1F31; # GREEK CAPITAL LETTER IOTA WITH DASIA +1F3A; C; 1F32; # GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA +1F3B; C; 1F33; # GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA +1F3C; C; 1F34; # GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA +1F3D; C; 1F35; # GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA +1F3E; C; 1F36; # GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI +1F3F; C; 1F37; # GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI +1F48; C; 1F40; # GREEK CAPITAL LETTER OMICRON WITH PSILI +1F49; C; 1F41; # GREEK CAPITAL LETTER OMICRON WITH DASIA +1F4A; C; 1F42; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA +1F4B; C; 1F43; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA +1F4C; C; 1F44; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA +1F4D; C; 1F45; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F50; F; 03C5 0313; # GREEK SMALL LETTER UPSILON WITH PSILI +1F52; F; 03C5 0313 0300; # GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA +1F54; F; 03C5 0313 0301; # GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA +1F56; F; 03C5 0313 0342; # GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI +1F59; C; 1F51; # GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B; C; 1F53; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D; C; 1F55; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F; C; 1F57; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F68; C; 1F60; # GREEK CAPITAL LETTER OMEGA WITH PSILI +1F69; C; 1F61; # GREEK CAPITAL LETTER OMEGA WITH DASIA +1F6A; C; 1F62; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA +1F6B; C; 1F63; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA +1F6C; C; 1F64; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA +1F6D; C; 1F65; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA +1F6E; C; 1F66; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI +1F6F; C; 1F67; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI +1F80; F; 1F00 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI +1F81; F; 1F01 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI +1F82; F; 1F02 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI +1F83; F; 1F03 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI +1F84; F; 1F04 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI +1F85; F; 1F05 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI +1F86; F; 1F06 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI +1F87; F; 1F07 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1F88; F; 1F00 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI +1F88; S; 1F80; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI +1F89; F; 1F01 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI +1F89; S; 1F81; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI +1F8A; F; 1F02 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F8A; S; 1F82; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F8B; F; 1F03 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F8B; S; 1F83; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F8C; F; 1F04 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F8C; S; 1F84; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F8D; F; 1F05 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F8D; S; 1F85; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F8E; F; 1F06 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F8E; S; 1F86; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F8F; F; 1F07 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1F8F; S; 1F87; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1F90; F; 1F20 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI +1F91; F; 1F21 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI +1F92; F; 1F22 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI +1F93; F; 1F23 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI +1F94; F; 1F24 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI +1F95; F; 1F25 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI +1F96; F; 1F26 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI +1F97; F; 1F27 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1F98; F; 1F20 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI +1F98; S; 1F90; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI +1F99; F; 1F21 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI +1F99; S; 1F91; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI +1F9A; F; 1F22 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F9A; S; 1F92; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F9B; F; 1F23 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F9B; S; 1F93; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F9C; F; 1F24 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F9C; S; 1F94; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F9D; F; 1F25 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F9D; S; 1F95; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F9E; F; 1F26 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F9E; S; 1F96; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F9F; F; 1F27 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1F9F; S; 1F97; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FA0; F; 1F60 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI +1FA1; F; 1F61 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI +1FA2; F; 1F62 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI +1FA3; F; 1F63 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI +1FA4; F; 1F64 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI +1FA5; F; 1F65 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI +1FA6; F; 1F66 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI +1FA7; F; 1F67 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1FA8; F; 1F60 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI +1FA8; S; 1FA0; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI +1FA9; F; 1F61 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI +1FA9; S; 1FA1; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI +1FAA; F; 1F62 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1FAA; S; 1FA2; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1FAB; F; 1F63 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1FAB; S; 1FA3; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1FAC; F; 1F64 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1FAC; S; 1FA4; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1FAD; F; 1F65 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1FAD; S; 1FA5; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1FAE; F; 1F66 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1FAE; S; 1FA6; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1FAF; F; 1F67 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FAF; S; 1FA7; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FB2; F; 1F70 03B9; # GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI +1FB3; F; 03B1 03B9; # GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI +1FB4; F; 03AC 03B9; # GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6; F; 03B1 0342; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI +1FB7; F; 03B1 0342 03B9; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI +1FB8; C; 1FB0; # GREEK CAPITAL LETTER ALPHA WITH VRACHY +1FB9; C; 1FB1; # GREEK CAPITAL LETTER ALPHA WITH MACRON +1FBA; C; 1F70; # GREEK CAPITAL LETTER ALPHA WITH VARIA +1FBB; C; 1F71; # GREEK CAPITAL LETTER ALPHA WITH OXIA +1FBC; F; 03B1 03B9; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBC; S; 1FB3; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBE; C; 03B9; # GREEK PROSGEGRAMMENI +1FC2; F; 1F74 03B9; # GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI +1FC3; F; 03B7 03B9; # GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI +1FC4; F; 03AE 03B9; # GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6; F; 03B7 0342; # GREEK SMALL LETTER ETA WITH PERISPOMENI +1FC7; F; 03B7 0342 03B9; # GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI +1FC8; C; 1F72; # GREEK CAPITAL LETTER EPSILON WITH VARIA +1FC9; C; 1F73; # GREEK CAPITAL LETTER EPSILON WITH OXIA +1FCA; C; 1F74; # GREEK CAPITAL LETTER ETA WITH VARIA +1FCB; C; 1F75; # GREEK CAPITAL LETTER ETA WITH OXIA +1FCC; F; 03B7 03B9; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FCC; S; 1FC3; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FD2; F; 03B9 0308 0300; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA +1FD3; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6; F; 03B9 0342; # GREEK SMALL LETTER IOTA WITH PERISPOMENI +1FD7; F; 03B9 0308 0342; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI +1FD8; C; 1FD0; # GREEK CAPITAL LETTER IOTA WITH VRACHY +1FD9; C; 1FD1; # GREEK CAPITAL LETTER IOTA WITH MACRON +1FDA; C; 1F76; # GREEK CAPITAL LETTER IOTA WITH VARIA +1FDB; C; 1F77; # GREEK CAPITAL LETTER IOTA WITH OXIA +1FE2; F; 03C5 0308 0300; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA +1FE3; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA +1FE4; F; 03C1 0313; # GREEK SMALL LETTER RHO WITH PSILI +1FE6; F; 03C5 0342; # GREEK SMALL LETTER UPSILON WITH PERISPOMENI +1FE7; F; 03C5 0308 0342; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI +1FE8; C; 1FE0; # GREEK CAPITAL LETTER UPSILON WITH VRACHY +1FE9; C; 1FE1; # GREEK CAPITAL LETTER UPSILON WITH MACRON +1FEA; C; 1F7A; # GREEK CAPITAL LETTER UPSILON WITH VARIA +1FEB; C; 1F7B; # GREEK CAPITAL LETTER UPSILON WITH OXIA +1FEC; C; 1FE5; # GREEK CAPITAL LETTER RHO WITH DASIA +1FF2; F; 1F7C 03B9; # GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI +1FF3; F; 03C9 03B9; # GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI +1FF4; F; 03CE 03B9; # GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6; F; 03C9 0342; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI +1FF7; F; 03C9 0342 03B9; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI +1FF8; C; 1F78; # GREEK CAPITAL LETTER OMICRON WITH VARIA +1FF9; C; 1F79; # GREEK CAPITAL LETTER OMICRON WITH OXIA +1FFA; C; 1F7C; # GREEK CAPITAL LETTER OMEGA WITH VARIA +1FFB; C; 1F7D; # GREEK CAPITAL LETTER OMEGA WITH OXIA +1FFC; F; 03C9 03B9; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +1FFC; S; 1FF3; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +2126; C; 03C9; # OHM SIGN +212A; C; 006B; # KELVIN SIGN +212B; C; 00E5; # ANGSTROM SIGN +2132; C; 214E; # TURNED CAPITAL F +2160; C; 2170; # ROMAN NUMERAL ONE +2161; C; 2171; # ROMAN NUMERAL TWO +2162; C; 2172; # ROMAN NUMERAL THREE +2163; C; 2173; # ROMAN NUMERAL FOUR +2164; C; 2174; # ROMAN NUMERAL FIVE +2165; C; 2175; # ROMAN NUMERAL SIX +2166; C; 2176; # ROMAN NUMERAL SEVEN +2167; C; 2177; # ROMAN NUMERAL EIGHT +2168; C; 2178; # ROMAN NUMERAL NINE +2169; C; 2179; # ROMAN NUMERAL TEN +216A; C; 217A; # ROMAN NUMERAL ELEVEN +216B; C; 217B; # ROMAN NUMERAL TWELVE +216C; C; 217C; # ROMAN NUMERAL FIFTY +216D; C; 217D; # ROMAN NUMERAL ONE HUNDRED +216E; C; 217E; # ROMAN NUMERAL FIVE HUNDRED +216F; C; 217F; # ROMAN NUMERAL ONE THOUSAND +2183; C; 2184; # ROMAN NUMERAL REVERSED ONE HUNDRED +24B6; C; 24D0; # CIRCLED LATIN CAPITAL LETTER A +24B7; C; 24D1; # CIRCLED LATIN CAPITAL LETTER B +24B8; C; 24D2; # CIRCLED LATIN CAPITAL LETTER C +24B9; C; 24D3; # CIRCLED LATIN CAPITAL LETTER D +24BA; C; 24D4; # CIRCLED LATIN CAPITAL LETTER E +24BB; C; 24D5; # CIRCLED LATIN CAPITAL LETTER F +24BC; C; 24D6; # CIRCLED LATIN CAPITAL LETTER G +24BD; C; 24D7; # CIRCLED LATIN CAPITAL LETTER H +24BE; C; 24D8; # CIRCLED LATIN CAPITAL LETTER I +24BF; C; 24D9; # CIRCLED LATIN CAPITAL LETTER J +24C0; C; 24DA; # CIRCLED LATIN CAPITAL LETTER K +24C1; C; 24DB; # CIRCLED LATIN CAPITAL LETTER L +24C2; C; 24DC; # CIRCLED LATIN CAPITAL LETTER M +24C3; C; 24DD; # CIRCLED LATIN CAPITAL LETTER N +24C4; C; 24DE; # CIRCLED LATIN CAPITAL LETTER O +24C5; C; 24DF; # CIRCLED LATIN CAPITAL LETTER P +24C6; C; 24E0; # CIRCLED LATIN CAPITAL LETTER Q +24C7; C; 24E1; # CIRCLED LATIN CAPITAL LETTER R +24C8; C; 24E2; # CIRCLED LATIN CAPITAL LETTER S +24C9; C; 24E3; # CIRCLED LATIN CAPITAL LETTER T +24CA; C; 24E4; # CIRCLED LATIN CAPITAL LETTER U +24CB; C; 24E5; # CIRCLED LATIN CAPITAL LETTER V +24CC; C; 24E6; # CIRCLED LATIN CAPITAL LETTER W +24CD; C; 24E7; # CIRCLED LATIN CAPITAL LETTER X +24CE; C; 24E8; # CIRCLED LATIN CAPITAL LETTER Y +24CF; C; 24E9; # CIRCLED LATIN CAPITAL LETTER Z +2C00; C; 2C30; # GLAGOLITIC CAPITAL LETTER AZU +2C01; C; 2C31; # GLAGOLITIC CAPITAL LETTER BUKY +2C02; C; 2C32; # GLAGOLITIC CAPITAL LETTER VEDE +2C03; C; 2C33; # GLAGOLITIC CAPITAL LETTER GLAGOLI +2C04; C; 2C34; # GLAGOLITIC CAPITAL LETTER DOBRO +2C05; C; 2C35; # GLAGOLITIC CAPITAL LETTER YESTU +2C06; C; 2C36; # GLAGOLITIC CAPITAL LETTER ZHIVETE +2C07; C; 2C37; # GLAGOLITIC CAPITAL LETTER DZELO +2C08; C; 2C38; # GLAGOLITIC CAPITAL LETTER ZEMLJA +2C09; C; 2C39; # GLAGOLITIC CAPITAL LETTER IZHE +2C0A; C; 2C3A; # GLAGOLITIC CAPITAL LETTER INITIAL IZHE +2C0B; C; 2C3B; # GLAGOLITIC CAPITAL LETTER I +2C0C; C; 2C3C; # GLAGOLITIC CAPITAL LETTER DJERVI +2C0D; C; 2C3D; # GLAGOLITIC CAPITAL LETTER KAKO +2C0E; C; 2C3E; # GLAGOLITIC CAPITAL LETTER LJUDIJE +2C0F; C; 2C3F; # GLAGOLITIC CAPITAL LETTER MYSLITE +2C10; C; 2C40; # GLAGOLITIC CAPITAL LETTER NASHI +2C11; C; 2C41; # GLAGOLITIC CAPITAL LETTER ONU +2C12; C; 2C42; # GLAGOLITIC CAPITAL LETTER POKOJI +2C13; C; 2C43; # GLAGOLITIC CAPITAL LETTER RITSI +2C14; C; 2C44; # GLAGOLITIC CAPITAL LETTER SLOVO +2C15; C; 2C45; # GLAGOLITIC CAPITAL LETTER TVRIDO +2C16; C; 2C46; # GLAGOLITIC CAPITAL LETTER UKU +2C17; C; 2C47; # GLAGOLITIC CAPITAL LETTER FRITU +2C18; C; 2C48; # GLAGOLITIC CAPITAL LETTER HERU +2C19; C; 2C49; # GLAGOLITIC CAPITAL LETTER OTU +2C1A; C; 2C4A; # GLAGOLITIC CAPITAL LETTER PE +2C1B; C; 2C4B; # GLAGOLITIC CAPITAL LETTER SHTA +2C1C; C; 2C4C; # GLAGOLITIC CAPITAL LETTER TSI +2C1D; C; 2C4D; # GLAGOLITIC CAPITAL LETTER CHRIVI +2C1E; C; 2C4E; # GLAGOLITIC CAPITAL LETTER SHA +2C1F; C; 2C4F; # GLAGOLITIC CAPITAL LETTER YERU +2C20; C; 2C50; # GLAGOLITIC CAPITAL LETTER YERI +2C21; C; 2C51; # GLAGOLITIC CAPITAL LETTER YATI +2C22; C; 2C52; # GLAGOLITIC CAPITAL LETTER SPIDERY HA +2C23; C; 2C53; # GLAGOLITIC CAPITAL LETTER YU +2C24; C; 2C54; # GLAGOLITIC CAPITAL LETTER SMALL YUS +2C25; C; 2C55; # GLAGOLITIC CAPITAL LETTER SMALL YUS WITH TAIL +2C26; C; 2C56; # GLAGOLITIC CAPITAL LETTER YO +2C27; C; 2C57; # GLAGOLITIC CAPITAL LETTER IOTATED SMALL YUS +2C28; C; 2C58; # GLAGOLITIC CAPITAL LETTER BIG YUS +2C29; C; 2C59; # GLAGOLITIC CAPITAL LETTER IOTATED BIG YUS +2C2A; C; 2C5A; # GLAGOLITIC CAPITAL LETTER FITA +2C2B; C; 2C5B; # GLAGOLITIC CAPITAL LETTER IZHITSA +2C2C; C; 2C5C; # GLAGOLITIC CAPITAL LETTER SHTAPIC +2C2D; C; 2C5D; # GLAGOLITIC CAPITAL LETTER TROKUTASTI A +2C2E; C; 2C5E; # GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE +2C60; C; 2C61; # LATIN CAPITAL LETTER L WITH DOUBLE BAR +2C62; C; 026B; # LATIN CAPITAL LETTER L WITH MIDDLE TILDE +2C63; C; 1D7D; # LATIN CAPITAL LETTER P WITH STROKE +2C64; C; 027D; # LATIN CAPITAL LETTER R WITH TAIL +2C67; C; 2C68; # LATIN CAPITAL LETTER H WITH DESCENDER +2C69; C; 2C6A; # LATIN CAPITAL LETTER K WITH DESCENDER +2C6B; C; 2C6C; # LATIN CAPITAL LETTER Z WITH DESCENDER +2C6D; C; 0251; # LATIN CAPITAL LETTER ALPHA +2C6E; C; 0271; # LATIN CAPITAL LETTER M WITH HOOK +2C6F; C; 0250; # LATIN CAPITAL LETTER TURNED A +2C70; C; 0252; # LATIN CAPITAL LETTER TURNED ALPHA +2C72; C; 2C73; # LATIN CAPITAL LETTER W WITH HOOK +2C75; C; 2C76; # LATIN CAPITAL LETTER HALF H +2C7E; C; 023F; # LATIN CAPITAL LETTER S WITH SWASH TAIL +2C7F; C; 0240; # LATIN CAPITAL LETTER Z WITH SWASH TAIL +2C80; C; 2C81; # COPTIC CAPITAL LETTER ALFA +2C82; C; 2C83; # COPTIC CAPITAL LETTER VIDA +2C84; C; 2C85; # COPTIC CAPITAL LETTER GAMMA +2C86; C; 2C87; # COPTIC CAPITAL LETTER DALDA +2C88; C; 2C89; # COPTIC CAPITAL LETTER EIE +2C8A; C; 2C8B; # COPTIC CAPITAL LETTER SOU +2C8C; C; 2C8D; # COPTIC CAPITAL LETTER ZATA +2C8E; C; 2C8F; # COPTIC CAPITAL LETTER HATE +2C90; C; 2C91; # COPTIC CAPITAL LETTER THETHE +2C92; C; 2C93; # COPTIC CAPITAL LETTER IAUDA +2C94; C; 2C95; # COPTIC CAPITAL LETTER KAPA +2C96; C; 2C97; # COPTIC CAPITAL LETTER LAULA +2C98; C; 2C99; # COPTIC CAPITAL LETTER MI +2C9A; C; 2C9B; # COPTIC CAPITAL LETTER NI +2C9C; C; 2C9D; # COPTIC CAPITAL LETTER KSI +2C9E; C; 2C9F; # COPTIC CAPITAL LETTER O +2CA0; C; 2CA1; # COPTIC CAPITAL LETTER PI +2CA2; C; 2CA3; # COPTIC CAPITAL LETTER RO +2CA4; C; 2CA5; # COPTIC CAPITAL LETTER SIMA +2CA6; C; 2CA7; # COPTIC CAPITAL LETTER TAU +2CA8; C; 2CA9; # COPTIC CAPITAL LETTER UA +2CAA; C; 2CAB; # COPTIC CAPITAL LETTER FI +2CAC; C; 2CAD; # COPTIC CAPITAL LETTER KHI +2CAE; C; 2CAF; # COPTIC CAPITAL LETTER PSI +2CB0; C; 2CB1; # COPTIC CAPITAL LETTER OOU +2CB2; C; 2CB3; # COPTIC CAPITAL LETTER DIALECT-P ALEF +2CB4; C; 2CB5; # COPTIC CAPITAL LETTER OLD COPTIC AIN +2CB6; C; 2CB7; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC EIE +2CB8; C; 2CB9; # COPTIC CAPITAL LETTER DIALECT-P KAPA +2CBA; C; 2CBB; # COPTIC CAPITAL LETTER DIALECT-P NI +2CBC; C; 2CBD; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC NI +2CBE; C; 2CBF; # COPTIC CAPITAL LETTER OLD COPTIC OOU +2CC0; C; 2CC1; # COPTIC CAPITAL LETTER SAMPI +2CC2; C; 2CC3; # COPTIC CAPITAL LETTER CROSSED SHEI +2CC4; C; 2CC5; # COPTIC CAPITAL LETTER OLD COPTIC SHEI +2CC6; C; 2CC7; # COPTIC CAPITAL LETTER OLD COPTIC ESH +2CC8; C; 2CC9; # COPTIC CAPITAL LETTER AKHMIMIC KHEI +2CCA; C; 2CCB; # COPTIC CAPITAL LETTER DIALECT-P HORI +2CCC; C; 2CCD; # COPTIC CAPITAL LETTER OLD COPTIC HORI +2CCE; C; 2CCF; # COPTIC CAPITAL LETTER OLD COPTIC HA +2CD0; C; 2CD1; # COPTIC CAPITAL LETTER L-SHAPED HA +2CD2; C; 2CD3; # COPTIC CAPITAL LETTER OLD COPTIC HEI +2CD4; C; 2CD5; # COPTIC CAPITAL LETTER OLD COPTIC HAT +2CD6; C; 2CD7; # COPTIC CAPITAL LETTER OLD COPTIC GANGIA +2CD8; C; 2CD9; # COPTIC CAPITAL LETTER OLD COPTIC DJA +2CDA; C; 2CDB; # COPTIC CAPITAL LETTER OLD COPTIC SHIMA +2CDC; C; 2CDD; # COPTIC CAPITAL LETTER OLD NUBIAN SHIMA +2CDE; C; 2CDF; # COPTIC CAPITAL LETTER OLD NUBIAN NGI +2CE0; C; 2CE1; # COPTIC CAPITAL LETTER OLD NUBIAN NYI +2CE2; C; 2CE3; # COPTIC CAPITAL LETTER OLD NUBIAN WAU +2CEB; C; 2CEC; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI +2CED; C; 2CEE; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC GANGIA +2CF2; C; 2CF3; # COPTIC CAPITAL LETTER BOHAIRIC KHEI +A640; C; A641; # CYRILLIC CAPITAL LETTER ZEMLYA +A642; C; A643; # CYRILLIC CAPITAL LETTER DZELO +A644; C; A645; # CYRILLIC CAPITAL LETTER REVERSED DZE +A646; C; A647; # CYRILLIC CAPITAL LETTER IOTA +A648; C; A649; # CYRILLIC CAPITAL LETTER DJERV +A64A; C; A64B; # CYRILLIC CAPITAL LETTER MONOGRAPH UK +A64C; C; A64D; # CYRILLIC CAPITAL LETTER BROAD OMEGA +A64E; C; A64F; # CYRILLIC CAPITAL LETTER NEUTRAL YER +A650; C; A651; # CYRILLIC CAPITAL LETTER YERU WITH BACK YER +A652; C; A653; # CYRILLIC CAPITAL LETTER IOTIFIED YAT +A654; C; A655; # CYRILLIC CAPITAL LETTER REVERSED YU +A656; C; A657; # CYRILLIC CAPITAL LETTER IOTIFIED A +A658; C; A659; # CYRILLIC CAPITAL LETTER CLOSED LITTLE YUS +A65A; C; A65B; # CYRILLIC CAPITAL LETTER BLENDED YUS +A65C; C; A65D; # CYRILLIC CAPITAL LETTER IOTIFIED CLOSED LITTLE YUS +A65E; C; A65F; # CYRILLIC CAPITAL LETTER YN +A660; C; A661; # CYRILLIC CAPITAL LETTER REVERSED TSE +A662; C; A663; # CYRILLIC CAPITAL LETTER SOFT DE +A664; C; A665; # CYRILLIC CAPITAL LETTER SOFT EL +A666; C; A667; # CYRILLIC CAPITAL LETTER SOFT EM +A668; C; A669; # CYRILLIC CAPITAL LETTER MONOCULAR O +A66A; C; A66B; # CYRILLIC CAPITAL LETTER BINOCULAR O +A66C; C; A66D; # CYRILLIC CAPITAL LETTER DOUBLE MONOCULAR O +A680; C; A681; # CYRILLIC CAPITAL LETTER DWE +A682; C; A683; # CYRILLIC CAPITAL LETTER DZWE +A684; C; A685; # CYRILLIC CAPITAL LETTER ZHWE +A686; C; A687; # CYRILLIC CAPITAL LETTER CCHE +A688; C; A689; # CYRILLIC CAPITAL LETTER DZZE +A68A; C; A68B; # CYRILLIC CAPITAL LETTER TE WITH MIDDLE HOOK +A68C; C; A68D; # CYRILLIC CAPITAL LETTER TWE +A68E; C; A68F; # CYRILLIC CAPITAL LETTER TSWE +A690; C; A691; # CYRILLIC CAPITAL LETTER TSSE +A692; C; A693; # CYRILLIC CAPITAL LETTER TCHE +A694; C; A695; # CYRILLIC CAPITAL LETTER HWE +A696; C; A697; # CYRILLIC CAPITAL LETTER SHWE +A698; C; A699; # CYRILLIC CAPITAL LETTER DOUBLE O +A69A; C; A69B; # CYRILLIC CAPITAL LETTER CROSSED O +A722; C; A723; # LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF +A724; C; A725; # LATIN CAPITAL LETTER EGYPTOLOGICAL AIN +A726; C; A727; # LATIN CAPITAL LETTER HENG +A728; C; A729; # LATIN CAPITAL LETTER TZ +A72A; C; A72B; # LATIN CAPITAL LETTER TRESILLO +A72C; C; A72D; # LATIN CAPITAL LETTER CUATRILLO +A72E; C; A72F; # LATIN CAPITAL LETTER CUATRILLO WITH COMMA +A732; C; A733; # LATIN CAPITAL LETTER AA +A734; C; A735; # LATIN CAPITAL LETTER AO +A736; C; A737; # LATIN CAPITAL LETTER AU +A738; C; A739; # LATIN CAPITAL LETTER AV +A73A; C; A73B; # LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR +A73C; C; A73D; # LATIN CAPITAL LETTER AY +A73E; C; A73F; # LATIN CAPITAL LETTER REVERSED C WITH DOT +A740; C; A741; # LATIN CAPITAL LETTER K WITH STROKE +A742; C; A743; # LATIN CAPITAL LETTER K WITH DIAGONAL STROKE +A744; C; A745; # LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE +A746; C; A747; # LATIN CAPITAL LETTER BROKEN L +A748; C; A749; # LATIN CAPITAL LETTER L WITH HIGH STROKE +A74A; C; A74B; # LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY +A74C; C; A74D; # LATIN CAPITAL LETTER O WITH LOOP +A74E; C; A74F; # LATIN CAPITAL LETTER OO +A750; C; A751; # LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER +A752; C; A753; # LATIN CAPITAL LETTER P WITH FLOURISH +A754; C; A755; # LATIN CAPITAL LETTER P WITH SQUIRREL TAIL +A756; C; A757; # LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER +A758; C; A759; # LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE +A75A; C; A75B; # LATIN CAPITAL LETTER R ROTUNDA +A75C; C; A75D; # LATIN CAPITAL LETTER RUM ROTUNDA +A75E; C; A75F; # LATIN CAPITAL LETTER V WITH DIAGONAL STROKE +A760; C; A761; # LATIN CAPITAL LETTER VY +A762; C; A763; # LATIN CAPITAL LETTER VISIGOTHIC Z +A764; C; A765; # LATIN CAPITAL LETTER THORN WITH STROKE +A766; C; A767; # LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER +A768; C; A769; # LATIN CAPITAL LETTER VEND +A76A; C; A76B; # LATIN CAPITAL LETTER ET +A76C; C; A76D; # LATIN CAPITAL LETTER IS +A76E; C; A76F; # LATIN CAPITAL LETTER CON +A779; C; A77A; # LATIN CAPITAL LETTER INSULAR D +A77B; C; A77C; # LATIN CAPITAL LETTER INSULAR F +A77D; C; 1D79; # LATIN CAPITAL LETTER INSULAR G +A77E; C; A77F; # LATIN CAPITAL LETTER TURNED INSULAR G +A780; C; A781; # LATIN CAPITAL LETTER TURNED L +A782; C; A783; # LATIN CAPITAL LETTER INSULAR R +A784; C; A785; # LATIN CAPITAL LETTER INSULAR S +A786; C; A787; # LATIN CAPITAL LETTER INSULAR T +A78B; C; A78C; # LATIN CAPITAL LETTER SALTILLO +A78D; C; 0265; # LATIN CAPITAL LETTER TURNED H +A790; C; A791; # LATIN CAPITAL LETTER N WITH DESCENDER +A792; C; A793; # LATIN CAPITAL LETTER C WITH BAR +A796; C; A797; # LATIN CAPITAL LETTER B WITH FLOURISH +A798; C; A799; # LATIN CAPITAL LETTER F WITH STROKE +A79A; C; A79B; # LATIN CAPITAL LETTER VOLAPUK AE +A79C; C; A79D; # LATIN CAPITAL LETTER VOLAPUK OE +A79E; C; A79F; # LATIN CAPITAL LETTER VOLAPUK UE +A7A0; C; A7A1; # LATIN CAPITAL LETTER G WITH OBLIQUE STROKE +A7A2; C; A7A3; # LATIN CAPITAL LETTER K WITH OBLIQUE STROKE +A7A4; C; A7A5; # LATIN CAPITAL LETTER N WITH OBLIQUE STROKE +A7A6; C; A7A7; # LATIN CAPITAL LETTER R WITH OBLIQUE STROKE +A7A8; C; A7A9; # LATIN CAPITAL LETTER S WITH OBLIQUE STROKE +A7AA; C; 0266; # LATIN CAPITAL LETTER H WITH HOOK +A7AB; C; 025C; # LATIN CAPITAL LETTER REVERSED OPEN E +A7AC; C; 0261; # LATIN CAPITAL LETTER SCRIPT G +A7AD; C; 026C; # LATIN CAPITAL LETTER L WITH BELT +A7AE; C; 026A; # LATIN CAPITAL LETTER SMALL CAPITAL I +A7B0; C; 029E; # LATIN CAPITAL LETTER TURNED K +A7B1; C; 0287; # LATIN CAPITAL LETTER TURNED T +A7B2; C; 029D; # LATIN CAPITAL LETTER J WITH CROSSED-TAIL +A7B3; C; AB53; # LATIN CAPITAL LETTER CHI +A7B4; C; A7B5; # LATIN CAPITAL LETTER BETA +A7B6; C; A7B7; # LATIN CAPITAL LETTER OMEGA +A7B8; C; A7B9; # LATIN CAPITAL LETTER U WITH STROKE +A7BA; C; A7BB; # LATIN CAPITAL LETTER GLOTTAL A +A7BC; C; A7BD; # LATIN CAPITAL LETTER GLOTTAL I +A7BE; C; A7BF; # LATIN CAPITAL LETTER GLOTTAL U +A7C2; C; A7C3; # LATIN CAPITAL LETTER ANGLICANA W +A7C4; C; A794; # LATIN CAPITAL LETTER C WITH PALATAL HOOK +A7C5; C; 0282; # LATIN CAPITAL LETTER S WITH HOOK +A7C6; C; 1D8E; # LATIN CAPITAL LETTER Z WITH PALATAL HOOK +A7C7; C; A7C8; # LATIN CAPITAL LETTER D WITH SHORT STROKE OVERLAY +A7C9; C; A7CA; # LATIN CAPITAL LETTER S WITH SHORT STROKE OVERLAY +A7F5; C; A7F6; # LATIN CAPITAL LETTER REVERSED HALF H +AB70; C; 13A0; # CHEROKEE SMALL LETTER A +AB71; C; 13A1; # CHEROKEE SMALL LETTER E +AB72; C; 13A2; # CHEROKEE SMALL LETTER I +AB73; C; 13A3; # CHEROKEE SMALL LETTER O +AB74; C; 13A4; # CHEROKEE SMALL LETTER U +AB75; C; 13A5; # CHEROKEE SMALL LETTER V +AB76; C; 13A6; # CHEROKEE SMALL LETTER GA +AB77; C; 13A7; # CHEROKEE SMALL LETTER KA +AB78; C; 13A8; # CHEROKEE SMALL LETTER GE +AB79; C; 13A9; # CHEROKEE SMALL LETTER GI +AB7A; C; 13AA; # CHEROKEE SMALL LETTER GO +AB7B; C; 13AB; # CHEROKEE SMALL LETTER GU +AB7C; C; 13AC; # CHEROKEE SMALL LETTER GV +AB7D; C; 13AD; # CHEROKEE SMALL LETTER HA +AB7E; C; 13AE; # CHEROKEE SMALL LETTER HE +AB7F; C; 13AF; # CHEROKEE SMALL LETTER HI +AB80; C; 13B0; # CHEROKEE SMALL LETTER HO +AB81; C; 13B1; # CHEROKEE SMALL LETTER HU +AB82; C; 13B2; # CHEROKEE SMALL LETTER HV +AB83; C; 13B3; # CHEROKEE SMALL LETTER LA +AB84; C; 13B4; # CHEROKEE SMALL LETTER LE +AB85; C; 13B5; # CHEROKEE SMALL LETTER LI +AB86; C; 13B6; # CHEROKEE SMALL LETTER LO +AB87; C; 13B7; # CHEROKEE SMALL LETTER LU +AB88; C; 13B8; # CHEROKEE SMALL LETTER LV +AB89; C; 13B9; # CHEROKEE SMALL LETTER MA +AB8A; C; 13BA; # CHEROKEE SMALL LETTER ME +AB8B; C; 13BB; # CHEROKEE SMALL LETTER MI +AB8C; C; 13BC; # CHEROKEE SMALL LETTER MO +AB8D; C; 13BD; # CHEROKEE SMALL LETTER MU +AB8E; C; 13BE; # CHEROKEE SMALL LETTER NA +AB8F; C; 13BF; # CHEROKEE SMALL LETTER HNA +AB90; C; 13C0; # CHEROKEE SMALL LETTER NAH +AB91; C; 13C1; # CHEROKEE SMALL LETTER NE +AB92; C; 13C2; # CHEROKEE SMALL LETTER NI +AB93; C; 13C3; # CHEROKEE SMALL LETTER NO +AB94; C; 13C4; # CHEROKEE SMALL LETTER NU +AB95; C; 13C5; # CHEROKEE SMALL LETTER NV +AB96; C; 13C6; # CHEROKEE SMALL LETTER QUA +AB97; C; 13C7; # CHEROKEE SMALL LETTER QUE +AB98; C; 13C8; # CHEROKEE SMALL LETTER QUI +AB99; C; 13C9; # CHEROKEE SMALL LETTER QUO +AB9A; C; 13CA; # CHEROKEE SMALL LETTER QUU +AB9B; C; 13CB; # CHEROKEE SMALL LETTER QUV +AB9C; C; 13CC; # CHEROKEE SMALL LETTER SA +AB9D; C; 13CD; # CHEROKEE SMALL LETTER S +AB9E; C; 13CE; # CHEROKEE SMALL LETTER SE +AB9F; C; 13CF; # CHEROKEE SMALL LETTER SI +ABA0; C; 13D0; # CHEROKEE SMALL LETTER SO +ABA1; C; 13D1; # CHEROKEE SMALL LETTER SU +ABA2; C; 13D2; # CHEROKEE SMALL LETTER SV +ABA3; C; 13D3; # CHEROKEE SMALL LETTER DA +ABA4; C; 13D4; # CHEROKEE SMALL LETTER TA +ABA5; C; 13D5; # CHEROKEE SMALL LETTER DE +ABA6; C; 13D6; # CHEROKEE SMALL LETTER TE +ABA7; C; 13D7; # CHEROKEE SMALL LETTER DI +ABA8; C; 13D8; # CHEROKEE SMALL LETTER TI +ABA9; C; 13D9; # CHEROKEE SMALL LETTER DO +ABAA; C; 13DA; # CHEROKEE SMALL LETTER DU +ABAB; C; 13DB; # CHEROKEE SMALL LETTER DV +ABAC; C; 13DC; # CHEROKEE SMALL LETTER DLA +ABAD; C; 13DD; # CHEROKEE SMALL LETTER TLA +ABAE; C; 13DE; # CHEROKEE SMALL LETTER TLE +ABAF; C; 13DF; # CHEROKEE SMALL LETTER TLI +ABB0; C; 13E0; # CHEROKEE SMALL LETTER TLO +ABB1; C; 13E1; # CHEROKEE SMALL LETTER TLU +ABB2; C; 13E2; # CHEROKEE SMALL LETTER TLV +ABB3; C; 13E3; # CHEROKEE SMALL LETTER TSA +ABB4; C; 13E4; # CHEROKEE SMALL LETTER TSE +ABB5; C; 13E5; # CHEROKEE SMALL LETTER TSI +ABB6; C; 13E6; # CHEROKEE SMALL LETTER TSO +ABB7; C; 13E7; # CHEROKEE SMALL LETTER TSU +ABB8; C; 13E8; # CHEROKEE SMALL LETTER TSV +ABB9; C; 13E9; # CHEROKEE SMALL LETTER WA +ABBA; C; 13EA; # CHEROKEE SMALL LETTER WE +ABBB; C; 13EB; # CHEROKEE SMALL LETTER WI +ABBC; C; 13EC; # CHEROKEE SMALL LETTER WO +ABBD; C; 13ED; # CHEROKEE SMALL LETTER WU +ABBE; C; 13EE; # CHEROKEE SMALL LETTER WV +ABBF; C; 13EF; # CHEROKEE SMALL LETTER YA +FB00; F; 0066 0066; # LATIN SMALL LIGATURE FF +FB01; F; 0066 0069; # LATIN SMALL LIGATURE FI +FB02; F; 0066 006C; # LATIN SMALL LIGATURE FL +FB03; F; 0066 0066 0069; # LATIN SMALL LIGATURE FFI +FB04; F; 0066 0066 006C; # LATIN SMALL LIGATURE FFL +FB05; F; 0073 0074; # LATIN SMALL LIGATURE LONG S T +FB06; F; 0073 0074; # LATIN SMALL LIGATURE ST +FB13; F; 0574 0576; # ARMENIAN SMALL LIGATURE MEN NOW +FB14; F; 0574 0565; # ARMENIAN SMALL LIGATURE MEN ECH +FB15; F; 0574 056B; # ARMENIAN SMALL LIGATURE MEN INI +FB16; F; 057E 0576; # ARMENIAN SMALL LIGATURE VEW NOW +FB17; F; 0574 056D; # ARMENIAN SMALL LIGATURE MEN XEH +FF21; C; FF41; # FULLWIDTH LATIN CAPITAL LETTER A +FF22; C; FF42; # FULLWIDTH LATIN CAPITAL LETTER B +FF23; C; FF43; # FULLWIDTH LATIN CAPITAL LETTER C +FF24; C; FF44; # FULLWIDTH LATIN CAPITAL LETTER D +FF25; C; FF45; # FULLWIDTH LATIN CAPITAL LETTER E +FF26; C; FF46; # FULLWIDTH LATIN CAPITAL LETTER F +FF27; C; FF47; # FULLWIDTH LATIN CAPITAL LETTER G +FF28; C; FF48; # FULLWIDTH LATIN CAPITAL LETTER H +FF29; C; FF49; # FULLWIDTH LATIN CAPITAL LETTER I +FF2A; C; FF4A; # FULLWIDTH LATIN CAPITAL LETTER J +FF2B; C; FF4B; # FULLWIDTH LATIN CAPITAL LETTER K +FF2C; C; FF4C; # FULLWIDTH LATIN CAPITAL LETTER L +FF2D; C; FF4D; # FULLWIDTH LATIN CAPITAL LETTER M +FF2E; C; FF4E; # FULLWIDTH LATIN CAPITAL LETTER N +FF2F; C; FF4F; # FULLWIDTH LATIN CAPITAL LETTER O +FF30; C; FF50; # FULLWIDTH LATIN CAPITAL LETTER P +FF31; C; FF51; # FULLWIDTH LATIN CAPITAL LETTER Q +FF32; C; FF52; # FULLWIDTH LATIN CAPITAL LETTER R +FF33; C; FF53; # FULLWIDTH LATIN CAPITAL LETTER S +FF34; C; FF54; # FULLWIDTH LATIN CAPITAL LETTER T +FF35; C; FF55; # FULLWIDTH LATIN CAPITAL LETTER U +FF36; C; FF56; # FULLWIDTH LATIN CAPITAL LETTER V +FF37; C; FF57; # FULLWIDTH LATIN CAPITAL LETTER W +FF38; C; FF58; # FULLWIDTH LATIN CAPITAL LETTER X +FF39; C; FF59; # FULLWIDTH LATIN CAPITAL LETTER Y +FF3A; C; FF5A; # FULLWIDTH LATIN CAPITAL LETTER Z +10400; C; 10428; # DESERET CAPITAL LETTER LONG I +10401; C; 10429; # DESERET CAPITAL LETTER LONG E +10402; C; 1042A; # DESERET CAPITAL LETTER LONG A +10403; C; 1042B; # DESERET CAPITAL LETTER LONG AH +10404; C; 1042C; # DESERET CAPITAL LETTER LONG O +10405; C; 1042D; # DESERET CAPITAL LETTER LONG OO +10406; C; 1042E; # DESERET CAPITAL LETTER SHORT I +10407; C; 1042F; # DESERET CAPITAL LETTER SHORT E +10408; C; 10430; # DESERET CAPITAL LETTER SHORT A +10409; C; 10431; # DESERET CAPITAL LETTER SHORT AH +1040A; C; 10432; # DESERET CAPITAL LETTER SHORT O +1040B; C; 10433; # DESERET CAPITAL LETTER SHORT OO +1040C; C; 10434; # DESERET CAPITAL LETTER AY +1040D; C; 10435; # DESERET CAPITAL LETTER OW +1040E; C; 10436; # DESERET CAPITAL LETTER WU +1040F; C; 10437; # DESERET CAPITAL LETTER YEE +10410; C; 10438; # DESERET CAPITAL LETTER H +10411; C; 10439; # DESERET CAPITAL LETTER PEE +10412; C; 1043A; # DESERET CAPITAL LETTER BEE +10413; C; 1043B; # DESERET CAPITAL LETTER TEE +10414; C; 1043C; # DESERET CAPITAL LETTER DEE +10415; C; 1043D; # DESERET CAPITAL LETTER CHEE +10416; C; 1043E; # DESERET CAPITAL LETTER JEE +10417; C; 1043F; # DESERET CAPITAL LETTER KAY +10418; C; 10440; # DESERET CAPITAL LETTER GAY +10419; C; 10441; # DESERET CAPITAL LETTER EF +1041A; C; 10442; # DESERET CAPITAL LETTER VEE +1041B; C; 10443; # DESERET CAPITAL LETTER ETH +1041C; C; 10444; # DESERET CAPITAL LETTER THEE +1041D; C; 10445; # DESERET CAPITAL LETTER ES +1041E; C; 10446; # DESERET CAPITAL LETTER ZEE +1041F; C; 10447; # DESERET CAPITAL LETTER ESH +10420; C; 10448; # DESERET CAPITAL LETTER ZHEE +10421; C; 10449; # DESERET CAPITAL LETTER ER +10422; C; 1044A; # DESERET CAPITAL LETTER EL +10423; C; 1044B; # DESERET CAPITAL LETTER EM +10424; C; 1044C; # DESERET CAPITAL LETTER EN +10425; C; 1044D; # DESERET CAPITAL LETTER ENG +10426; C; 1044E; # DESERET CAPITAL LETTER OI +10427; C; 1044F; # DESERET CAPITAL LETTER EW +104B0; C; 104D8; # OSAGE CAPITAL LETTER A +104B1; C; 104D9; # OSAGE CAPITAL LETTER AI +104B2; C; 104DA; # OSAGE CAPITAL LETTER AIN +104B3; C; 104DB; # OSAGE CAPITAL LETTER AH +104B4; C; 104DC; # OSAGE CAPITAL LETTER BRA +104B5; C; 104DD; # OSAGE CAPITAL LETTER CHA +104B6; C; 104DE; # OSAGE CAPITAL LETTER EHCHA +104B7; C; 104DF; # OSAGE CAPITAL LETTER E +104B8; C; 104E0; # OSAGE CAPITAL LETTER EIN +104B9; C; 104E1; # OSAGE CAPITAL LETTER HA +104BA; C; 104E2; # OSAGE CAPITAL LETTER HYA +104BB; C; 104E3; # OSAGE CAPITAL LETTER I +104BC; C; 104E4; # OSAGE CAPITAL LETTER KA +104BD; C; 104E5; # OSAGE CAPITAL LETTER EHKA +104BE; C; 104E6; # OSAGE CAPITAL LETTER KYA +104BF; C; 104E7; # OSAGE CAPITAL LETTER LA +104C0; C; 104E8; # OSAGE CAPITAL LETTER MA +104C1; C; 104E9; # OSAGE CAPITAL LETTER NA +104C2; C; 104EA; # OSAGE CAPITAL LETTER O +104C3; C; 104EB; # OSAGE CAPITAL LETTER OIN +104C4; C; 104EC; # OSAGE CAPITAL LETTER PA +104C5; C; 104ED; # OSAGE CAPITAL LETTER EHPA +104C6; C; 104EE; # OSAGE CAPITAL LETTER SA +104C7; C; 104EF; # OSAGE CAPITAL LETTER SHA +104C8; C; 104F0; # OSAGE CAPITAL LETTER TA +104C9; C; 104F1; # OSAGE CAPITAL LETTER EHTA +104CA; C; 104F2; # OSAGE CAPITAL LETTER TSA +104CB; C; 104F3; # OSAGE CAPITAL LETTER EHTSA +104CC; C; 104F4; # OSAGE CAPITAL LETTER TSHA +104CD; C; 104F5; # OSAGE CAPITAL LETTER DHA +104CE; C; 104F6; # OSAGE CAPITAL LETTER U +104CF; C; 104F7; # OSAGE CAPITAL LETTER WA +104D0; C; 104F8; # OSAGE CAPITAL LETTER KHA +104D1; C; 104F9; # OSAGE CAPITAL LETTER GHA +104D2; C; 104FA; # OSAGE CAPITAL LETTER ZA +104D3; C; 104FB; # OSAGE CAPITAL LETTER ZHA +10C80; C; 10CC0; # OLD HUNGARIAN CAPITAL LETTER A +10C81; C; 10CC1; # OLD HUNGARIAN CAPITAL LETTER AA +10C82; C; 10CC2; # OLD HUNGARIAN CAPITAL LETTER EB +10C83; C; 10CC3; # OLD HUNGARIAN CAPITAL LETTER AMB +10C84; C; 10CC4; # OLD HUNGARIAN CAPITAL LETTER EC +10C85; C; 10CC5; # OLD HUNGARIAN CAPITAL LETTER ENC +10C86; C; 10CC6; # OLD HUNGARIAN CAPITAL LETTER ECS +10C87; C; 10CC7; # OLD HUNGARIAN CAPITAL LETTER ED +10C88; C; 10CC8; # OLD HUNGARIAN CAPITAL LETTER AND +10C89; C; 10CC9; # OLD HUNGARIAN CAPITAL LETTER E +10C8A; C; 10CCA; # OLD HUNGARIAN CAPITAL LETTER CLOSE E +10C8B; C; 10CCB; # OLD HUNGARIAN CAPITAL LETTER EE +10C8C; C; 10CCC; # OLD HUNGARIAN CAPITAL LETTER EF +10C8D; C; 10CCD; # OLD HUNGARIAN CAPITAL LETTER EG +10C8E; C; 10CCE; # OLD HUNGARIAN CAPITAL LETTER EGY +10C8F; C; 10CCF; # OLD HUNGARIAN CAPITAL LETTER EH +10C90; C; 10CD0; # OLD HUNGARIAN CAPITAL LETTER I +10C91; C; 10CD1; # OLD HUNGARIAN CAPITAL LETTER II +10C92; C; 10CD2; # OLD HUNGARIAN CAPITAL LETTER EJ +10C93; C; 10CD3; # OLD HUNGARIAN CAPITAL LETTER EK +10C94; C; 10CD4; # OLD HUNGARIAN CAPITAL LETTER AK +10C95; C; 10CD5; # OLD HUNGARIAN CAPITAL LETTER UNK +10C96; C; 10CD6; # OLD HUNGARIAN CAPITAL LETTER EL +10C97; C; 10CD7; # OLD HUNGARIAN CAPITAL LETTER ELY +10C98; C; 10CD8; # OLD HUNGARIAN CAPITAL LETTER EM +10C99; C; 10CD9; # OLD HUNGARIAN CAPITAL LETTER EN +10C9A; C; 10CDA; # OLD HUNGARIAN CAPITAL LETTER ENY +10C9B; C; 10CDB; # OLD HUNGARIAN CAPITAL LETTER O +10C9C; C; 10CDC; # OLD HUNGARIAN CAPITAL LETTER OO +10C9D; C; 10CDD; # OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG OE +10C9E; C; 10CDE; # OLD HUNGARIAN CAPITAL LETTER RUDIMENTA OE +10C9F; C; 10CDF; # OLD HUNGARIAN CAPITAL LETTER OEE +10CA0; C; 10CE0; # OLD HUNGARIAN CAPITAL LETTER EP +10CA1; C; 10CE1; # OLD HUNGARIAN CAPITAL LETTER EMP +10CA2; C; 10CE2; # OLD HUNGARIAN CAPITAL LETTER ER +10CA3; C; 10CE3; # OLD HUNGARIAN CAPITAL LETTER SHORT ER +10CA4; C; 10CE4; # OLD HUNGARIAN CAPITAL LETTER ES +10CA5; C; 10CE5; # OLD HUNGARIAN CAPITAL LETTER ESZ +10CA6; C; 10CE6; # OLD HUNGARIAN CAPITAL LETTER ET +10CA7; C; 10CE7; # OLD HUNGARIAN CAPITAL LETTER ENT +10CA8; C; 10CE8; # OLD HUNGARIAN CAPITAL LETTER ETY +10CA9; C; 10CE9; # OLD HUNGARIAN CAPITAL LETTER ECH +10CAA; C; 10CEA; # OLD HUNGARIAN CAPITAL LETTER U +10CAB; C; 10CEB; # OLD HUNGARIAN CAPITAL LETTER UU +10CAC; C; 10CEC; # OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG UE +10CAD; C; 10CED; # OLD HUNGARIAN CAPITAL LETTER RUDIMENTA UE +10CAE; C; 10CEE; # OLD HUNGARIAN CAPITAL LETTER EV +10CAF; C; 10CEF; # OLD HUNGARIAN CAPITAL LETTER EZ +10CB0; C; 10CF0; # OLD HUNGARIAN CAPITAL LETTER EZS +10CB1; C; 10CF1; # OLD HUNGARIAN CAPITAL LETTER ENT-SHAPED SIGN +10CB2; C; 10CF2; # OLD HUNGARIAN CAPITAL LETTER US +118A0; C; 118C0; # WARANG CITI CAPITAL LETTER NGAA +118A1; C; 118C1; # WARANG CITI CAPITAL LETTER A +118A2; C; 118C2; # WARANG CITI CAPITAL LETTER WI +118A3; C; 118C3; # WARANG CITI CAPITAL LETTER YU +118A4; C; 118C4; # WARANG CITI CAPITAL LETTER YA +118A5; C; 118C5; # WARANG CITI CAPITAL LETTER YO +118A6; C; 118C6; # WARANG CITI CAPITAL LETTER II +118A7; C; 118C7; # WARANG CITI CAPITAL LETTER UU +118A8; C; 118C8; # WARANG CITI CAPITAL LETTER E +118A9; C; 118C9; # WARANG CITI CAPITAL LETTER O +118AA; C; 118CA; # WARANG CITI CAPITAL LETTER ANG +118AB; C; 118CB; # WARANG CITI CAPITAL LETTER GA +118AC; C; 118CC; # WARANG CITI CAPITAL LETTER KO +118AD; C; 118CD; # WARANG CITI CAPITAL LETTER ENY +118AE; C; 118CE; # WARANG CITI CAPITAL LETTER YUJ +118AF; C; 118CF; # WARANG CITI CAPITAL LETTER UC +118B0; C; 118D0; # WARANG CITI CAPITAL LETTER ENN +118B1; C; 118D1; # WARANG CITI CAPITAL LETTER ODD +118B2; C; 118D2; # WARANG CITI CAPITAL LETTER TTE +118B3; C; 118D3; # WARANG CITI CAPITAL LETTER NUNG +118B4; C; 118D4; # WARANG CITI CAPITAL LETTER DA +118B5; C; 118D5; # WARANG CITI CAPITAL LETTER AT +118B6; C; 118D6; # WARANG CITI CAPITAL LETTER AM +118B7; C; 118D7; # WARANG CITI CAPITAL LETTER BU +118B8; C; 118D8; # WARANG CITI CAPITAL LETTER PU +118B9; C; 118D9; # WARANG CITI CAPITAL LETTER HIYO +118BA; C; 118DA; # WARANG CITI CAPITAL LETTER HOLO +118BB; C; 118DB; # WARANG CITI CAPITAL LETTER HORR +118BC; C; 118DC; # WARANG CITI CAPITAL LETTER HAR +118BD; C; 118DD; # WARANG CITI CAPITAL LETTER SSUU +118BE; C; 118DE; # WARANG CITI CAPITAL LETTER SII +118BF; C; 118DF; # WARANG CITI CAPITAL LETTER VIYO +16E40; C; 16E60; # MEDEFAIDRIN CAPITAL LETTER M +16E41; C; 16E61; # MEDEFAIDRIN CAPITAL LETTER S +16E42; C; 16E62; # MEDEFAIDRIN CAPITAL LETTER V +16E43; C; 16E63; # MEDEFAIDRIN CAPITAL LETTER W +16E44; C; 16E64; # MEDEFAIDRIN CAPITAL LETTER ATIU +16E45; C; 16E65; # MEDEFAIDRIN CAPITAL LETTER Z +16E46; C; 16E66; # MEDEFAIDRIN CAPITAL LETTER KP +16E47; C; 16E67; # MEDEFAIDRIN CAPITAL LETTER P +16E48; C; 16E68; # MEDEFAIDRIN CAPITAL LETTER T +16E49; C; 16E69; # MEDEFAIDRIN CAPITAL LETTER G +16E4A; C; 16E6A; # MEDEFAIDRIN CAPITAL LETTER F +16E4B; C; 16E6B; # MEDEFAIDRIN CAPITAL LETTER I +16E4C; C; 16E6C; # MEDEFAIDRIN CAPITAL LETTER K +16E4D; C; 16E6D; # MEDEFAIDRIN CAPITAL LETTER A +16E4E; C; 16E6E; # MEDEFAIDRIN CAPITAL LETTER J +16E4F; C; 16E6F; # MEDEFAIDRIN CAPITAL LETTER E +16E50; C; 16E70; # MEDEFAIDRIN CAPITAL LETTER B +16E51; C; 16E71; # MEDEFAIDRIN CAPITAL LETTER C +16E52; C; 16E72; # MEDEFAIDRIN CAPITAL LETTER U +16E53; C; 16E73; # MEDEFAIDRIN CAPITAL LETTER YU +16E54; C; 16E74; # MEDEFAIDRIN CAPITAL LETTER L +16E55; C; 16E75; # MEDEFAIDRIN CAPITAL LETTER Q +16E56; C; 16E76; # MEDEFAIDRIN CAPITAL LETTER HP +16E57; C; 16E77; # MEDEFAIDRIN CAPITAL LETTER NY +16E58; C; 16E78; # MEDEFAIDRIN CAPITAL LETTER X +16E59; C; 16E79; # MEDEFAIDRIN CAPITAL LETTER D +16E5A; C; 16E7A; # MEDEFAIDRIN CAPITAL LETTER OE +16E5B; C; 16E7B; # MEDEFAIDRIN CAPITAL LETTER N +16E5C; C; 16E7C; # MEDEFAIDRIN CAPITAL LETTER R +16E5D; C; 16E7D; # MEDEFAIDRIN CAPITAL LETTER O +16E5E; C; 16E7E; # MEDEFAIDRIN CAPITAL LETTER AI +16E5F; C; 16E7F; # MEDEFAIDRIN CAPITAL LETTER Y +1E900; C; 1E922; # ADLAM CAPITAL LETTER ALIF +1E901; C; 1E923; # ADLAM CAPITAL LETTER DAALI +1E902; C; 1E924; # ADLAM CAPITAL LETTER LAAM +1E903; C; 1E925; # ADLAM CAPITAL LETTER MIIM +1E904; C; 1E926; # ADLAM CAPITAL LETTER BA +1E905; C; 1E927; # ADLAM CAPITAL LETTER SINNYIIYHE +1E906; C; 1E928; # ADLAM CAPITAL LETTER PE +1E907; C; 1E929; # ADLAM CAPITAL LETTER BHE +1E908; C; 1E92A; # ADLAM CAPITAL LETTER RA +1E909; C; 1E92B; # ADLAM CAPITAL LETTER E +1E90A; C; 1E92C; # ADLAM CAPITAL LETTER FA +1E90B; C; 1E92D; # ADLAM CAPITAL LETTER I +1E90C; C; 1E92E; # ADLAM CAPITAL LETTER O +1E90D; C; 1E92F; # ADLAM CAPITAL LETTER DHA +1E90E; C; 1E930; # ADLAM CAPITAL LETTER YHE +1E90F; C; 1E931; # ADLAM CAPITAL LETTER WAW +1E910; C; 1E932; # ADLAM CAPITAL LETTER NUN +1E911; C; 1E933; # ADLAM CAPITAL LETTER KAF +1E912; C; 1E934; # ADLAM CAPITAL LETTER YA +1E913; C; 1E935; # ADLAM CAPITAL LETTER U +1E914; C; 1E936; # ADLAM CAPITAL LETTER JIIM +1E915; C; 1E937; # ADLAM CAPITAL LETTER CHI +1E916; C; 1E938; # ADLAM CAPITAL LETTER HA +1E917; C; 1E939; # ADLAM CAPITAL LETTER QAAF +1E918; C; 1E93A; # ADLAM CAPITAL LETTER GA +1E919; C; 1E93B; # ADLAM CAPITAL LETTER NYA +1E91A; C; 1E93C; # ADLAM CAPITAL LETTER TU +1E91B; C; 1E93D; # ADLAM CAPITAL LETTER NHA +1E91C; C; 1E93E; # ADLAM CAPITAL LETTER VA +1E91D; C; 1E93F; # ADLAM CAPITAL LETTER KHA +1E91E; C; 1E940; # ADLAM CAPITAL LETTER GBE +1E91F; C; 1E941; # ADLAM CAPITAL LETTER ZAL +1E920; C; 1E942; # ADLAM CAPITAL LETTER KPO +1E921; C; 1E943; # ADLAM CAPITAL LETTER SHA +# +# EOF diff --git a/claimtrie/normalization/case_folder.go b/claimtrie/normalization/case_folder.go new file mode 100644 index 00000000..0d7e5747 --- /dev/null +++ b/claimtrie/normalization/case_folder.go @@ -0,0 +1,61 @@ +package normalization + +import ( + "bytes" + _ "embed" + "regexp" + "strconv" + "strings" + "unicode/utf8" +) + +//go:embed CaseFolding_v11.txt +var v11 string + +var foldMap map[rune][]rune + +func init() { + foldMap = map[rune][]rune{} + r, _ := regexp.Compile(`([[:xdigit:]]+?); (.); ([[:xdigit:] ]+?);`) + matches := r.FindAllStringSubmatch(v11, 1000000000) + for i := range matches { + if matches[i][2] == "C" || matches[i][2] == "F" { + key, err := strconv.ParseUint(matches[i][1], 16, len(matches[i][1])*4) + if err != nil { + panic(err) + } + splits := strings.Split(matches[i][3], " ") + var values []rune + for j := range splits { + value, err := strconv.ParseUint(splits[j], 16, len(splits[j])*4) + if err != nil { + panic(err) + } + values = append(values, rune(value)) + } + foldMap[rune(key)] = values + } + } +} + +func CaseFold(name []byte) []byte { + var b bytes.Buffer + b.Grow(len(name)) + for i := 0; i < len(name); { + r, w := utf8.DecodeRune(name[i:]) + if r == utf8.RuneError && w < 2 { + // HACK: their RuneError is actually a valid character if coming from a width of 2 or more + return name + } + replacements := foldMap[r] + if len(replacements) > 0 { + for j := range replacements { + b.WriteRune(replacements[j]) + } + } else { + b.WriteRune(r) + } + i += w + } + return b.Bytes() +} diff --git a/claimtrie/normalization/normalizer.go b/claimtrie/normalization/normalizer.go new file mode 100644 index 00000000..55275105 --- /dev/null +++ b/claimtrie/normalization/normalizer.go @@ -0,0 +1,23 @@ +package normalization + +import ( + "github.com/lbryio/lbcd/claimtrie/param" + "golang.org/x/text/unicode/norm" +) + +var Normalize = normalizeGo +var NormalizeTitle = "Normalizing strings via Go. Casefold table version = 11.0.0, NFD version = " + norm.Version + +func NormalizeIfNecessary(name []byte, height int32) []byte { + if height < param.ActiveParams.NormalizedNameForkHeight { + return name + } + return Normalize(name) +} + +func normalizeGo(value []byte) []byte { + + normalized := norm.NFD.Bytes(value) // may need to hard-code the version on this + // not using x/text/cases because it does too good of a job; it seems to use v14 tables even when it claims v13 + return CaseFold(normalized) +} diff --git a/claimtrie/normalization/normalizer_icu.go b/claimtrie/normalization/normalizer_icu.go new file mode 100644 index 00000000..d5093ba2 --- /dev/null +++ b/claimtrie/normalization/normalizer_icu.go @@ -0,0 +1,67 @@ +//go:build use_icu_normalization +// +build use_icu_normalization + +package normalization + +// #cgo CFLAGS: -O2 +// #cgo LDFLAGS: -licuio -licui18n -licuuc -licudata +// #include +// #include +// #include +// int icu_version() { +// UVersionInfo info; +// u_getVersion(info); +// return ((int)(info[0]) << 16) + info[1]; +// } +// int normalize(char* name, int length, char* result) { +// UErrorCode ec = U_ZERO_ERROR; +// static const UNormalizer2* normalizer = NULL; +// if (normalizer == NULL) normalizer = unorm2_getNFDInstance(&ec); +// UChar dest[256]; // maximum claim name size is 255; we won't have more UTF16 chars than bytes +// int dest_len; +// u_strFromUTF8(dest, 256, &dest_len, name, length, &ec); +// if (U_FAILURE(ec) || dest_len == 0) return 0; +// UChar normalized[256]; +// dest_len = unorm2_normalize(normalizer, dest, dest_len, normalized, 256, &ec); +// if (U_FAILURE(ec) || dest_len == 0) return 0; +// dest_len = u_strFoldCase(dest, 256, normalized, dest_len, U_FOLD_CASE_DEFAULT, &ec); +// if (U_FAILURE(ec) || dest_len == 0) return 0; +// u_strToUTF8(result, 512, &dest_len, dest, dest_len, &ec); +// return dest_len; +// } +import "C" +import ( + "fmt" + "unsafe" +) + +func init() { + Normalize = normalizeICU + NormalizeTitle = "Normalizing strings via ICU. ICU version = " + IcuVersion() +} + +func IcuVersion() string { + // TODO: we probably need to explode if it's not 63.2 as it affects consensus + result := C.icu_version() + return fmt.Sprintf("%d.%d", result>>16, result&0xffff) +} + +func normalizeICU(value []byte) []byte { + if len(value) <= 0 { + return value + } + name := (*C.char)(unsafe.Pointer(&value[0])) + length := C.int(len(value)) + + // hopefully this is a stack alloc (but it may be a bit large for that): + var resultName [512]byte // inputs are restricted to 255 chars; it shouldn't expand too much past that + result := unsafe.Pointer(&resultName[0]) + + resultLength := C.normalize(name, length, (*C.char)(result)) + if resultLength == 0 { + return value + } + + // return resultName[0:resultLength] -- we want to shrink the result (not use a slice on 1024) + return C.GoBytes(result, resultLength) +} diff --git a/claimtrie/normalization/normalizer_icu_test.go b/claimtrie/normalization/normalizer_icu_test.go new file mode 100644 index 00000000..b02f315a --- /dev/null +++ b/claimtrie/normalization/normalizer_icu_test.go @@ -0,0 +1,65 @@ +//go:build use_icu_normalization +// +build use_icu_normalization + +package normalization + +import ( + "encoding/hex" + "testing" + "unicode/utf8" + + "github.com/stretchr/testify/assert" +) + +func TestNormalizationICU(t *testing.T) { + testNormalization(t, normalizeICU) +} + +func BenchmarkNormalizeICU(b *testing.B) { + benchmarkNormalize(b, normalizeICU) +} + +var testStrings = []string{ + "Les-Masques-Blancs-Die-Dead-place-Sathonay-28-Août", + "Bez-komentu-výbuch-z-vnútra,-radšej-pozri-video...-", + "၂-နစ်အကြာမှာ", + "ငရဲပြည်မှ-6", + "@happyvision", + "ကမ္ဘာပျက်ကိန်း-9", + "ဝိညာဉ်နား၊-3", + "un-amore-nuovo-o-un-ritorno-cosa-mi-dona", + "è-innamorato-di-me-anche-se-non-lo-dice", + "ပြင်ဆင်ပါ-no.1", + "ပြင်ဆင်ပါ-no.4", + "ပြင်ဆင်ပါ-no.2", + "ပြင်ဆင်ပါ-no.3", + "ငရဲပြည်မှ-5", + "ပြင်ဆင်ပါ-no.6", + "ပြင်ဆင်ပါ-no.5", + "ပြင်ဆင်ပါ-no.7", + "ပြင်ဆင်ပါ-no.8", + "အချိန်-2", + "ဝိညာဉ်နား၊-4", + "ပြင်ဆင်ပါ-no.-13", + "ပြင်ဆင်ပါ-no.15", + "ပြင်ဆင်ပါ-9", + "schilddrüsenhormonsubstitution-nach", + "Linxextremismus-JPzuG_UBtEg", + "Ꮖ-Ꮩ-Ꭺ-N--------Ꭺ-N-Ꮹ-Ꭼ-Ꮮ-Ꭺ-on-Instagram_-“Our-next-destination-is-East-and-Southeast-Asia--selfie--asia”", + "ABCDEFGHIJKLMNOPQRSTUVWXYZ", +} + +func TestBlock760150_1020105(t *testing.T) { + test, _ := hex.DecodeString("43efbfbd") + assert.True(t, utf8.Valid(test)) + a := normalizeGo(test) + b := normalizeICU(test) + assert.Equal(t, a, b) + + for i, s := range testStrings { + a = normalizeGo([]byte(s)) + b = normalizeICU([]byte(s)) + assert.Equal(t, a, b, "%d: %s != %s", i, string(a), string(b)) + // t.Logf("%s -> %s", s, string(b)) + } +} diff --git a/claimtrie/normalization/normalizer_test.go b/claimtrie/normalization/normalizer_test.go new file mode 100644 index 00000000..ea43b677 --- /dev/null +++ b/claimtrie/normalization/normalizer_test.go @@ -0,0 +1,54 @@ +package normalization + +import ( + "math/rand" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNormalizationGo(t *testing.T) { + testNormalization(t, normalizeGo) +} + +func testNormalization(t *testing.T, normalize func(value []byte) []byte) { + + r := require.New(t) + + r.Equal("test", string(normalize([]byte("TESt")))) + r.Equal("test 23", string(normalize([]byte("tesT 23")))) + r.Equal("\xFF", string(normalize([]byte("\xFF")))) + r.Equal("\xC3\x28", string(normalize([]byte("\xC3\x28")))) + r.Equal("\xCF\x89", string(normalize([]byte("\xE2\x84\xA6")))) + r.Equal("\xD1\x84", string(normalize([]byte("\xD0\xA4")))) + r.Equal("\xD5\xA2", string(normalize([]byte("\xD4\xB2")))) + r.Equal("\xE3\x81\xB5\xE3\x82\x99", string(normalize([]byte("\xE3\x81\xB6")))) + r.Equal("\xE1\x84\x81\xE1\x85\xAA\xE1\x86\xB0", string(normalize([]byte("\xEA\xBD\x91")))) +} + +func randSeq(n int) []byte { + var alphabet = []rune("abcdefghijklmnopqrstuvwxyz̃ABCDEFGHIJKLMNOPQRSTUVWXYZ̃") + + b := make([]rune, n) + for i := range b { + b[i] = alphabet[rand.Intn(len(alphabet))] + } + return []byte(string(b)) +} + +func BenchmarkNormalize(b *testing.B) { + benchmarkNormalize(b, normalizeGo) +} + +func benchmarkNormalize(b *testing.B, normalize func(value []byte) []byte) { + rand.Seed(42) + strings := make([][]byte, b.N) + for i := 0; i < b.N; i++ { + strings[i] = randSeq(32) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + s := normalize(strings[i]) + require.True(b, len(s) >= 8) + } +} diff --git a/claimtrie/param/delays.go b/claimtrie/param/delays.go new file mode 100644 index 00000000..d310877f --- /dev/null +++ b/claimtrie/param/delays.go @@ -0,0 +1,285 @@ +package param + +var DelayWorkarounds = generateDelayWorkarounds() // called "removal workarounds" in previous versions + +func generateDelayWorkarounds() map[string][]int32 { + return map[string][]int32{ + "travtest01": {426898}, + "gauntlet-invade-the-darkness-lvl-1-of": {583305}, + "fr-let-s-play-software-inc-jay": {588308}, + "fr-motorsport-manager-jay-s-racing": {588308}, + "fr-crusader-kings-2-la-dynastie-6": {588318}, + "fr-jurassic-world-evolution-let-s-play": {588318}, + "calling-tech-support-scammers-live-3": {588683, 646584}, + "let-s-play-jackbox-games": {589013}, + "lets-play-jackbox-games-5": {589013}, + "kabutothesnake-s-live-ps4-broadcast": {589538}, + "no-eas-strong-thunderstorm-advisory": {589554}, + "geometry-dash-level-requests": {589564}, + "geometry-dash-level-requests-2": {589564}, + "star-ocean-integrity-and-faithlessness": {589609}, + "@pop": {589613}, + "ullash": {589630}, + "today-s-professionals-2018-winter-3": {589640}, + "today-s-professionals-2018-winter-4": {589640}, + "today-s-professionals-2018-winter-10": {589641}, + "today-s-professionals-big-brother-6-13": {589641}, + "today-s-professionals-big-brother-6-14": {589641}, + "today-s-professionals-big-brother-6-26": {589641}, + "today-s-professionals-big-brother-6-27": {589641}, + "today-s-professionals-big-brother-6-28": {589641}, + "today-s-professionals-big-brother-6-29": {589641}, + "dark-souls-iii": {589697}, + "bobby-blades": {589760}, + "adrian": {589803}, + "roblox-2": {589803, 597925}, + "roblox-4": {589803}, + "roblox-5": {589803}, + "roblox-6": {589803}, + "roblox-7": {589803}, + "roblox-8": {589803}, + "madden-17": {589809}, + "madden-18-franchise": {589810}, + "fifa-14-android-astrodude44-vs": {589831}, + "gaming-with-silverwolf-live-stream-3": {589849}, + "gaming-with-silverwolf-live-stream-4": {589849}, + "gaming-with-silverwolf-live-stream-5": {589849}, + "gaming-with-silverwolf-videos-live": {589849}, + "gaming-with-silverwolf-live-stream-6": {589851}, + "live-q-a": {589851}, + "classic-sonic-games": {589870}, + "gta": {589926}, + "j-dog7973-s-fortnite-squad": {589926}, + "wow-warlords-of-draenor-horde-side": {589967}, + "minecraft-ps4-hardcore-survival-2-the-5": {589991}, + "happy-new-year-2017": {590013}, + "come-chill-with-rekzzey-2": {590020}, + "counter-strike-global-offensive-funny": {590031}, + "father-vs-son-stickfight-stickfight": {590178}, + "little-t-playing-subnautica-livestream": {590178}, + "today-s-professionals-big-brother-7-26-5": {590200}, + "50585be4e3159a7-1": {590206}, + "dark-souls-iii-soul-level-1-challenge": {590223}, + "dark-souls-iii-soul-level-1-challenge-3": {590223}, + "let-s-play-sniper-elite-4-authentic-2": {590225}, + "skyrim-special-edition-ps4-platinum-4": {590225}, + "let-s-play-final-fantasy-the-zodiac-2": {590226}, + "let-s-play-final-fantasy-the-zodiac-3": {590226}, + "ls-h-ppchen-halloween-stream-vom-31-10": {590401}, + "a-new-stream": {590669}, + "danganronpa-v3-killing-harmony-episode": {590708}, + "danganronpa-v3-killing-harmony-episode-4": {590708}, + "danganronpa-v3-killing-harmony-episode-6": {590708}, + "danganronpa-v3-killing-harmony-episode-8": {590708}, + "danganronpa-v3-killing-harmony-episode-9": {590708}, + "call-of-duty-infinite-warfare-gameplay-2": {591982}, + "destiny-the-taken-king-gameplay": {591982}, + "horizon-zero-dawn-100-complete-4": {591983}, + "ghost-recon-wildlands-100-complete-4": {591984}, + "nier-automata-100-complete-gameplay-25": {591985}, + "frustrert": {592291}, + "call-of-duty-black-ops-3-multiplayer": {593504}, + "rayman-legends-challenges-app-the": {593551}, + "super-mario-sunshine-3-player-race-2": {593552}, + "some-new-stuff-might-play-a-game": {593698}, + "memory-techniques-1-000-people-system": {595537}, + "propresenter-6-tutorials-new-features-4": {595559}, + "rocket-league-live": {595559}, + "fortnite-battle-royale": {595818}, + "fortnite-battle-royale-2": {595818}, + "ohare12345-s-live-ps4-broadcast": {595818}, + "super-smash-bros-u-home-run-contest-13": {595838}, + "super-smash-bros-u-home-run-contest-15": {595838}, + "super-smash-bros-u-home-run-contest-2": {595838, 595844}, + "super-smash-bros-u-home-run-contest-22": {595838, 595845}, + "super-smash-bros-u-multi-man-smash-3": {595838}, + "minecraft-survival-biedronka-i-czarny-2": {596828}, + "gramy-minecraft-jasmc-pl": {596829}, + "farcry-5-gameplay": {595818}, + "my-channel-trailer": {595818}, + "full-song-production-tutorial-aeternum": {596934}, + "blackboxglobalreview-hd": {597091}, + "tom-clancy-s-rainbow-six-siege": {597633}, + "5-new-technology-innovations-in-5": {597635}, + "5-new-technology-innovations-in-5-2": {597635}, + "how-to-play-nothing-else-matters-on": {597637}, + "rb6": {597639}, + "borderlands-2-tiny-tina-s-assault-on": {597658}, + "let-s-play-borderlands-the-pre-sequel": {597658}, + "caveman-world-mountains-of-unga-boonga": {597660}, + "for-honor-ps4-2": {597706}, + "fortnite-episode-1": {597728}, + "300-subscribers": {597750}, + "viscera-cleanup-detail-santa-s-rampage": {597755}, + "infinite-voxel-terrain-in-unity-update": {597777}, + "let-s-play-pok-mon-light-platinum": {597783}, + "video-2": {597785}, + "video-8": {597785}, + "finally": {597793}, + "let-s-play-mario-party-luigi-s-engine": {597796}, + "my-edited-video": {597799}, + "we-need-to-talk": {597800}, + "tf2-stream-2": {597811}, + "royal-thumble-tuesday-night-thumbdown": {597814}, + "beat-it-michael-jackson-cover": {597815}, + "black-ops-3": {597816}, + "call-of-duty-black-ops-3-campaign": {597819}, + "skyrim-special-edition-silent-2": {597822}, + "the-chainsmokers-everybody-hates-me": {597823}, + "experiment-glowing-1000-degree-knife-vs": {597824}, + "l1011widebody-friends-let-s-play-2": {597824}, + "call-of-duty-black-ops-4": {597825}, + "let-s-play-fallout-2-restoration-3": {597825}, + "let-s-play-fallout-2-restoration-19": {597826}, + "let-s-play-fallout-2-restoration-27": {597826}, + "2015": {597828}, + "payeer": {597829}, + "youtube-3": {597829}, + "bitcoin-5": {597830}, + "2016": {597831}, + "bitcoin-2": {597831}, + "dreamtowards": {597831}, + "surfearner": {597831}, + "100-000": {597832}, + "20000": {597833}, + "remme": {597833}, + "hycon": {597834}, + "robocraft": {597834}, + "saturday-night-baseball-with-37": {597834}, + "let-s-play-command-conquer-red-alert-9": {597835}, + "15-curiosidades-que-probablemente-ya": {597837}, + "elder-scrolls-online-road-to-level-20": {597893}, + "playerunknown-s-battlegrounds": {597894}, + "black-ops-3-fun": {597897}, + "mortal-kombat-xl-the-funniest": {597899}, + "try-not-to-laugh-2": {597899}, + "call-of-duty-advanced-warfare-domination": {597898}, + "my-live-stream-with-du-recorder-5": {597900}, + "ls-h-ppchen-halloween-stream-vom-31-10-2": {597904}, + "ls-h-ppchen-halloween-stream-vom-31-10-3": {597904}, + "how-it-feels-to-chew-5-gum-funny-8": {597905}, + "live-stream-mu-club-america-3": {597918}, + "black-death": {597927}, + "lets-play-spore-with-3": {597929}, + "true-mov-2": {597933}, + "fortnite-w-pat-the-rat-pat-the-rat": {597935}, + "jugando-pokemon-esmeralda-gba": {597935}, + "talking-about-my-channel-and-much-more-4": {597936}, + "-14": {597939}, + "-15": {597939}, + "-16": {597939}, + "-17": {597939}, + "-18": {597939}, + "-20": {597939}, + "-21": {597939}, + "-24": {597939}, + "-25": {597939}, + "-26": {597939}, + "-27": {597939}, + "-28": {597939}, + "-29": {597939}, + "-31": {597941}, + "-34": {597941}, + "-6": {597939}, + "-7": {597939}, + "10-4": {612097}, + "10-6": {612097}, + "10-7": {612097}, + "10-diy": {612097}, + "10-twitch": {612097}, + "100-5": {597909}, + "189f2f04a378c02-1": {612097}, + "2011-2": {597917}, + "2011-3": {597917}, + "2c61c818687ed09-1": {612097}, + "5-diy-4": {612097}, + "@andymcdandycdn": {640212}, + "@lividjava": {651654}, + "@mhx": {653957}, + "@tipwhatyoulike": {599792}, + "@wibbels": {612195}, + "@yisraeldov": {647416}, + "beyaz-hap-biseks-el-evlat": {657957}, + "bilgisayar-al-t-rma-s-recinde-ya-ananlar": {657957}, + "brave-como-ganhar-dinheiro-todos-os-dias": {598494}, + "c81e728d9d4c2f6-1": {598178}, + "call-of-duty-world-war-2": {597935}, + "chain-reaction": {597940}, + "commodore-64-an-lar-ve-oyunlar": {657957}, + "counter-strike-global-offensive-gameplay": {597900}, + "dead-island-riptide-co-op-walkthrough-2": {597904, 598105}, + "diy-10": {612097}, + "diy-11": {612097}, + "diy-13": {612097}, + "diy-14": {612097}, + "diy-19": {612097}, + "diy-4": {612097}, + "diy-6": {612097}, + "diy-7": {612097}, + "diy-9": {612097}, + "doktor-ve-patron-sahnesinin-haz-rl-k-ve": {657957}, + "eat-the-street": {597910}, + "fallout-4-modded": {597901}, + "fallout-4-walkthrough": {597900}, + "filmli-efecast-129-film-inde-film-inde": {657957}, + "filmli-efecast-130-ger-ek-hayatta-anime": {657957}, + "filmli-efecast-97-netflix-filmi-form-l": {657957}, + "for-honor-2": {597932}, + "for-honor-4": {597932}, + "gta-5": {597902}, + "gta-5-2": {597902}, + "helldriver-g-n-n-ekstrem-filmi": {657957}, + "hi-4": {597933}, + "hi-5": {597933}, + "hi-7": {597933}, + "kizoa-movie-video-slideshow-maker": {597900, 597932}, + "l1011widebody-friends-let-s-play-3": {598070}, + "lbry": {608276}, + "lets-play-spore-with": {597930}, + "madants": {625032}, + "mechwarrior-2-soundtrack-clan-jade": {598070}, + "milo-forbidden-conversation": {655173}, + "mobile-record": {597910}, + "mouths": {607379}, + "mp-aleyna-tilki-nin-zorla-seyrettirilen": {657957}, + "mp-atat-rk-e-eytan-diyen-yunan-as-ll": {657957}, + "mp-bah-eli-calan-avukatlar-yla-g-r-s-n": {657957}, + "mp-bu-podcast-babalar-in": {657957}, + "mp-bu-podcasti-akp-li-tan-d-klar-n-za": {657957}, + "mp-gaziantep-te-tacizle-su-lan-p-dayak": {650409}, + "mp-hatipo-lu-nun-ermeni-bir-ocu-u-canl": {657957}, + "mp-k-rt-annelerin-hdp-ye-tepkisi": {657957}, + "mp-kenan-sofuo-lu-nun-mamo-lu-na-destek": {657957}, + "mp-mamo-lu-nun-muhafazakar-g-r-nmesi": {657957}, + "mp-mhp-akp-gerginli-i": {657957}, + "mp-otob-ste-t-rkle-meyin-diye-ba-ran-svi": {657957}, + "mp-pace-i-kazand-m-diyip-21-bin-dolar": {657957}, + "mp-rusya-da-kad-nlara-tecav-zc-s-n-ld": {657957}, + "mp-s-n-rs-z-nafakan-n-kalkmas-adil-mi": {657957}, + "mp-susamam-ark-s-ve-serkan-nci-nin-ark": {657957}, + "mp-y-lmaz-zdil-in-kitap-paralar-yla-yard": {657957}, + "mp-yang-n-u-aklar-pahal-diyen-orman": {657957}, + "mp-yeni-zelanda-katliam-ndan-siyasi-rant": {657957}, + "my-edited-video-4": {597932}, + "my-live-stream-with-du-recorder": {597900}, + "my-live-stream-with-du-recorder-3": {597900}, + "new-channel-intro": {598235}, + "paladins-3": {597900}, + "popstar-sahnesi-kamera-arkas-g-r-nt-leri": {657957}, + "retro-bilgisayar-bulu-mas": {657957}, + "scp-t-rk-e-scp-002-canl-oda": {657957}, + "steep": {597900}, + "stephen-hicks-postmodernism-reprise": {655173}, + "super-smash-bros-u-brawl-co-op-event": {595841}, + "super-smash-bros-u-super-mario-u-smash": {595839}, + "super-smash-bros-u-zelda-smash-series": {595841}, + "superonline-fiber-den-efsane-kaz-k-yedim": {657957}, + "talking-about-my-channel-and-much-more-5": {597936}, + "test1337reflector356": {627814}, + "the-last-of-us-remastered-2": {597915}, + "tom-clancy-s-ghost-recon-wildlands-2": {597916}, + "tom-clancy-s-rainbow-six-siege-3": {597935}, + "wwe-2k18-with-that-guy-and-tricky": {597901}, + "yay-nc-bob-afet-kamera-arkas": {657957}, + } +} diff --git a/claimtrie/param/general.go b/claimtrie/param/general.go new file mode 100644 index 00000000..92ff06fe --- /dev/null +++ b/claimtrie/param/general.go @@ -0,0 +1,74 @@ +package param + +import "github.com/lbryio/lbcd/wire" + +type ClaimTrieParams struct { + MaxActiveDelay int32 + ActiveDelayFactor int32 + + MaxNodeManagerCacheSize int + + OriginalClaimExpirationTime int32 + ExtendedClaimExpirationTime int32 + ExtendedClaimExpirationForkHeight int32 + + MaxRemovalWorkaroundHeight int32 + + NormalizedNameForkHeight int32 + AllClaimsInMerkleForkHeight int32 +} + +var ( + ActiveParams = MainNet + + MainNet = ClaimTrieParams{ + MaxActiveDelay: 4032, + ActiveDelayFactor: 32, + MaxNodeManagerCacheSize: 32000, + + OriginalClaimExpirationTime: 262974, + ExtendedClaimExpirationTime: 2102400, + ExtendedClaimExpirationForkHeight: 400155, // https://lbry.io/news/hf1807 + MaxRemovalWorkaroundHeight: 658300, + NormalizedNameForkHeight: 539940, // targeting 21 March 2019}, https://lbry.com/news/hf1903 + AllClaimsInMerkleForkHeight: 658309, // targeting 30 Oct 2019}, https://lbry.com/news/hf1910 + } + + TestNet = ClaimTrieParams{ + MaxActiveDelay: 4032, + ActiveDelayFactor: 32, + MaxNodeManagerCacheSize: 32000, + + OriginalClaimExpirationTime: 262974, + ExtendedClaimExpirationTime: 2102400, + ExtendedClaimExpirationForkHeight: 278160, + MaxRemovalWorkaroundHeight: 1, // if you get a hash mismatch, come back to this + NormalizedNameForkHeight: 993380, + AllClaimsInMerkleForkHeight: 1198559, + } + + Regtest = ClaimTrieParams{ + MaxActiveDelay: 4032, + ActiveDelayFactor: 32, + MaxNodeManagerCacheSize: 32000, + + OriginalClaimExpirationTime: 500, + ExtendedClaimExpirationTime: 600, + ExtendedClaimExpirationForkHeight: 800, + MaxRemovalWorkaroundHeight: -1, + NormalizedNameForkHeight: 250, + AllClaimsInMerkleForkHeight: 349, + } +) + +func SetNetwork(net wire.BitcoinNet) { + + switch net { + case wire.MainNet: + ActiveParams = MainNet + case wire.TestNet3: + ActiveParams = TestNet + case wire.TestNet, wire.SimNet: // "regtest" + ActiveParams = Regtest + } +} diff --git a/claimtrie/param/takeovers.go b/claimtrie/param/takeovers.go new file mode 100644 index 00000000..7ba125ac --- /dev/null +++ b/claimtrie/param/takeovers.go @@ -0,0 +1,451 @@ +package param + +var TakeoverWorkarounds = generateTakeoverWorkarounds() + +func generateTakeoverWorkarounds() map[string]int { // TODO: the values here are unused; bools would probably be better + return map[string]int{ + "496856_HunterxHunterAMV": 496835, + "542978_namethattune1": 542429, + "543508_namethattune-5": 543306, + "546780_forecasts": 546624, + "548730_forecasts": 546780, + "551540_forecasts": 548730, + "552380_chicthinkingofyou": 550804, + "560363_takephotowithlbryteam": 559962, + "563710_test-img": 563700, + "566750_itila": 543261, + "567082_malabarismo-com-bolas-de-futebol-vs-chap": 563592, + "596860_180mphpullsthrougheurope": 596757, + "617743_vaccines": 572756, + "619609_copface-slamshandcuffedteengirlintoconcrete": 539940, + "620392_banker-exposes-satanic-elite": 597788, + "624997_direttiva-sulle-armi-ue-in-svizzera-di": 567908, + "624997_best-of-apex": 585580, + "629970_cannot-ignore-my-veins": 629914, + "633058_bio-waste-we-programmed-your-brain": 617185, + "633601_macrolauncher-overview-first-look": 633058, + "640186_its-up-to-you-and-i-2019": 639116, + "640241_tor-eas-3-20": 592645, + "640522_seadoxdark": 619531, + "640617_lbry-przewodnik-1-instalacja": 451186, + "640623_avxchange-2019-the-next-netflix-spotify": 606790, + "640684_algebra-introduction": 624152, + "640684_a-high-school-math-teacher-does-a": 600885, + "640684_another-random-life-update": 600884, + "640684_who-is-the-taylor-series-for": 600882, + "640684_tedx-talk-released": 612303, + "640730_e-mental": 615375, + "641143_amiga-1200-bespoke-virgin-cinema": 623542, + "641161_dreamscape-432-omega": 618894, + "641162_2019-topstone-carbon-force-etap-axs-bike": 639107, + "641186_arin-sings-big-floppy-penis-live-jazz-2": 638904, + "641421_edward-snowden-on-bitcoin-and-privacy": 522729, + "641421_what-is-libra-facebook-s-new": 598236, + "641421_what-are-stablecoins-counter-party-risk": 583508, + "641421_anthony-pomp-pompliano-discusses-crypto": 564416, + "641421_tim-draper-crypto-invest-summit-2019": 550329, + "641421_mass-adoption-and-what-will-it-take-to": 549781, + "641421_dragonwolftech-youtube-channel-trailer": 567128, + "641421_naomi-brockwell-s-weekly-crypto-recap": 540006, + "641421_blockchain-based-youtube-twitter": 580809, + "641421_andreas-antonopoulos-on-privacy-privacy": 533522, + "641817_mexico-submits-and-big-tech-worsens": 582977, + "641817_why-we-need-travel-bans": 581354, + "641880_censored-by-patreon-bitchute-shares": 482460, + "641880_crypto-wonderland": 485218, + "642168_1-diabolo-julio-cezar-16-cbmcp-freestyle": 374999, + "642314_tough-students": 615780, + "642697_gamercauldronep2": 642153, + "643406_the-most-fun-i-ve-had-in-a-long-time": 616506, + "643893_spitshine69-and-uk-freedom-audits": 616876, + "644480_my-mum-getting-attacked-a-duck": 567624, + "644486_the-cryptocurrency-experiment": 569189, + "644486_tag-you-re-it": 558316, + "644486_orange-county-mineral-society-rock-and": 397138, + "644486_sampling-with-the-gold-rush-nugget": 527960, + "644562_september-15-21-a-new-way-of-doing": 634792, + "644562_july-week-3-collective-frequency-general": 607942, + "644562_september-8-14-growing-up-general": 630977, + "644562_august-4-10-collective-frequency-general": 612307, + "644562_august-11-17-collective-frequency": 617279, + "644562_september-1-7-gentle-wake-up-call": 627104, + "644607_no-more-lol": 643497, + "644607_minion-masters-who-knew": 641313, + "645236_danganronpa-3-the-end-of-hope-s-peak": 644153, + "645348_captchabot-a-discord-bot-to-protect-your": 592810, + "645701_the-xero-hour-saint-greta-of-thunberg": 644081, + "645701_batman-v-superman-theological-notions": 590189, + "645918_emacs-is-great-ep-0-init-el-from-org": 575666, + "645918_emacs-is-great-ep-1-packages": 575666, + "645918_emacs-is-great-ep-40-pt-2-hebrew": 575668, + "645923_nasal-snuff-review-osp-batch-2": 575658, + "645923_why-bit-coin": 575658, + "645929_begin-quest": 598822, + "645929_filthy-foe": 588386, + "645929_unsanitary-snow": 588386, + "645929_famispam-1-music-box": 588386, + "645929_running-away": 598822, + "645931_my-beloved-chris-madsen": 589114, + "645931_space-is-consciousness-chris-madsen": 589116, + "645947_gasifier-rocket-stove-secondary-burn": 590595, + "645949_mouse-razer-abyssus-v2-e-mousepad": 591139, + "645949_pr-temporada-2018-league-of-legends": 591138, + "645949_windows-10-build-9901-pt-br": 591137, + "645949_abrindo-pacotes-do-festival-lunar-2018": 591139, + "645949_unboxing-camisetas-personalizadas-play-e": 591138, + "645949_abrindo-envelopes-do-festival-lunar-2017": 591138, + "645951_grub-my-grub-played-guruku-tersayang": 618033, + "645951_ismeeltimepiece": 618038, + "645951_thoughts-on-doom": 596485, + "645951_thoughts-on-god-of-war-about-as-deep-as": 596485, + "645956_linux-lite-3-6-see-what-s-new": 645195, + "646191_kahlil-gibran-the-prophet-part-1": 597637, + "646551_crypto-market-crash-should-you-sell-your": 442613, + "646551_live-crypto-trading-and-market-analysis": 442615, + "646551_5-reasons-trading-is-always-better-than": 500850, + "646551_digitex-futures-dump-panic-selling-or": 568065, + "646552_how-to-install-polarr-on-kali-linux-bynp": 466235, + "646586_electoral-college-kids-civics-lesson": 430818, + "646602_grapes-full-90-minute-watercolour": 537108, + "646602_meizu-mx4-the-second-ubuntu-phone": 537109, + "646609_how-to-set-up-the-ledger-nano-x": 569992, + "646609_how-to-buy-ethereum": 482354, + "646609_how-to-install-setup-the-exodus-multi": 482356, + "646609_how-to-manage-your-passwords-using": 531987, + "646609_cryptodad-s-live-q-a-friday-may-3rd-2019": 562303, + "646638_resident-evil-ada-chapter-5-final": 605612, + "646639_taurus-june-2019-career-love-tarot": 586910, + "646652_digital-bullpen-ep-5-building-a-digital": 589274, + "646661_sunlight": 591076, + "646661_grasp-lab-nasa-open-mct-series": 589414, + "646663_bunnula-s-creepers-tim-pool-s-beanie-a": 599669, + "646663_bunnula-music-hey-ya-by-outkast": 605685, + "646663_bunnula-tv-s-music-television-eunoia": 644437, + "646663_the-pussy-centipede-40-sneakers-and": 587265, + "646663_bunnula-reacts-ashton-titty-whitty": 596988, + "646677_filip-reviews-jeromes-dream-cataracts-so": 589751, + "646691_fascism-and-its-mobilizing-passions": 464342, + "646692_hsb-color-layers-action-for-adobe": 586533, + "646692_master-colorist-action-pack-extracting": 631830, + "646693_how-to-protect-your-garden-from-animals": 588476, + "646693_gardening-for-the-apocalypse-epic": 588472, + "646693_my-first-bee-hive-foundationless-natural": 588469, + "646693_dragon-fruit-and-passion-fruit-planting": 588470, + "646693_installing-my-first-foundationless": 588469, + "646705_first-naza-fpv": 590411, + "646717_first-burning-man-2019-detour-034": 630247, + "646717_why-bob-marley-was-an-idiot-test-driving": 477558, + "646717_we-are-addicted-to-gambling-ufc-207-w": 481398, + "646717_ghetto-swap-meet-selling-storage-lockers": 498291, + "646738_1-kings-chapter-7-summary-and-what-god": 586599, + "646814_brand-spanking-new-junior-high-school": 592378, + "646814_lupe-fiasco-freestyle-at-end-of-the-weak": 639535, + "646824_how-to-one-stroke-painting-doodles-mixed": 592404, + "646824_acrylic-pouring-landscape-with-a-tree": 592404, + "646824_how-to-make-a-diy-concrete-paste-planter": 595976, + "646824_how-to-make-a-rustic-sand-planter-sand": 592404, + "646833_3-day-festival-at-the-galilee-lake-and": 592842, + "646833_rainbow-circle-around-the-noon-sun-above": 592842, + "646833_energetic-self-control-demonstration": 623811, + "646833_bees-congregating": 592842, + "646856_formula-offroad-honefoss-sunday-track2": 592872, + "646862_h3video1-dc-vs-mb-1": 593237, + "646862_h3video1-iwasgoingto-load-up-gmod-but": 593237, + "646883_watch-this-game-developer-make-a-video": 592593, + "646883_how-to-write-secure-javascript": 592593, + "646883_blockchain-technology-explained-2-hour": 592593, + "646888_fl-studio-bits": 608155, + "646914_andy-s-shed-live-s03e02-the-longest": 592200, + "646914_gpo-telephone-776-phone-restoration": 592201, + "646916_toxic-studios-co-stream-pubg": 597126, + "646916_hyperlapse-of-prague-praha-from-inside": 597109, + "646933_videobits-1": 597378, + "646933_clouds-developing-daytime-8": 597378, + "646933_slechtvalk-in-watertoren-bodegraven": 597378, + "646933_timelapse-maansverduistering-16-juli": 605880, + "646933_startrails-27": 597378, + "646933_passing-clouds-daytime-3": 597378, + "646940_nerdgasm-unboxing-massive-playing-cards": 597421, + "646946_debunking-cops-volume-3-the-murder-of": 630570, + "646961_kingsong-ks16x-electric-unicycle-250km": 636725, + "646968_wild-mountain-goats-amazing-rock": 621940, + "646968_no-shelter-backcountry-camping-in": 621940, + "646968_can-i-live-in-this-through-winter-lets": 645750, + "646968_why-i-wear-a-chest-rig-backcountry-or": 621940, + "646989_marc-ivan-o-gorman-promo-producer-editor": 645656, + "647045_@moraltis": 646367, + "647045_moraltis-twitch-highlights-first-edit": 646368, + "647075_the-3-massive-tinder-convo-mistakes": 629464, + "647075_how-to-get-friend-zoned-via-text": 592298, + "647075_don-t-do-this-on-tinder": 624591, + "647322_world-of-tanks-7-kills": 609905, + "647322_the-tier-6-auto-loading-swedish-meatball": 591338, + "647416_hypnotic-soundscapes-garden-of-the": 596923, + "647416_hypnotic-soundscapes-the-cauldron-sacred": 596928, + "647416_schumann-resonance-to-theta-sweep": 596920, + "647416_conversational-indirect-hypnosis-why": 596913, + "647493_mimirs-brunnr": 590498, + "648143_live-ita-completiamo-the-evil-within-2": 646568, + "648203_why-we-love-people-that-hurt-us": 591128, + "648203_i-didn-t-like-my-baby-and-considered": 591128, + "648220_trade-talk-001-i-m-a-vlogger-now-fielder": 597303, + "648220_vise-restoration-record-no-6-vise": 597303, + "648540_amv-reign": 571863, + "648540_amv-virus": 571863, + "648588_audial-drift-(a-journey-into-sound)": 630217, + "648616_quick-zbrush-tip-transpose-master-scale": 463205, + "648616_how-to-create-3d-horns-maya-to-zbrush-2": 463205, + "648815_arduino-based-cartridge-game-handheld": 593252, + "648815_a-maze-update-3-new-game-modes-amazing": 593252, + "649209_denmark-trip": 591428, + "649209_stunning-4k-drone-footage": 591428, + "649215_how-to-create-a-channel-and-publish-a": 414908, + "649215_lbryclass-11-how-to-get-your-deposit": 632420, + "649543_spring-break-madness-at-universal": 599698, + "649921_navegador-brave-navegador-da-web-seguro": 649261, + "650191_stream-intro": 591301, + "650946_platelet-chan-fan-art": 584601, + "650946_aqua-fanart": 584601, + "650946_virginmedia-stores-password-in-plain": 619537, + "650946_running-linux-on-android-teaser": 604441, + "650946_hatsune-miku-ievan-polka": 600126, + "650946_digital-security-and-privacy-2-and-a-new": 600135, + "650993_my-editorial-comment-on-recent-youtube": 590305, + "650993_drive-7-18-2018": 590305, + "651011_old-world-put-on-realm-realms-gg": 591899, + "651011_make-your-own-soundboard-with-autohotkey": 591899, + "651011_ark-survival-https-discord-gg-ad26xa": 637680, + "651011_minecraft-featuring-seus-8-just-came-4": 596488, + "651057_found-footage-bikinis-at-the-beach-with": 593586, + "651057_found-footage-sexy-mom-a-mink-stole": 593586, + "651067_who-are-the-gentiles-gomer": 597094, + "651067_take-back-the-kingdom-ep-2-450-million": 597094, + "651067_mmxtac-implemented-footstep-sounds-and": 597094, + "651067_dynasoul-s-blender-to-unreal-animated": 597094, + "651103_calling-a-scammer-syntax-error": 612532, + "651103_quick-highlight-of-my-day": 647651, + "651103_calling-scammers-and-singing-christmas": 612531, + "651109_@livingtzm": 637322, + "651109_living-tzm-juuso-from-finland-september": 643412, + "651373_se-voc-rir-ou-sorrir-reinicie-o-v-deo": 649302, + "651476_what-is-pagan-online-polished-new-arpg": 592157, + "651476_must-have-elder-scrolls-online-addons": 592156, + "651476_who-should-play-albion-online": 592156, + "651730_person-detection-with-keras-tensorflow": 621276, + "651730_youtube-censorship-take-two": 587249, + "651730_new-red-tail-shark-and-two-silver-sharks": 587251, + "651730_around-auckland": 587250, + "651730_humanism-in-islam": 587250, + "651730_tigers-at-auckland-zoo": 587250, + "651730_gravity-demonstration": 587250, + "651730_copyright-question": 587249, + "651730_uberg33k-the-ultimate-software-developer": 599522, + "651730_chl-e-swarbrick-auckland-mayoral": 587250, + "651730_code-reviews": 587249, + "651730_raising-robots": 587251, + "651730_teaching-python": 587250, + "651730_kelly-tarlton-2016": 587250, + "652172_where-is-everything": 589491, + "652172_some-guy-and-his-camera": 617062, + "652172_practical-information-pt-1": 589491, + "652172_latent-vibrations": 589491, + "652172_maldek-compilation": 589491, + "652444_thank-you-etika-thank-you-desmond": 652121, + "652611_plants-vs-zombies-gw2-20190827183609": 624339, + "652611_wolfenstein-the-new-order-playthrough-6": 650299, + "652887_a-codeigniter-cms-open-source-download": 652737, + "652966_@pokesadventures": 632391, + "653009_flat-earth-uk-convention-is-a-bust": 585786, + "653009_flat-earth-reset-flat-earth-money-tree": 585786, + "653011_veil-of-thorns-dispirit-brutal-leech-3": 652475, + "653069_being-born-after-9-11": 632218, + "653069_8-years-on-youtube-what-it-has-done-for": 637130, + "653069_answering-questions-how-original": 521447, + "653069_talking-about-my-first-comedy-stand-up": 583450, + "653069_doing-push-ups-in-public": 650920, + "653069_vlog-extra": 465997, + "653069_crying-myself": 465997, + "653069_xbox-rejection": 465992, + "653354_msps-how-to-find-a-linux-job-where-no": 642537, + "653354_windows-is-better-than-linux-vlog-it-and": 646306, + "653354_luke-smith-is-wrong-about-everything": 507717, + "653354_advice-for-those-starting-out-in-tech": 612452, + "653354_treating-yourself-to-make-studying-more": 623561, + "653354_lpi-linux-essential-dns-tools-vlog-what": 559464, + "653354_is-learning-linux-worth-it-in-2019-vlog": 570886, + "653354_huawei-linux-and-cellphones-in-2019-vlog": 578501, + "653354_how-to-use-webmin-to-manage-linux": 511507, + "653354_latency-concurrency-and-the-best-value": 596857, + "653354_how-to-use-the-pomodoro-method-in-it": 506632, + "653354_negotiating-compensation-vlog-it-and": 542317, + "653354_procedural-goals-vs-outcome-goals-vlog": 626785, + "653354_intro-to-raid-understanding-how-raid": 529341, + "653354_smokeping": 574693, + "653354_richard-stallman-should-not-be-fired": 634928, + "653354_unusual-or-specialty-certifications-vlog": 620146, + "653354_gratitude-and-small-projects-vlog-it": 564900, + "653354_why-linux-on-the-smartphone-is-important": 649543, + "653354_opportunity-costs-vlog-it-devops-career": 549708, + "653354_double-giveaway-lpi-class-dates-and": 608129, + "653354_linux-on-the-smartphone-in-2019-librem": 530426, + "653524_celtic-folk-music-full-live-concert-mps": 589762, + "653745_aftermath-of-the-mac": 592768, + "653745_b-c-a-glock-17-threaded-barrel": 592770, + "653800_middle-earth-shadow-of-mordor-by": 590229, + "654079_tomand-jeremy-chirs45": 614296, + "654096_achamos-carteira-com-grana-olha-o-que": 466262, + "654096_viagem-bizarra-e-cansativa-ao-nordeste": 466263, + "654096_tedio-na-tailandia-limpeza-de-area": 466265, + "654425_schau-bung-2014-in-windischgarsten": 654410, + "654425_mitternachtseinlage-ball-rk": 654410, + "654425_zugabe-ball-rk-windischgarsten": 654412, + "654722_skytrain-in-korea": 463145, + "654722_luwak-coffee-the-shit-coffee": 463155, + "654722_puppet-show-in-bangkok-thailand": 462812, + "654722_kyaito-market-myanmar": 462813, + "654724_wipeout-zombies-bo3-custom-zombies-1st": 589569, + "654724_the-street-bo3-custom-zombies": 589544, + "654880_wwii-airsoft-pow": 586968, + "654880_dueling-geese-fight-to-the-death": 586968, + "654880_wwii-airsoft-torgau-raw-footage-part4": 586968, + "655173_april-2019-q-and-a": 554032, + "655173_the-meaning-and-reality-of-individual": 607892, + "655173_steven-pinker-progress-despite": 616984, + "655173_we-make-stories-out-of-totem-poles": 549090, + "655173_jamil-jivani-author-of-why-young-men": 542035, + "655173_commentaries-on-jb-peterson-rebel-wisdom": 528898, + "655173_auckland-clip-4-on-cain-and-abel": 629242, + "655173_peterson-vs-zizek-livestream-tickets": 545285, + "655173_auckland-clip-3-the-dawning-of-the-moral": 621154, + "655173_religious-belief-and-the-enlightenment": 606269, + "655173_auckland-lc-highlight-1-the-presumption": 565783, + "655173_q-a-sir-roger-scruton-dr-jordan-b": 544184, + "655173_cancellation-polish-national-foundation": 562529, + "655173_the-coddling-of-the-american-mind-haidt": 440185, + "655173_02-harris-weinstein-peterson-discussion": 430896, + "655173_jordan-peterson-threatens-everything-of": 519737, + "655173_on-claiming-belief-in-god-commentary": 581738, + "655173_how-to-make-the-world-better-really-with": 482317, + "655173_quillette-discussion-with-founder-editor": 413749, + "655173_jb-peterson-on-free-thought-and-speech": 462849, + "655173_marxism-zizek-peterson-official-video": 578453, + "655173_patreon-problem-solution-dave-rubin-dr": 490394, + "655173_next-week-st-louis-salt-lake-city": 445933, + "655173_conversations-with-john-anderson-jordan": 529981, + "655173_nz-australia-12-rules-tour-next-2-weeks": 518649, + "655173_a-call-to-rebellion-for-ontario-legal": 285451, + "655173_2016-personality-lecture-12": 578465, + "655173_on-the-vital-necessity-of-free-speech": 427404, + "655173_2017-01-23-social-justice-freedom-of": 578465, + "655173_discussion-sam-harris-the-idw-and-the": 423332, + "655173_march-2018-patreon-q-a": 413749, + "655173_take-aim-even-badly": 490395, + "655173_jp-f-wwbgo6a2w": 539940, + "655173_patreon-account-deletion": 503477, + "655173_canada-us-europe-tour-august-dec-2018": 413749, + "655173_leaders-myth-reality-general-stanley": 514333, + "655173_jp-ifi5kkxig3s": 539940, + "655173_documentary-a-glitch-in-the-matrix-david": 413749, + "655173_2017-08-14-patreon-q-and-a": 285451, + "655173_postmodernism-history-and-diagnosis": 285451, + "655173_23-minutes-from-maps-of-meaning-the": 413749, + "655173_milo-forbidden-conversation": 578493, + "655173_jp-wnjbasba-qw": 539940, + "655173_uk-12-rules-tour-october-and-november": 462849, + "655173_2015-maps-of-meaning-10-culture-anomaly": 578465, + "655173_ayaan-hirsi-ali-islam-mecca-vs-medina": 285452, + "655173_jp-f9393el2z1i": 539940, + "655173_campus-indoctrination-the-parasitization": 285453, + "655173_jp-owgc63khcl8": 539940, + "655173_the-death-and-resurrection-of-christ-a": 413749, + "655173_01-harris-weinstein-peterson-discussion": 430896, + "655173_enlightenment-now-steven-pinker-jb": 413749, + "655173_the-lindsay-shepherd-affair-update": 413749, + "655173_jp-g3fwumq5k8i": 539940, + "655173_jp-evvs3l-abv4": 539940, + "655173_former-australian-deputy-pm-john": 413750, + "655173_message-to-my-korean-readers-90-seconds": 477424, + "655173_jp--0xbomwjkgm": 539940, + "655173_ben-shapiro-jordan-peterson-and-a-12": 413749, + "655173_jp-91jwsb7zyhw": 539940, + "655173_deconstruction-the-lindsay-shepherd": 299272, + "655173_september-patreon-q-a": 285451, + "655173_jp-2c3m0tt5kce": 539940, + "655173_australia-s-john-anderson-dr-jordan-b": 413749, + "655173_jp-hdrlq7dpiws": 539940, + "655173_stephen-hicks-postmodernism-reprise": 578480, + "655173_october-patreon-q-a": 285451, + "655173_an-animated-intro-to-truth-order-and": 413749, + "655173_jp-bsh37-x5rny": 539940, + "655173_january-2019-q-a": 503477, + "655173_comedians-canaries-and-coalmines": 498586, + "655173_the-democrats-apology-and-promise": 465433, + "655173_jp-s4c-jodptn8": 539940, + "655173_2014-personality-lecture-16-extraversion": 578465, + "655173_dr-jordan-b-peterson-on-femsplainers": 490395, + "655173_higher-ed-our-cultural-inflection-point": 527291, + "655173_archetype-reality-friendship-and": 519736, + "655173_sir-roger-scruton-dr-jordan-b-peterson": 490395, + "655173_jp-cf2nqmqifxc": 539940, + "655173_penguin-uk-12-rules-for-life": 413749, + "655173_march-2019-q-and-a": 537138, + "655173_jp-ne5vbomsqjc": 539940, + "655173_dublin-london-harris-murray-new-usa-12": 413749, + "655173_12-rules-12-cities-tickets-now-available": 413749, + "655173_jp-j9j-bvdrgdi": 539940, + "655173_responsibility-conscience-and-meaning": 499123, + "655173_04-harris-murray-peterson-discussion": 436678, + "655173_jp-ayhaz9k008q": 539940, + "655173_with-jocko-willink-the-catastrophe-of": 490395, + "655173_interview-with-the-grievance-studies": 501296, + "655173_russell-brand-jordan-b-peterson-under": 413750, + "655173_goodbye-to-patreon": 496771, + "655173_revamped-podcast-announcement-with": 540943, + "655173_swedes-want-to-know": 285453, + "655173_auckland-clip-2-the-four-fundamental": 607892, + "655173_jp-dtirzqmgbdm": 539940, + "655173_political-correctness-a-force-for-good-a": 413750, + "655173_sean-plunket-full-interview-new-zealand": 597638, + "655173_q-a-the-meaning-and-reality-of": 616984, + "655173_lecture-and-q-a-with-jordan-peterson-the": 413749, + "655173_2017-personality-07-carl-jung-and-the": 578465, + "655173_nina-paley-animator-extraordinaire": 413750, + "655173_truth-as-the-antidote-to-suffering-with": 455127, + "655173_bishop-barron-word-on-fire": 599814, + "655173_zizek-vs-peterson-april-19": 527291, + "655173_revamped-podcast-with-westwood-one": 540943, + "655173_2016-11-19-university-of-toronto-free": 578465, + "655173_jp-1emrmtrj5jc": 539940, + "655173_who-is-joe-rogan-with-jordan-peterson": 585578, + "655173_who-dares-say-he-believes-in-god": 581738, + "655252_games-with-live2d": 589978, + "655252_kaenbyou-rin-live2d": 589978, + "655374_steam-groups-are-crazy": 607590, + "655379_asmr-captain-falcon-happily-beats-you-up": 644574, + "655379_pixel-art-series-5-link-holding-the": 442952, + "655379_who-can-cross-the-planck-length-the-hero": 610830, + "655379_ssbb-the-yoshi-grab-release-crash": 609747, + "655379_tas-captain-falcon-s-bizarre-adventure": 442958, + "655379_super-smash-bros-in-360-test": 442963, + "655379_what-if-luigi-was-b-u-f-f": 442971, + "655803_sun-time-lapse-test-7": 610634, + "655952_upper-build-complete": 591728, + "656758_cryptocurrency-awareness-adoption-the": 541770, + "656829_3d-printing-technologies-comparison": 462685, + "656829_3d-printing-for-everyone": 462685, + "657052_tni-punya-ilmu-kanuragan-gaya-baru": 657045, + "657052_papa-sunimah-nelpon-sri-utami-emon": 657045, + "657274_rapforlife-4-win": 656856, + "657274_bizzilion-proof-of-withdrawal": 656856, + "657420_quick-drawing-prince-tribute-colored": 605630, + "657453_white-boy-tom-mcdonald-facts": 597169, + "657453_is-it-ok-to-look-when-you-with-your-girl": 610508, + "657584_need-for-speed-ryzen-5-1600-gtx-1050-ti": 657161, + "657584_quantum-break-ryzen-5-1600-gtx-1050-ti-4": 657161, + "657584_nightcore-legends-never-die": 657161, + "657706_mtb-enduro-ferragosto-2019-sestri": 638904, + "657706_warface-free-for-all": 638908, + "657782_nick-warren-at-loveland-but-not-really": 444299, + "658098_le-temps-nous-glisse-entre-les-doigts": 600099, + } +} diff --git a/claimtrie/temporal/repo.go b/claimtrie/temporal/repo.go new file mode 100644 index 00000000..6b2df037 --- /dev/null +++ b/claimtrie/temporal/repo.go @@ -0,0 +1,9 @@ +package temporal + +// Repo defines APIs for Temporal to access persistence layer. +type Repo interface { + SetNodesAt(names [][]byte, heights []int32) error + NodesAt(height int32) ([][]byte, error) + Close() error + Flush() error +} diff --git a/claimtrie/temporal/temporalrepo/memory.go b/claimtrie/temporal/temporalrepo/memory.go new file mode 100644 index 00000000..0c1c8591 --- /dev/null +++ b/claimtrie/temporal/temporalrepo/memory.go @@ -0,0 +1,45 @@ +package temporalrepo + +type Memory struct { + cache map[int32]map[string]bool +} + +func NewMemory() *Memory { + return &Memory{ + cache: map[int32]map[string]bool{}, + } +} + +func (repo *Memory) SetNodesAt(names [][]byte, heights []int32) error { + + for i, height := range heights { + c, ok := repo.cache[height] + if !ok { + c = map[string]bool{} + repo.cache[height] = c + } + name := string(names[i]) + c[name] = true + } + + return nil +} + +func (repo *Memory) NodesAt(height int32) ([][]byte, error) { + + var names [][]byte + + for name := range repo.cache[height] { + names = append(names, []byte(name)) + } + + return names, nil +} + +func (repo *Memory) Close() error { + return nil +} + +func (repo *Memory) Flush() error { + return nil +} diff --git a/claimtrie/temporal/temporalrepo/pebble.go b/claimtrie/temporal/temporalrepo/pebble.go new file mode 100644 index 00000000..f7b083dc --- /dev/null +++ b/claimtrie/temporal/temporalrepo/pebble.go @@ -0,0 +1,87 @@ +package temporalrepo + +import ( + "bytes" + "encoding/binary" + + "github.com/pkg/errors" + + "github.com/cockroachdb/pebble" +) + +type Pebble struct { + db *pebble.DB +} + +func NewPebble(path string) (*Pebble, error) { + + db, err := pebble.Open(path, &pebble.Options{Cache: pebble.NewCache(16 << 20), MaxOpenFiles: 2000}) + repo := &Pebble{db: db} + + return repo, errors.Wrapf(err, "unable to open %s", path) +} + +func (repo *Pebble) SetNodesAt(name [][]byte, heights []int32) error { + + // key format: height(4B) + 0(1B) + name(varable length) + key := bytes.NewBuffer(nil) + batch := repo.db.NewBatch() + defer batch.Close() + for i, name := range name { + key.Reset() + binary.Write(key, binary.BigEndian, heights[i]) + binary.Write(key, binary.BigEndian, byte(0)) + key.Write(name) + + err := batch.Set(key.Bytes(), nil, pebble.NoSync) + if err != nil { + return errors.Wrap(err, "in set") + } + } + return errors.Wrap(batch.Commit(pebble.NoSync), "in commit") +} + +func (repo *Pebble) NodesAt(height int32) ([][]byte, error) { + + prefix := bytes.NewBuffer(nil) + binary.Write(prefix, binary.BigEndian, height) + binary.Write(prefix, binary.BigEndian, byte(0)) + + end := bytes.NewBuffer(nil) + binary.Write(end, binary.BigEndian, height) + binary.Write(end, binary.BigEndian, byte(1)) + + prefixIterOptions := &pebble.IterOptions{ + LowerBound: prefix.Bytes(), + UpperBound: end.Bytes(), + } + + var names [][]byte + + iter := repo.db.NewIter(prefixIterOptions) + for iter.First(); iter.Valid(); iter.Next() { + // Skipping the first 5 bytes (height and a null byte), we get the name. + name := make([]byte, len(iter.Key())-5) + copy(name, iter.Key()[5:]) // iter.Key() reuses its buffer + names = append(names, name) + } + + return names, errors.Wrap(iter.Close(), "in close") +} + +func (repo *Pebble) Close() error { + + err := repo.db.Flush() + if err != nil { + // if we fail to close are we going to try again later? + return errors.Wrap(err, "on flush") + } + + err = repo.db.Close() + return errors.Wrap(err, "on close") +} + +func (repo *Pebble) Flush() error { + _, err := repo.db.AsyncFlush() + return err +} diff --git a/claimtrie/temporal/temporalrepo/temporalrepo_test.go b/claimtrie/temporal/temporalrepo/temporalrepo_test.go new file mode 100644 index 00000000..090dc187 --- /dev/null +++ b/claimtrie/temporal/temporalrepo/temporalrepo_test.go @@ -0,0 +1,80 @@ +package temporalrepo + +import ( + "testing" + + "github.com/lbryio/lbcd/claimtrie/temporal" + + "github.com/stretchr/testify/require" +) + +func TestMemory(t *testing.T) { + + repo := NewMemory() + testTemporalRepo(t, repo) +} + +func TestPebble(t *testing.T) { + + repo, err := NewPebble(t.TempDir()) + require.NoError(t, err) + + testTemporalRepo(t, repo) +} + +func testTemporalRepo(t *testing.T, repo temporal.Repo) { + + r := require.New(t) + + nameA := []byte("a") + nameB := []byte("b") + nameC := []byte("c") + + testcases := []struct { + name []byte + heights []int32 + }{ + {nameA, []int32{1, 3, 2}}, + {nameA, []int32{2, 3}}, + {nameB, []int32{5, 4}}, + {nameB, []int32{5, 1}}, + {nameC, []int32{4, 3, 8}}, + } + + for _, i := range testcases { + names := make([][]byte, 0, len(i.heights)) + for range i.heights { + names = append(names, i.name) + } + err := repo.SetNodesAt(names, i.heights) + r.NoError(err) + } + + // a: 1, 2, 3 + // b: 1, 5, 4 + // c: 4, 3, 8 + + names, err := repo.NodesAt(2) + r.NoError(err) + r.ElementsMatch([][]byte{nameA}, names) + + names, err = repo.NodesAt(5) + r.NoError(err) + r.ElementsMatch([][]byte{nameB}, names) + + names, err = repo.NodesAt(8) + r.NoError(err) + r.ElementsMatch([][]byte{nameC}, names) + + names, err = repo.NodesAt(1) + r.NoError(err) + r.ElementsMatch([][]byte{nameA, nameB}, names) + + names, err = repo.NodesAt(4) + r.NoError(err) + r.ElementsMatch([][]byte{nameB, nameC}, names) + + names, err = repo.NodesAt(3) + r.NoError(err) + r.ElementsMatch([][]byte{nameA, nameC}, names) +}