Fix disconnect block info status

Signed-off-by: Anthony Fieroni <bvbfan@abv.bg>

.github/ISSUE_TEMPLATE.md

@@ -1,8 +1,8 @@
-<!-- This issue tracker is only for technical issues related to lbrycrd (the LBRY blockchain).
+<!-- This issue tracker is only for technical issues related to Bitcoin Core.
 
-General questions and/or support requests are best directed to the community chat at https://chat.lbry.org.
+General bitcoin questions and/or support requests are best directed to the Bitcoin StackExchange at https://bitcoin.stackexchange.com.
 
-For reporting security issues, please email security@lbry.com.
+For reporting security issues, please read instructions at https://bitcoincore.org/en/contact/.
 
 If the node is "stuck" during sync or giving "block checksum mismatch" errors, please ensure your hardware is stable by running memtest and observe CPU temperature with a load-test tool such as linpack before creating an issue! -->
 
@@ -13,7 +13,7 @@ If the node is "stuck" during sync or giving "block checksum mismatch" errors, p
 
 <!--- How reliably can you reproduce the issue, what are the steps to do so? -->
 
-<!-- What version of lbrycrd are you using, where did you get it (website, self-compiled, etc)? -->
+<!-- What version of Bitcoin Core are you using, where did you get it (website, self-compiled, etc)? -->
 
 <!-- What type of machine are you observing the error on (OS/CPU and disk type)? -->
 

.gitignore

@@ -121,3 +121,7 @@ contrib/devtools/split-debug.sh
 cmake-build-*/
 
 compile_commands\.json
+**/compiler*\.d
+conftest.dir/
+
+confdefs.h

.travis.yml

@@ -30,6 +30,7 @@ jobs:
         - mkdir -p dist
         - sudo zip -Xj dist/lbrycrd-${NAME}.zip src/lbrycrdd${EXT} src/lbrycrd-cli${EXT} src/lbrycrd-tx${EXT}
         - sudo zip -Xj dist/lbrycrd-${NAME}-test.zip src/test/test_lbrycrd${EXT} src/test/test_lbrycrd_fuzzy${EXT}
+        - sudo cp dist/lbrycrd-${NAME}.zip packaging/docker-for-binary/lbrycrd-${NAME}.zip
         - sha256sum dist/lbrycrd-${NAME}.zip
         - sha256sum dist/lbrycrd-${NAME}-test.zip
       deploy:
@@ -45,6 +46,13 @@ jobs:
           on:
             repo: lbryio/lbrycrd
             all_branches: true
+        - provider: script
+          script: bash packaging/docker-for-binary/auto_deploy.sh ${TRAVIS_BRANCH}
+          skip_cleanup: true
+          on:
+            repo: lbryio/lbrycrd
+            all_branches: true
+            condition: $NAME = linux
 
     - <<: *build-template
       name: windows
@@ -85,4 +93,3 @@ jobs:
       os: osx
       osx_image: xcode8.3
       env: NAME=darwin
-

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2015-2019 LBRY Inc
+Copyright (c) 2015-2020 LBRY Inc
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
 "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,

README.md

@@ -225,6 +225,10 @@ It is easy to solo mine on testnet. (It's easy on mainnet too, but much harder t
 
 We maintain a mailing list for notifications of upgrades, security issues, and soft/hard forks. To join, visit [https://lbry.com/forklist](https://lbry.com/forklist).
 
+## Mailing List
+
+We maintain a mailing list for notifications of upgrades, security issues, and soft/hard forks. To join, visit [https://lbry.com/forklist](https://lbry.com/forklist).
+
 ## License
 
 This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
@@ -232,7 +236,7 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
 ## Security
 
 We take security seriously. Please contact [security@lbry.com](mailto:security@lbry.com) regarding any security issues.
-Our PGP key is [here](https://lbry.com/faq/pgp-key) if you need it.
+Our PGP key is [here](https://keybase.io/lbry/key.asc) if you need it.
 
 ## Contact
 

build.sh

@@ -12,7 +12,7 @@ function HELP {
     echo "-q: compile the QT GUI (not working at present)"
     echo "-d: force a rebuild of dependencies"
     echo "-u: run the unit tests when done"
-    echo "-g: include debug symbols"
+    echo "-g: compile in debug mode"
     echo "-h: show help"
     exit 1
 }
@@ -56,16 +56,18 @@ done
 
 echo "Compiling with ${PARALLEL_JOBS} jobs in parallel."
 
-BUILD_FLAGS=(CXXFLAGS="-O3 -march=native")
+BUILD_FLAGS=(CXXFLAGS="-O3 -march=native -g")
+DEBUG_DEPENDS=""
 if test "$COMPILE_WITH_DEBUG" = true; then
-    BUILD_FLAGS=(--with-debug CXXFLAGS="-Og -g")
+    BUILD_FLAGS=(--with-debug CXXFLAGS="-O0 -g")
+    DEBUG_DEPENDS="DEBUG=1"
 fi
 
 cd depends
 if test "$REBUILD_DEPENDENCIES" = true; then
     make clean
 fi
-make -j${PARALLEL_JOBS} ${DO_NOT_COMPILE_THE_GUI} V=1
+make -j${PARALLEL_JOBS} ${DO_NOT_COMPILE_THE_GUI} ${DEBUG_DEPENDS} V=1
 cd ..
 
 LC_ALL=C autoreconf --install
@@ -79,4 +81,4 @@ fi
 
 if test $? -eq 0 && "$RUN_UNIT_TESTS" = true; then
     ./src/test/test_lbrycrd
-fi
+fi

configure.ac

@@ -2,10 +2,10 @@ dnl require autoconf 2.60 (AS_ECHO/AS_ECHO_N)
 AC_PREREQ([2.60])
 define(_CLIENT_VERSION_MAJOR, 0)
 define(_CLIENT_VERSION_MINOR, 17)
-define(_CLIENT_VERSION_REVISION, 3)
+define(_CLIENT_VERSION_REVISION, 4)
-define(_CLIENT_VERSION_BUILD, 3)
+define(_CLIENT_VERSION_BUILD, 6)
 define(_CLIENT_VERSION_IS_RELEASE, true)
-define(_COPYRIGHT_YEAR, 2021)
+define(_COPYRIGHT_YEAR, 2020)
 define(_COPYRIGHT_HOLDERS,[The %s developers])
 define(_COPYRIGHT_HOLDERS_SUBSTITUTION,[[LBRYcrd Core]])
 AC_INIT([LBRYcrd Core],[_CLIENT_VERSION_MAJOR._CLIENT_VERSION_MINOR._CLIENT_VERSION_REVISION],[https://github.com/lbryio/lbrycrd/issues],[lbrycrd],[https://lbry.com/])
@@ -60,8 +60,8 @@ case $host in
      lt_cv_deplibs_check_method="pass_all"
   ;;
 esac
-dnl Require C++11 compiler (no GNU extensions)
+dnl Require C++14 compiler (no GNU extensions)
-AX_CXX_COMPILE_STDCXX([11], [noext], [mandatory], [nodefault])
+AX_CXX_COMPILE_STDCXX([14], [noext], [mandatory], [nodefault])
 dnl Check if -latomic is required for <std::atomic>
 CHECK_ATOMIC
 

depends/glibc_version_header/force_link_glibc_2.19.h

@@ -3726,4 +3726,3 @@ __asm__(".symver ynf32x,ynf32x@GLIBC_WRAP_ERROR_SYMBOL_NOT_PRESENT_IN_REQUESTED_
 __asm__(".symver ynf64,ynf64@GLIBC_WRAP_ERROR_SYMBOL_NOT_PRESENT_IN_REQUESTED_VERSION");
 __asm__(".symver ynf64x,ynf64x@GLIBC_WRAP_ERROR_SYMBOL_NOT_PRESENT_IN_REQUESTED_VERSION");
 #endif
-

depends/hosts/darwin.mk

@@ -16,4 +16,3 @@ darwin_debug_CFLAGS=-Og -g
 darwin_debug_CXXFLAGS=-O0 -g
 
 darwin_native_toolchain=native_cctools
-

depends/hosts/linux.mk

@@ -1,10 +1,7 @@
 linux_CFLAGS=-pipe
 linux_CXXFLAGS=$(linux_CFLAGS) -std=c++11
 
-linux_release_CFLAGS=-O3 -g
+linux_release_CFLAGS=-O3 -g -include $(BASEDIR)/glibc_version_header/force_link_glibc_2.19.h
-ifeq (1,$(shell ldd --version | head -1 | awk '{print $$NF < 2.28}'))
-linux_release_CFLAGS+= -include $(BASEDIR)/glibc_version_header/force_link_glibc_2.19.h
-endif
 linux_release_CXXFLAGS=$(linux_release_CFLAGS)
 
 linux_debug_CFLAGS=-O1 -g

depends/hosts/mingw32.mk

@@ -8,4 +8,3 @@ mingw32_debug_CFLAGS=-O1 -g
 mingw32_debug_CXXFLAGS=-O0 -g
 
 mingw32_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC
-

depends/packages/bdb.mk

@@ -29,4 +29,3 @@ endef
 define $(package)_stage_cmds
   $(MAKE) DESTDIR=$($(package)_staging_dir) install_lib install_include
 endef
-

depends/packages/boost.mk

@@ -1,6 +1,6 @@
 package=boost
 $(package)_version=1_69_0
-$(package)_download_path=https://boostorg.jfrog.io/artifactory/main/release/1.69.0/source/
+$(package)_download_path=https://dl.bintray.com/boostorg/release/1.69.0/source/
 $(package)_file_name=$(package)_$($(package)_version).tar.bz2
 $(package)_sha256_hash=8f32d4617390d1c2d16f26a27ab60d97807b35440d45891fa340fc2648b04406
 $(package)_dependencies=icu

depends/packages/icu.mk

@@ -15,7 +15,6 @@ define $(package)_set_vars
   $(package)_archiver_darwin=$($(package)_libtool)
   $(package)_cflags_linux=-fPIC
   $(package)_cppflags_linux=-fPIC
-  $(package)_cxxflags=-std=c++11
 endef
 
 define $(package)_preprocess_cmds

depends/packages/miniupnpc.mk

@@ -1,6 +1,6 @@
 package=miniupnpc
 $(package)_version=2.0.20180203
-$(package)_download_path=https://miniupnp.tuxfamily.org/files/
+$(package)_download_path=http://miniupnp.free.fr/files
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
 $(package)_sha256_hash=90dda8c7563ca6cd4a83e23b3c66dbbea89603a1675bfdb852897c2c9cc220b7
 

packaging/Dockerfile

@@ -1,17 +1,12 @@
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 ENV LANG C.UTF-8
 
 RUN set -xe; \
     apt-get update; \
     apt-get install --no-install-recommends -y build-essential libtool autotools-dev automake pkg-config git wget apt-utils \
-        librsvg2-bin libtiff-tools cmake imagemagick libcap-dev libz-dev libbz2-dev python-setuptools xz-utils ccache g++-multilib \
+        librsvg2-bin libtiff-tools cmake imagemagick libcap-dev libz-dev libbz2-dev python-setuptools python3-setuptools xz-utils ccache g++-multilib \
-        g++-mingw-w64-i686 mingw-w64-i686-dev bsdmainutils curl ca-certificates g++-mingw-w64-x86-64 mingw-w64-x86-64-dev; \
+        g++-mingw-w64-i686 mingw-w64-i686-dev bsdmainutils curl ca-certificates g++-mingw-w64-x86-64 mingw-w64-x86-64-dev \
-    rm -rf /var/lib/apt/lists/*;
+        clang-8 lldb-8 lld-8 libc++-8-dev; \
-
-RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -; \
-    echo 'deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main' >> /etc/apt/sources.list; \
-    apt-get update; \
-    apt-get install --no-install-recommends -y clang-8 lldb-8 lld-8 libc++-8-dev; \
     rm -rf /var/lib/apt/lists/*;
 
 RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang-cpp-8 80; \
@@ -28,7 +23,7 @@ ARG VCS_REF
 ARG BUILD_DATE
 LABEL maintainer="blockchain@lbry.com" \
     decription="build_lbrycrd" \
-    version="1.1" \
+    version="1.2" \
     org.label-schema.name="build_lbrycrd" \
     org.label-schema.description="Use this to generate a reproducible build of LBRYcrd" \
     org.label-schema.build-date=$BUILD_DATE \

packaging/docker-for-binary/Dockerfile

@@ -0,0 +1,36 @@
+FROM ubuntu:18.04 as prep
+LABEL MAINTAINER="leopere [at] nixc [dot] us"
+## TODO: Implement version pinning. `apt-get install curl=<version>`
+RUN apt-get update && \
+  apt-get -y install unzip curl build-essential && \
+  apt-get autoclean -y && \
+  rm -rf /var/lib/apt/lists/*
+WORKDIR /
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+COPY ./stuff/start.sh start
+COPY ./stuff/healthcheck.sh healthcheck
+COPY ./stuff/advance_blocks.sh advance
+COPY ./stuff/fix-permissions.c fix-permissions.c
+ARG release_url
+# require that release_url is set
+RUN test -n "$release_url"
+RUN curl --progress-bar -L -o ./lbrycrd-linux.zip "$release_url" && \
+  unzip ./lbrycrd-linux.zip && \
+  gcc fix-permissions.c -o fix-permissions && \
+  chmod +x ./lbrycrdd ./lbrycrd-cli ./lbrycrd-tx ./start ./healthcheck ./fix-permissions ./advance
+
+FROM ubuntu:18.04 as app
+COPY --from=prep /lbrycrdd /lbrycrd-cli /lbrycrd-tx /start /healthcheck /fix-permissions /advance /usr/bin/
+RUN addgroup --gid 1000 lbrycrd && \
+  adduser lbrycrd --uid 1000 --gid 1000 --gecos GECOS --shell /bin/bash --disabled-password --home /data && \
+  mkdir /etc/lbry && \
+  chown lbrycrd /etc/lbry && \
+  chmod a+s /usr/bin/fix-permissions
+RUN apt-get update && apt-get -y install wget && apt-get autoclean -y && rm -rf /var/lib/apt/lists/*
+VOLUME ["/data"]
+WORKDIR /data
+HEALTHCHECK CMD /usr/bin/healthcheck
+EXPOSE 9246 9245 11337 29245
+
+USER lbrycrd
+CMD ["start"]

packaging/docker-for-binary/Dockerfile.Auto

@@ -0,0 +1,34 @@
+FROM ubuntu:18.04 as prep
+LABEL MAINTAINER="leopere [at] nixc [dot] us"
+## TODO: Implement version pinning. `apt-get install curl=<version>`
+RUN apt-get update && \
+  apt-get -y install unzip curl build-essential && \
+  apt-get autoclean -y && \
+  rm -rf /var/lib/apt/lists/*
+WORKDIR /
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+COPY ./stuff/start.sh start
+COPY ./stuff/healthcheck.sh healthcheck
+COPY ./stuff/advance_blocks.sh advance
+COPY ./stuff/fix-permissions.c fix-permissions.c
+COPY ./lbrycrd-linux.zip lbrycrd-linux.zip
+
+RUN unzip ./lbrycrd-linux.zip && \
+  gcc fix-permissions.c -o fix-permissions && \
+  chmod +x ./lbrycrdd ./lbrycrd-cli ./lbrycrd-tx ./start ./healthcheck ./fix-permissions ./advance
+
+FROM ubuntu:18.04 as app
+COPY --from=prep /lbrycrdd /lbrycrd-cli /lbrycrd-tx /start /healthcheck /fix-permissions /advance /usr/bin/
+RUN addgroup --gid 1000 lbrycrd && \
+  adduser lbrycrd --uid 1000 --gid 1000 --gecos GECOS --shell /bin/bash --disabled-password --home /data && \
+  mkdir /etc/lbry && \
+  chown lbrycrd /etc/lbry && \
+  chmod a+s /usr/bin/fix-permissions
+RUN apt-get update && apt-get -y install wget && apt-get autoclean -y && rm -rf /var/lib/apt/lists/*
+VOLUME ["/data"]
+WORKDIR /data
+HEALTHCHECK CMD /usr/bin/healthcheck
+EXPOSE 9246 9245 11337 29245
+
+USER lbrycrd
+CMD ["start"]

packaging/docker-for-binary/README.md

@@ -0,0 +1,76 @@
+# lbrycrd Docker image
+
+## Scripts
+
+There are two scripts `deploy.sh` and `auto_deploy.sh` These are used to create 
+docker images to push to lbry's Docker Hub.
+
+### `auto_deploy.sh`
+
+This script is used by the LBRYcrd's CI. When a branch or tag is built 
+it will create a docker image for it and push it to DockerHub. This should
+not be used outside of the CI environment. In addition to this, the 
+`Dockerfile.Auto` is associated with this script. This will only run on the
+Linux build job. 
+
+#### Requirements
+
+- You would need to build lbrycrd with the Docker image for reproducible 
+builds. https://hub.docker.com/repository/docker/lbry/build_lbrycrd
+- You will need DockerHub credentials to run this locally. 
+- When the script is executed you will need the parameter as
+ `./auto_deploy.sh ${tag_name}`
+
+### `deploy.sh`
+
+This can be used locally to manually create a docker image based on a 
+release. `release_url` is requested as a parameter to the script when 
+run locally. This will grab the binary from github releases inside the
+image build. 
+
+#### Requirements
+
+- You will need DockerHub credentials to run this locally. 
+
+## Configuration
+
+The lbrycrd container comes with a default configuration you can use for
+production. Extra configuration is optional.
+
+The container includes a `start` script that offers a flexible configuration
+style. It allows you to mount your own `lbrycrd.conf` file, or use environment
+variables, or a mix of both.
+
+### Environment variables
+
+The environment variables override the values in the mounted config file. If no
+mounted config file exists, these variables are used to create a fresh config.
+
+ * `PORT` - The main lbrycrd port
+ * `RPC_USER` - The rpc user
+ * `RPC_PASSWORD` - The rpc user's password
+ * `RPC_ALLOW_IP` - the subnet that is allowed rpc access
+ * `RPC_PORT` - The port to bind the rpc service to
+ * `RPC_BIND` - The ip address to bind the rpc service to
+ 
+
+### Example run commands
+
+Running the default configuration:
+
+```
+docker run --rm -it -e RUN_MODE=default -e SNAPSHOT_URL="https://lbry.com/snapshot/blockchain" lbry/lbrycrd:latest-release
+```
+
+Running with RPC password changed:
+
+```
+docker run --rm -it -e RUN_MODE=default -e RPC_PASSWORD=hunter2 lbry/lbrycrd:latest-release
+```
+
+Running with a config file but with the RPC password still overridden:
+
+```
+docker run --rm -it -v /path/to/lbrycrd.conf:/etc/lbry/lbrycrd.conf -e RUN_MODE=default -e RPC_PASSWORD=hunter2 lbry/lbrycrd:latest-release
+```
+

packaging/docker-for-binary/auto_deploy.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+
+set -euo pipefail
+hash docker 2>/dev/null || { echo >&2 'Make sure Docker is installed'; exit 1;  }
+
+set +eo pipefail
+docker version | grep -q Server
+ret=$?
+set -eo pipefail
+
+if [ $ret -ne 0 ]; then
+  echo "Cannot connect to Docker server. Is it running? Do you have the right user permissions?"
+  exit 1
+fi
+
+if [ -z "$1" ]; then
+  echo "Docker tag parameter cannot be empty"
+  exit 1
+fi
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd packaging/docker-for-binary
+docker build --tag "lbry/lbrycrd:$1" -f Dockerfile.Auto "$DIR"
+echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
+docker push "lbry/lbrycrd:$1"

packaging/docker-for-binary/compose/docker-compose.yml-prod-example

@@ -0,0 +1,26 @@
+version: '3.4'
+
+networks:
+  lbry-network:
+    external: true
+
+services:
+#############
+## Lbrycrd ##
+#############
+  lbrycrd:
+    build: .
+    restart: always
+    networks:
+      lbry-network:
+        ipv4_address: 10.6.1.2
+    environment:
+      RUN_MODE: default
+    env_file:
+     - env
+    expose:
+      - 9245
+      - 9246
+    ## host volumes for persistent data such as wallet private keys.
+    volumes:
+      - "../persist/data:/data"

packaging/docker-for-binary/compose/docker-compose.yml-regtest

@@ -0,0 +1,28 @@
+version: '3.4'
+
+networks:
+  lbry-network:
+    external: true
+
+services:
+#############
+## Lbrycrd ##
+#############
+  lbrycrd:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    restart: always
+    ports:
+      - "11336:29246"
+      - "11337:29245"
+    ## host volumes for persistent data such as wallet private keys.
+    volumes:
+      - "../persist/data:/data"
+    networks:
+      lbry-network:
+        ipv4_address: 10.6.1.2
+    environment:
+      - RUN_MODE=regtest
+      - PORT=29245
+      - AUTO_ADVANCE=1

packaging/docker-for-binary/compose/docker-compose.yml-testnet

@@ -0,0 +1,17 @@
+version: '3.4'
+
+services:
+#############
+## Lbrycrd ##
+#############
+  lbrycrd:
+    image: lbry/lbrycrd:latest-release
+    restart: always
+    ports:
+      - "11336:9246"
+      - "11337:11337"
+    ## host volumes for persistent data such as wallet private keys.
+    volumes:
+      - "../persist/data:/data"
+    environment:
+      - RUN_MODE=testnet

packaging/docker-for-binary/deploy.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -euo pipefail
+
+hash docker 2>/dev/null || { echo >&2 'Make sure Docker is installed'; exit 1;  }
+
+set +eo pipefail
+docker version | grep -q Server
+ret=$?
+set -eo pipefail
+
+if [ $ret -ne 0 ]; then
+  echo "Cannot connect to Docker server. Is it running? Do you have the right user permissions?"
+  exit 1
+fi
+
+echo "This will build the docker image using the latest lbrycrd release and push that image to Docker Hub"
+echo ""
+
+echo "What Docker tag should I use? It's the part that goes here: lbry/lbrycrd:TAG"
+
+read -p "Docker tag: " docker_tag
+if [ -z "$docker_tag" ]; then
+  echo "Docker tag cannot be empty"
+  exit 1
+fi
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+release_url=$(curl -s https://api.github.com/repos/lbryio/lbrycrd/releases | grep -F 'lbrycrd-linux' | grep download | head -n 1 | cut -d'"' -f4)
+
+docker build --build-arg "release_url=$release_url" --tag "lbry/lbrycrd:${docker_tag}" -f Dockerfile "$DIR"
+docker push "lbry/lbrycrd:${docker_tag}"

packaging/docker-for-binary/stuff/advance_blocks.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+while sleep 2; do
+    lbrycrd-cli -conf=/etc/lbry/lbrycrd.conf generate 1
+done

packaging/docker-for-binary/stuff/env-example

@@ -0,0 +1,12 @@
+COMPOSE_PROJECT_NAME=lbrycrd
+
+#############
+## Lbrycrd ##
+#############
+## TODO: The credentials are a formality but we should randomize these with magic.
+RPC_USER=${RPC_USER=lbryrpc}
+RPC_PASSWORD=${RPC_PASSWORD:-changeme}
+
+## This should be the internal container IP from which you'll be calling the RPC for Lbrycrdd from.
+## TODO: make this more dynamic before we move to scalability
+RPC_ALLOW_IP=${RPC_ALLOW_IP:-10.6.1.3}

packaging/docker-for-binary/stuff/fix-permissions.c

@@ -0,0 +1,9 @@
+#include <unistd.h>
+int main() {
+  // This program needs to run with setuid == root
+  // This needs to be in a compiled language because you cannot setuid bash scripts
+  setuid(0);
+  execle("/bin/bash", "bash", "-c",
+         "/bin/chown -R lbrycrd:lbrycrd /data && /bin/chmod -R 755 /data/",
+         (char*) NULL, (char*) NULL);
+}

packaging/docker-for-binary/stuff/healthcheck.sh

@@ -0,0 +1 @@
+/usr/bin/lbrycrd-cli -conf=/etc/lbry/lbrycrd.conf getnetworkinfo | grep -q '"networkactive": true,'

packaging/docker-for-binary/stuff/start.sh

@@ -0,0 +1,137 @@
+#!/usr/bin/env bash
+
+CONFIG_PATH=/etc/lbry/lbrycrd.conf
+
+
+function override_config_option() {
+    # Remove existing config line from a config file
+    #  and replace with environment fed value.
+    # Does nothing if the variable does not exist.
+    #  var     Name of ENV variable
+    #  option  Name of config option
+    #  config  Path of config file
+    local var=$1 option=$2 config=$3
+    if [[ -v $var ]]; then
+        # Remove the existing config option:
+        sed -i "/^$option\W*=/d" $config
+        # Add the value from the environment:
+        echo "$option=${!var}" >> $config
+    fi
+}
+
+
+function set_config() {
+  if [ -d "$CONFIG_PATH" ]; then
+      echo "$CONFIG_PATH is a directory when it should be a file."
+      exit 1
+  elif [ -f "$CONFIG_PATH" ]; then
+      echo "Merging the mounted config file with environment variables."
+      local MERGED_CONFIG=/tmp/lbrycrd_merged.conf
+      cp $CONFIG_PATH $MERGED_CONFIG
+      echo "" >> $MERGED_CONFIG
+      override_config_option PORT port $MERGED_CONFIG
+      override_config_option RPC_USER rpcuser $MERGED_CONFIG
+      override_config_option RPC_PASSWORD rpcpassword $MERGED_CONFIG
+      override_config_option RPC_ALLOW_IP rpcallowip $MERGED_CONFIG
+      override_config_option RPC_PORT rpcport $MERGED_CONFIG
+      override_config_option RPC_BIND rpcbind $MERGED_CONFIG
+      override_config_option MAX_TX_FEE maxtxfee $MERGED_CONFIG
+      override_config_option DUST_RELAY_FEE dustrelayfee $MERGED_CONFIG
+      # Make the new merged config file the new CONFIG_PATH
+      # This ensures that the original file the user mounted remains unmodified
+      CONFIG_PATH=$MERGED_CONFIG
+  else
+      echo "Creating a fresh config file from environment variables."
+      cat << EOF > $CONFIG_PATH
+server=1
+printtoconsole=1
+
+port=${PORT:-9246}
+rpcuser=${RPC_USER:-lbry}
+rpcpassword=${RPC_PASSWORD:-lbry}
+## Be careful what you set this to when running mainnet. By default it only allows RPC connections from localhost
+## if running inside a composition of services Then pass the environment variable `RPC_ALLOW_IP=0.0.0.0/0`
+rpcallowip=${RPC_ALLOW_IP:-127.0.0.1}
+rpcport=${RPC_PORT:-9245}
+rpcbind=${RPC_BIND:-"0.0.0.0"}
+
+maxtxfee=${MAX_TX_FEE:-"0.5"}
+dustrelayfee=${DUST_RELAY_FEE:-"0.00000001"}
+EOF
+  fi
+  echo "Config: "
+  cat $CONFIG_PATH
+}
+
+
+function download_snapshot() {
+  local url="${SNAPSHOT_URL:-}" #off by default. latest snapshot at https://lbry.com/snapshot/blockchain
+  if [[ -n "$url" ]] && [[ ! -d ./.lbrycrd/blocks ]]; then
+    echo "Downloading blockchain snapshot from $url"
+    wget --no-verbose -O snapshot.tar.bz2 "$url"
+    echo "Extracting snapshot..."
+    mkdir -p ./.lbrycrd
+    tar xvjf snapshot.tar.bz2 --directory ./.lbrycrd
+    rm snapshot.tar.bz2
+  fi
+}
+
+
+## Ensure perms are correct prior to running main binary
+/usr/bin/fix-permissions
+
+
+## You can optionally specify a run mode if you want to use lbry defined presets for compatibility.
+case $RUN_MODE in
+  default )
+    set_config
+    download_snapshot
+    exec lbrycrdd -conf=$CONFIG_PATH
+    ;;
+  ## If for some reason one is told to reindex their blockchain this run mode will launch lbrycrd with the reindex parameter.
+  ## Don't forget to change it back to default once complete ( you will need to remove the container to re-apply the run mode).
+  reindex )
+    ## Apply this RUN_MODE in the case you need to update a dataset.  NOTE: you do not need to use `RUN_MODE reindex` for more than one complete run.
+    set_config
+    exec lbrycrdd -conf=$CONFIG_PATH -reindex
+    ;;
+  ## Regtest requires ports to be the port listed below. It is hardcoded to be this port for regtest when using a config.
+  ## Only way to override it is to run lbrycrd from the commandline and set the port there.
+  regtest )
+    ## Set config params
+    ## TODO: Make this more automagic in the future.
+    mkdir -p `dirname $CONFIG_PATH`
+    echo "rpcuser=lbry" >           $CONFIG_PATH
+    echo "rpcpassword=lbry" >>      $CONFIG_PATH
+    echo "rpcport=29245" >>         $CONFIG_PATH
+    echo "rpcbind=0.0.0.0" >>       $CONFIG_PATH
+    echo "rpcallowip=0.0.0.0/0" >>  $CONFIG_PATH
+    echo "regtest=1" >>             $CONFIG_PATH
+    echo "server=1" >>              $CONFIG_PATH
+    echo "printtoconsole=1" >>      $CONFIG_PATH
+
+    [ "${AUTO_ADVANCE:-0}" == "1" ] && nohup advance &>/dev/null &
+
+    exec lbrycrdd -conf=$CONFIG_PATH $1
+    ;;
+  testnet )
+    ## Set config params
+    ## TODO: Make this more automagic in the future.
+    mkdir -p `dirname $CONFIG_PATH`
+    echo "rpcuser=lbry" >           $CONFIG_PATH
+    echo "rpcpassword=lbry" >>      $CONFIG_PATH
+    echo "rpcport=29245" >>         $CONFIG_PATH
+    echo "rpcbind=0.0.0.0" >>       $CONFIG_PATH
+    echo "rpcallowip=0.0.0.0/0" >>  $CONFIG_PATH
+    echo "testnet=1" >>             $CONFIG_PATH
+    echo "server=1" >>              $CONFIG_PATH
+    echo "printtoconsole=1" >>      $CONFIG_PATH
+
+    #nohup advance &>/dev/null &
+    exec lbrycrdd -conf=$CONFIG_PATH $1
+    ;;
+  * )
+    echo "Error, you must define a RUN_MODE environment variable."
+    echo "Available options are testnet, regtest, chainquery, default, and reindex"
+    ;;
+esac

packaging/docker-for-gcc/Dockerfile

@@ -0,0 +1,28 @@
+FROM ubuntu:18.04
+ENV LANG C.UTF-8
+
+RUN set -xe; \
+    apt-get update; \
+    apt-get install --no-install-recommends -y build-essential libtool autotools-dev automake pkg-config git wget apt-utils \
+        librsvg2-bin cmake libcap-dev libz-dev libbz2-dev python-setuptools python3-setuptools xz-utils ccache \
+        bsdmainutils curl ca-certificates; \
+    rm -rf /var/lib/apt/lists/*; \
+    /usr/sbin/update-ccache-symlinks;
+    
+ARG VCS_REF
+ARG BUILD_DATE
+LABEL maintainer="blockchain@lbry.com" \
+    decription="build_lbrycrd_gcc" \
+    version="1.2" \
+    org.label-schema.name="build_lbrycrd_gcc" \
+    org.label-schema.description="Use this to generate a reproducible build of LBRYcrd for Linux" \
+    org.label-schema.build-date=$BUILD_DATE \
+    org.label-schema.vcs-ref=$VCS_REF \
+    org.label-schema.vcs-url="https://github.com/lbryio/lbrycrd" \
+    org.label-schema.schema-version="1.0.0-rc1" \
+    org.label-schema.vendor="LBRY" \
+    org.label-schema.docker.cmd="docker build --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` -t lbry/build_lbrycrd_gcc packaging"
+
+ENV PATH "/usr/lib/ccache:$PATH"
+WORKDIR /home
+CMD ["/bin/bash"]

src/Makefile.am

@@ -19,16 +19,18 @@ else
 LIBUNIVALUE = $(UNIVALUE_LIBS)
 endif
 
-BITCOIN_INCLUDES=-I$(builddir) $(BDB_CPPFLAGS) $(ICU_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) $(CRYPTO_CFLAGS) $(ICU_CFLAGS)
+BITCOIN_INCLUDES=-I$(builddir) $(BDB_CPPFLAGS) $(ICU_CPPFLAGS) $(BOOST_CPPFLAGS) $(CRYPTO_CFLAGS) $(ICU_CFLAGS)
 
 BITCOIN_INCLUDES += -I$(srcdir)/secp256k1/include
 BITCOIN_INCLUDES += $(UNIVALUE_CFLAGS)
+BITCOIN_INCLUDES += -I$(srcdir)/claimtrie
 
 LIBBITCOIN_SERVER=libbitcoin_server.a
 LIBBITCOIN_COMMON=libbitcoin_common.a
 LIBBITCOIN_CONSENSUS=libbitcoin_consensus.a
 LIBBITCOIN_CLI=libbitcoin_cli.a
 LIBBITCOIN_UTIL=libbitcoin_util.a
+LIBCLAIMTRIE=claimtrie/libclaimtrie.a
 LIBBITCOIN_CRYPTO_BASE=crypto/libbitcoin_crypto_base.a
 LIBBITCOINQT=qt/libbitcoinqt.a
 LIBSECP256K1=secp256k1/libsecp256k1.la
@@ -63,6 +65,7 @@ $(LIBSECP256K1): $(wildcard secp256k1/src/*.h) $(wildcard secp256k1/src/*.c) $(w
 # Make is not made aware of per-object dependencies to avoid limiting building parallelization
 # But to build the less dependent modules first, we manually select their order here:
 EXTRA_LIBRARIES += \
+  $(LIBCLAIMTRIE) \
   $(LIBBITCOIN_CRYPTO) \
   $(LIBBITCOIN_UTIL) \
   $(LIBBITCOIN_COMMON) \
@@ -103,7 +106,7 @@ BITCOIN_CORE_H = \
   checkpoints.h \
   checkqueue.h \
   claimscriptop.h \
-  claimtrie.h \
+  claimtrie_serial.h \
   clientversion.h \
   coins.h \
   compat.h \
@@ -119,8 +122,6 @@ BITCOIN_CORE_H = \
   fs.h \
   httprpc.h \
   httpserver.h \
-  index/base.h \
-  index/txindex.h \
   indirectmap.h \
   init.h \
   interfaces/handler.h \
@@ -130,7 +131,6 @@ BITCOIN_CORE_H = \
   key_io.h \
   keystore.h \
   lbry.h \
-  dbwrapper.h \
   limitedmap.h \
   logging.h \
   memusage.h \
@@ -148,8 +148,8 @@ BITCOIN_CORE_H = \
   policy/fees.h \
   policy/policy.h \
   policy/rbf.h \
+  primitives/robin_hood.h \
   pow.h \
-  prefixtrie.h \
   protocol.h \
   random.h \
   reverse_iterator.h \
@@ -230,15 +230,10 @@ libbitcoin_server_a_SOURCES = \
   chain.cpp \
   checkpoints.cpp \
   claimscriptop.cpp \
-  claimtrie.cpp \
-  claimtrieforks.cpp \
   consensus/tx_verify.cpp \
   httprpc.cpp \
   httpserver.cpp \
-  index/base.cpp \
-  index/txindex.cpp \
   init.cpp \
-  dbwrapper.cpp \
   lbry.cpp \
   merkleblock.cpp \
   miner.cpp \
@@ -251,7 +246,6 @@ libbitcoin_server_a_SOURCES = \
   policy/policy.cpp \
   policy/rbf.cpp \
   pow.cpp \
-  prefixtrie.cpp \
   rest.cpp \
   rpc/blockchain.cpp \
   rpc/claimtrie.cpp \
@@ -413,6 +407,21 @@ libbitcoin_common_a_SOURCES = \
   warnings.cpp \
   $(BITCOIN_CORE_H)
 
+# claimtrie: shared between all executables.
+claimtrie_libclaimtrie_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES)
+claimtrie_libclaimtrie_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
+claimtrie_libclaimtrie_a_CFLAGS = $(PIE_FLAGS)
+claimtrie_libclaimtrie_a_SOURCES = \
+  claimtrie/blob.cpp \
+  claimtrie/data.cpp \
+  claimtrie/forks.cpp \
+  claimtrie/hashes.cpp \
+  claimtrie/log.cpp \
+  claimtrie/sqlite/sqlite3.c \
+  claimtrie/trie.cpp \
+  claimtrie/txoutpoint.cpp \
+  claimtrie/uints.cpp
+
 # util: shared between all executables.
 # This library *must* be included to make sure that the glibc
 # backward-compatibility objects and their sanity checks are linked.
@@ -475,12 +484,10 @@ lbrycrdd_LDADD = \
   $(LIBBITCOIN_ZMQ) \
   $(LIBBITCOIN_CONSENSUS) \
   $(LIBBITCOIN_CRYPTO) \
-  $(LIBLEVELDB) \
-  $(LIBLEVELDB_SSE42) \
   $(LIBMEMENV) \
   $(LIBSECP256K1)
 
-lbrycrdd_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(ICU_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS)
+lbrycrdd_LDADD += $(BOOST_LIBS) $(LIBCLAIMTRIE) $(BOOST_LOCALE_LIB) $(BDB_LIBS) $(CRYPTO_LIBS) $(ICU_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS)
 
 # lbrycrd-cli binary #
 lbrycrd_cli_SOURCES = bitcoin-cli.cpp
@@ -498,7 +505,7 @@ lbrycrd_cli_LDADD = \
   $(LIBBITCOIN_UTIL) \
   $(LIBBITCOIN_CRYPTO)
 
-lbrycrd_cli_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) $(ICU_LIBS) $(EVENT_LIBS)
+lbrycrd_cli_LDADD += $(BOOST_LIBS) $(LIBCLAIMTRIE) $(BOOST_LOCALE_LIB) $(CRYPTO_LIBS) $(ICU_LIBS) $(EVENT_LIBS)
 #
 
 # bitcoin-tx binary #
@@ -519,7 +526,7 @@ lbrycrd_tx_LDADD = \
   $(LIBBITCOIN_CRYPTO) \
   $(LIBSECP256K1)
 
-lbrycrd_tx_LDADD += $(BOOST_LIBS) $(ICU_LIBS) $(CRYPTO_LIBS)
+lbrycrd_tx_LDADD += $(BOOST_LIBS) $(LIBCLAIMTRIE) $(BOOST_LOCALE_LIB) $(ICU_LIBS) $(CRYPTO_LIBS)
 #
 
 # bitcoinconsensus library #
@@ -533,7 +540,7 @@ endif
 
 libbitcoinconsensus_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS)
 libbitcoinconsensus_la_LIBADD = $(LIBSECP256K1)
-libbitcoinconsensus_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL
+libbitcoinconsensus_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(srcdir)/claimtrie -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL
 libbitcoinconsensus_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
 
 endif
@@ -574,7 +581,6 @@ $(top_srcdir)/$(subdir)/config/bitcoin-config.h.in:  $(am__configure_deps)
 clean-local:
 	-$(MAKE) -C secp256k1 clean
 	-$(MAKE) -C univalue clean
-	-rm -f leveldb/*/*.gcda leveldb/*/*.gcno leveldb/helpers/memenv/*.gcda leveldb/helpers/memenv/*.gcno
 	-rm -f config.h
 	-rm -rf test/__pycache__
 
@@ -599,10 +605,6 @@ endif
 	@test -f $(PROTOC)
 	$(AM_V_GEN) $(PROTOC) --cpp_out=$(@D) --proto_path=$(<D) $<
 
-if EMBEDDED_LEVELDB
-include Makefile.leveldb.include
-endif
-
 if ENABLE_TESTS
 include Makefile.test.include
 endif

src/Makefile.bench.include

@@ -41,8 +41,6 @@ bench_bench_bitcoin_LDADD = \
   $(LIBBITCOIN_UTIL) \
   $(LIBBITCOIN_CONSENSUS) \
   $(LIBBITCOIN_CRYPTO) \
-  $(LIBLEVELDB) \
-  $(LIBLEVELDB_SSE42) \
   $(LIBMEMENV) \
   $(LIBSECP256K1) \
   $(LIBUNIVALUE)
@@ -55,7 +53,7 @@ if ENABLE_WALLET
 bench_bench_bitcoin_SOURCES += bench/coin_selection.cpp
 endif
 
-bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(ICU_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS)
+bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(LIBCLAIMTRIE) $(BOOST_LOCALE_LIB) $(BDB_LIBS) $(CRYPTO_LIBS) $(ICU_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS)
 bench_bench_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS)
 
 CLEAN_BITCOIN_BENCH = bench/*.gcda bench/*.gcno $(GENERATED_BENCH_FILES)

src/Makefile.leveldb.include

@@ -1,149 +0,0 @@
-# Copyright (c) 2016 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-LIBLEVELDB_INT = leveldb/libleveldb.a
-LIBMEMENV_INT  = leveldb/libmemenv.a
-LIBLEVELDB_SSE42_INT  = leveldb/libleveldb_sse42.a
-
-EXTRA_LIBRARIES += $(LIBLEVELDB_INT)
-EXTRA_LIBRARIES += $(LIBMEMENV_INT)
-EXTRA_LIBRARIES += $(LIBLEVELDB_SSE42_INT)
-
-LIBLEVELDB += $(LIBLEVELDB_INT)
-LIBMEMENV += $(LIBMEMENV_INT)
-LIBLEVELDB_SSE42 = $(LIBLEVELDB_SSE42_INT)
-
-LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/include
-LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/helpers/memenv
-
-LEVELDB_CPPFLAGS_INT =
-LEVELDB_CPPFLAGS_INT += -I$(srcdir)/leveldb
-LEVELDB_CPPFLAGS_INT += $(LEVELDB_TARGET_FLAGS)
-LEVELDB_CPPFLAGS_INT += -DLEVELDB_ATOMIC_PRESENT
-LEVELDB_CPPFLAGS_INT += -D__STDC_LIMIT_MACROS
-
-if TARGET_WINDOWS
-LEVELDB_CPPFLAGS_INT += -DLEVELDB_PLATFORM_WINDOWS -DWINVER=0x0500 -D__USE_MINGW_ANSI_STDIO=1
-else
-LEVELDB_CPPFLAGS_INT += -DLEVELDB_PLATFORM_POSIX
-endif
-
-leveldb_libleveldb_a_CPPFLAGS = $(AM_CPPFLAGS) $(LEVELDB_CPPFLAGS_INT) $(LEVELDB_CPPFLAGS)
-leveldb_libleveldb_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
-
-leveldb_libleveldb_a_SOURCES=
-leveldb_libleveldb_a_SOURCES += leveldb/port/atomic_pointer.h
-leveldb_libleveldb_a_SOURCES += leveldb/port/port_example.h
-leveldb_libleveldb_a_SOURCES += leveldb/port/port_posix.h
-leveldb_libleveldb_a_SOURCES += leveldb/port/win/stdint.h
-leveldb_libleveldb_a_SOURCES += leveldb/port/port.h
-leveldb_libleveldb_a_SOURCES += leveldb/port/port_win.h
-leveldb_libleveldb_a_SOURCES += leveldb/port/thread_annotations.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/db.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/options.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/comparator.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/filter_policy.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/slice.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/table_builder.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/env.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/c.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/iterator.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/cache.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/dumpfile.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/table.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/write_batch.h
-leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/status.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/log_format.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/memtable.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/version_set.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/write_batch_internal.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/filename.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/version_edit.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/dbformat.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/builder.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/log_writer.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/db_iter.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/skiplist.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/db_impl.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/table_cache.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/snapshot.h
-leveldb_libleveldb_a_SOURCES += leveldb/db/log_reader.h
-leveldb_libleveldb_a_SOURCES += leveldb/table/filter_block.h
-leveldb_libleveldb_a_SOURCES += leveldb/table/block_builder.h
-leveldb_libleveldb_a_SOURCES += leveldb/table/block.h
-leveldb_libleveldb_a_SOURCES += leveldb/table/two_level_iterator.h
-leveldb_libleveldb_a_SOURCES += leveldb/table/merger.h
-leveldb_libleveldb_a_SOURCES += leveldb/table/format.h
-leveldb_libleveldb_a_SOURCES += leveldb/table/iterator_wrapper.h
-leveldb_libleveldb_a_SOURCES += leveldb/util/crc32c.h
-leveldb_libleveldb_a_SOURCES += leveldb/util/env_posix_test_helper.h
-leveldb_libleveldb_a_SOURCES += leveldb/util/arena.h
-leveldb_libleveldb_a_SOURCES += leveldb/util/random.h
-leveldb_libleveldb_a_SOURCES += leveldb/util/posix_logger.h
-leveldb_libleveldb_a_SOURCES += leveldb/util/hash.h
-leveldb_libleveldb_a_SOURCES += leveldb/util/histogram.h
-leveldb_libleveldb_a_SOURCES += leveldb/util/coding.h
-leveldb_libleveldb_a_SOURCES += leveldb/util/testutil.h
-leveldb_libleveldb_a_SOURCES += leveldb/util/mutexlock.h
-leveldb_libleveldb_a_SOURCES += leveldb/util/logging.h
-leveldb_libleveldb_a_SOURCES += leveldb/util/testharness.h
-
-leveldb_libleveldb_a_SOURCES += leveldb/db/builder.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/c.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/dbformat.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/db_impl.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/db_iter.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/dumpfile.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/filename.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/log_reader.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/log_writer.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/memtable.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/repair.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/table_cache.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/version_edit.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/version_set.cc
-leveldb_libleveldb_a_SOURCES += leveldb/db/write_batch.cc
-leveldb_libleveldb_a_SOURCES += leveldb/table/block_builder.cc
-leveldb_libleveldb_a_SOURCES += leveldb/table/block.cc
-leveldb_libleveldb_a_SOURCES += leveldb/table/filter_block.cc
-leveldb_libleveldb_a_SOURCES += leveldb/table/format.cc
-leveldb_libleveldb_a_SOURCES += leveldb/table/iterator.cc
-leveldb_libleveldb_a_SOURCES += leveldb/table/merger.cc
-leveldb_libleveldb_a_SOURCES += leveldb/table/table_builder.cc
-leveldb_libleveldb_a_SOURCES += leveldb/table/table.cc
-leveldb_libleveldb_a_SOURCES += leveldb/table/two_level_iterator.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/arena.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/bloom.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/cache.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/coding.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/comparator.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/crc32c.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/env.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/env_posix.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/filter_policy.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/hash.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/histogram.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/logging.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/options.cc
-leveldb_libleveldb_a_SOURCES += leveldb/util/status.cc
-
-if TARGET_WINDOWS
-leveldb_libleveldb_a_SOURCES += leveldb/util/env_win.cc
-leveldb_libleveldb_a_SOURCES += leveldb/port/port_win.cc
-else
-leveldb_libleveldb_a_SOURCES += leveldb/port/port_posix.cc
-endif
-
-leveldb_libmemenv_a_CPPFLAGS = $(leveldb_libleveldb_a_CPPFLAGS)
-leveldb_libmemenv_a_CXXFLAGS = $(leveldb_libleveldb_a_CXXFLAGS)
-leveldb_libmemenv_a_SOURCES =  leveldb/helpers/memenv/memenv.cc
-leveldb_libmemenv_a_SOURCES += leveldb/helpers/memenv/memenv.h
-
-leveldb_libleveldb_sse42_a_CPPFLAGS = $(leveldb_libleveldb_a_CPPFLAGS)
-leveldb_libleveldb_sse42_a_CXXFLAGS = $(leveldb_libleveldb_a_CXXFLAGS)
-if ENABLE_HWCRC32
-leveldb_libleveldb_sse42_a_CPPFLAGS += -DLEVELDB_PLATFORM_POSIX_SSE
-leveldb_libleveldb_sse42_a_CXXFLAGS += $(SSE42_CXXFLAGS)
-endif
-leveldb_libleveldb_sse42_a_SOURCES =  leveldb/port/port_posix_sse.cc

src/Makefile.qt.include

@@ -408,8 +408,8 @@ endif
 if ENABLE_ZMQ
 qt_lbrycrd_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS)
 endif
-qt_lbrycrd_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \
+qt_lbrycrd_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBMEMENV) \
-  $(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(PROTOBUF_LIBS) $(ICU_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \
+  $(BOOST_LIBS) $(LIBCLAIMTRIE) $(BOOST_LOCALE_LIB) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(PROTOBUF_LIBS) $(ICU_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \
   $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS)
 qt_lbrycrd_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS)
 qt_lbrycrd_qt_LIBTOOLFLAGS = $(AM_LIBTOOLFLAGS) --tag CXX
@@ -443,10 +443,10 @@ CLEAN_QT = $(nodist_qt_libbitcoinqt_a_SOURCES) $(QT_QM) $(QT_FORMS_H) qt/*.gcda
 
 CLEANFILES += $(CLEAN_QT)
 
-bitcoin_qt_clean: FORCE
+lbrycrd_qt_clean: FORCE
 	rm -f $(CLEAN_QT) $(qt_libbitcoinqt_a_OBJECTS) $(qt_lbrycrd_qt_OBJECTS) qt/lbrycrd-qt$(EXEEXT) $(LIBBITCOINQT)
 
-bitcoin_qt : qt/lbrycrd-qt$(EXEEXT)
+lbrycrd_qt : qt/lbrycrd-qt$(EXEEXT)
 
 ui_%.h: %.ui
 	@test -f $(UIC)

src/Makefile.qttest.include

@@ -4,6 +4,7 @@
 
 bin_PROGRAMS += qt/test/test_lbrycrd-qt
 TESTS += qt/test/test_lbrycrd-qt
+TEST_QT_BINARY=qt/test/test_lbrycrd-qt$(EXEEXT)
 
 TEST_QT_MOC_CPP = \
   qt/test/moc_compattests.cpp \
@@ -62,8 +63,8 @@ endif
 if ENABLE_ZMQ
 qt_test_test_lbrycrd_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS)
 endif
-qt_test_test_lbrycrd_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) \
+qt_test_test_lbrycrd_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \
-  $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) $(QT_LIBS) \
+  $(LIBMEMENV) $(BOOST_LIBS) $(LIBCLAIMTRIE) $(BOOST_LOCALE_LIB) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) $(QT_LIBS) \
   $(QR_LIBS) $(PROTOBUF_LIBS) $(ICU_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \
   $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS)
 qt_test_test_lbrycrd_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS)
@@ -73,10 +74,10 @@ CLEAN_BITCOIN_QT_TEST = $(TEST_QT_MOC_CPP) qt/test/*.gcda qt/test/*.gcno
 
 CLEANFILES += $(CLEAN_BITCOIN_QT_TEST)
 
-test_lbrycrd_qt : qt/test/test_bitcoin-qt$(EXEEXT)
+test_lbrycrd_qt : $(TEST_QT_BINARY)
 
-test_lbrycrd_qt_check : qt/test/test_bitcoin-qt$(EXEEXT) FORCE
+test_lbrycrd_qt_check : $(TEST_QT_BINARY) FORCE
 	$(MAKE) check-TESTS TESTS=$^
 
 test_lbrycrd_qt_clean: FORCE
-	rm -f $(CLEAN_BITCOIN_QT_TEST) $(qt_test_test_lbrycrd_qt_OBJECTS)
+	rm -f $(CLEAN_BITCOIN_QT_TEST) $(qt_test_test_lbrycrd_qt_OBJECTS) $(TEST_QT_BINARY)

src/Makefile.test.include

@@ -56,7 +56,6 @@ BITCOIN_TESTS =\
   test/key_io_tests.cpp \
   test/key_tests.cpp \
   test/limitedmap_tests.cpp \
-  test/dbwrapper_tests.cpp \
   test/main_tests.cpp \
   test/mempool_tests.cpp \
   test/merkle_tests.cpp \
@@ -76,7 +75,6 @@ BITCOIN_TESTS =\
   test/pmt_tests.cpp \
   test/policyestimator_tests.cpp \
   test/pow_tests.cpp \
-  test/prefixtrie_tests.cpp \
   test/prevector_tests.cpp \
   test/raii_event_tests.cpp \
   test/random_tests.cpp \
@@ -96,7 +94,6 @@ BITCOIN_TESTS =\
   test/timedata_tests.cpp \
   test/torcontrol_tests.cpp \
   test/transaction_tests.cpp \
-  test/txindex_tests.cpp \
   test/txvalidation_tests.cpp \
   test/txvalidationcache_tests.cpp \
   test/uint256_tests.cpp \
@@ -127,7 +124,7 @@ test_test_lbrycrd_LDADD += $(LIBBITCOIN_WALLET)
 endif
 
 test_test_lbrycrd_LDADD += $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \
-  $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS)
+  $(LIBMEMENV) $(BOOST_LIBS) $(LIBCLAIMTRIE) $(BOOST_LOCALE_LIB) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS)
 test_test_lbrycrd_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
 
 test_test_lbrycrd_LDADD += $(LIBBITCOIN_CONSENSUS) $(BDB_LIBS) $(CRYPTO_LIBS) $(ICU_LIBS) $(MINIUPNPC_LIBS)
@@ -156,7 +153,7 @@ test_test_lbrycrd_fuzzy_LDADD = \
   $(LIBBITCOIN_CRYPTO_SHANI) \
   $(LIBSECP256K1)
 
-test_test_lbrycrd_fuzzy_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) $(ICU_LIBS)
+test_test_lbrycrd_fuzzy_LDADD += $(BOOST_LIBS) $(LIBCLAIMTRIE) $(BOOST_LOCALE_LIB) $(CRYPTO_LIBS) $(ICU_LIBS)
 
 nodist_test_test_lbrycrd_SOURCES = $(GENERATED_TEST_FILES)
 

src/addrman.cpp

@@ -11,23 +11,23 @@
 
 int CAddrInfo::GetTriedBucket(const uint256& nKey) const
 {
-    uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetKey()).GetHash().GetCheapHash();
+    auto hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetKey()).GetHash();
-    uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup() << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetHash().GetCheapHash();
+    auto hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup() << (GetCheapHash(hash1) % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetHash();
-    return hash2 % ADDRMAN_TRIED_BUCKET_COUNT;
+    return GetCheapHash(hash2) % ADDRMAN_TRIED_BUCKET_COUNT;
 }
 
 int CAddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src) const
 {
     std::vector<unsigned char> vchSourceGroupKey = src.GetGroup();
-    uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup() << vchSourceGroupKey).GetHash().GetCheapHash();
+    auto hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup() << vchSourceGroupKey).GetHash();
-    uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << vchSourceGroupKey << (hash1 % ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP)).GetHash().GetCheapHash();
+    auto hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << vchSourceGroupKey << (GetCheapHash(hash1) % ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP)).GetHash();
-    return hash2 % ADDRMAN_NEW_BUCKET_COUNT;
+    return GetCheapHash(hash2) % ADDRMAN_NEW_BUCKET_COUNT;
 }
 
 int CAddrInfo::GetBucketPosition(const uint256 &nKey, bool fNew, int nBucket) const
 {
-    uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << (fNew ? 'N' : 'K') << nBucket << GetKey()).GetHash().GetCheapHash();
+    auto hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << (fNew ? 'N' : 'K') << nBucket << GetKey()).GetHash();
-    return hash1 % ADDRMAN_BUCKET_SIZE;
+    return GetCheapHash(hash1) % ADDRMAN_BUCKET_SIZE;
 }
 
 bool CAddrInfo::IsTerrible(int64_t nNow) const

src/arith_uint256.h

@@ -56,8 +56,8 @@ public:
     {
         static_assert(BITS/32 > 0 && BITS%32 == 0, "Template parameter BITS must be a positive multiple of 32.");
 
-        pn[0] = (unsigned int)b;
+        pn[0] = uint32_t(b);
-        pn[1] = (unsigned int)(b >> 32);
+        pn[1] = uint32_t(b >> 32U);
         for (int i = 2; i < WIDTH; i++)
             pn[i] = 0;
     }

src/bench/block_assemble.cpp

@@ -13,6 +13,7 @@
 #include <scheduler.h>
 #include <txdb.h>
 #include <txmempool.h>
+#include <util.h>
 #include <utiltime.h>
 #include <validation.h>
 #include <validationinterface.h>
@@ -40,7 +41,7 @@ static CTxIn MineBlock(const CScript& coinbase_scriptPubKey)
 {
     auto block = PrepareBlock(coinbase_scriptPubKey);
 
-    while (!CheckProofOfWork(block->GetHash(), block->nBits, Params().GetConsensus())) {
+    while (!CheckProofOfWork(block->GetPoWHash(), block->nBits, Params().GetConsensus())) {
         ++block->nNonce;
         assert(block->nNonce);
     }
@@ -72,18 +73,29 @@ static void AssembleBlock(benchmark::State& state)
     boost::thread_group thread_group;
     CScheduler scheduler;
     {
+        delete ::pclaimTrie;
+        const CChainParams& chainparams = Params();
+        auto &consensus = chainparams.GetConsensus();
         ::pblocktree.reset(new CBlockTreeDB(1 << 20, true));
         ::pcoinsdbview.reset(new CCoinsViewDB(1 << 23, true));
         ::pcoinsTip.reset(new CCoinsViewCache(pcoinsdbview.get()));
+        ::pclaimTrie = new CClaimTrie(1 << 25, true, 0, GetDataDir().string(),
+                                      consensus.nNormalizedNameForkHeight,
+                                      consensus.nMinRemovalWorkaroundHeight,
+                                      consensus.nMaxRemovalWorkaroundHeight,
+                                      consensus.nOriginalClaimExpirationTime,
+                                      consensus.nExtendedClaimExpirationTime,
+                                      consensus.nExtendedClaimExpirationForkHeight,
+                                      consensus.nAllClaimsInMerkleForkHeight, 1);
 
-        const CChainParams& chainparams = Params();
         thread_group.create_thread(boost::bind(&CScheduler::serviceQueue, &scheduler));
         GetMainSignals().RegisterBackgroundSignalScheduler(scheduler);
         LoadGenesisBlock(chainparams);
         CValidationState state;
         ActivateBestChain(state, chainparams);
         assert(::chainActive.Tip() != nullptr);
-        const bool witness_enabled{IsWitnessEnabled(::chainActive.Tip(), chainparams.GetConsensus())};
+        const_cast<int&>(consensus.nWitnessForkHeight) = ::chainActive.Height();
+        const bool witness_enabled{IsWitnessEnabled(::chainActive.Tip(), consensus)};
         assert(witness_enabled);
     }
 

src/bench/checkblock.cpp

@@ -9,9 +9,50 @@
 #include <streams.h>
 #include <consensus/validation.h>
 
+
+/* Don't use raw bitcoin blocks
 namespace block_bench {
 #include <bench/data/block413567.raw.h>
 } // namespace block_bench
+*/
+
+CDataStream getTestBlockStream()
+{
+    static CBlock block;
+    static CDataStream stream(SER_NETWORK, PROTOCOL_VERSION);
+    if (block.IsNull()) {
+        block.nVersion = 5;
+        block.hashPrevBlock = uint256S("e8fc54d1e6581fceaaf64cc8afeedb9c4ba1eb2349eaf638f1fc41e331869dee");
+        block.hashMerkleRoot = uint256S("a926580973e1204aa179a4c536023c69212ce00b56dc85d3516488f3c51dd022");
+        block.hashClaimTrie = uint256S("7bae80d60b09031265f8a9f6282e4b9653764fadfac2c8b4c1f416c972b58814");
+        block.nTime = 1545050539;
+        block.nBits = 0x207fffff;
+        block.nNonce = 128913;
+        block.vtx.resize(60);
+        uint256 prevHash;
+        for (int i = 0; i < 60; ++i) {
+            CMutableTransaction tx;
+            tx.nVersion = 5;
+            tx.nLockTime = 0;
+            tx.vin.resize(1);
+            tx.vout.resize(1);
+            if (!prevHash.IsNull()) {
+                tx.vin[0].prevout.hash = prevHash;
+                tx.vin[0].prevout.n = 0;
+            }
+            tx.vin[0].scriptSig = CScript() << OP_0 << OP_0;
+            tx.vin[0].nSequence = std::numeric_limits<unsigned int>::max();
+            tx.vout[0].scriptPubKey = CScript();
+            tx.vout[0].nValue = 0;
+            prevHash = tx.GetHash();
+            block.vtx[i] = MakeTransactionRef(std::move(tx));
+        }
+        stream << block;
+        char a = '\0';
+        stream.write(&a, 1); // Prevent compaction
+    }
+    return stream;
+}
 
 // These are the two major time-sinks which happen after we have fully received
 // a block off the wire, but before we can relay the block on to peers using
@@ -19,33 +60,25 @@ namespace block_bench {
 
 static void DeserializeBlockTest(benchmark::State& state)
 {
-    CDataStream stream((const char*)block_bench::block413567,
+    auto stream = getTestBlockStream();
-            (const char*)&block_bench::block413567[sizeof(block_bench::block413567)],
+    const auto size = stream.size() - 1;
-            SER_NETWORK, PROTOCOL_VERSION);
-    char a = '\0';
-    stream.write(&a, 1); // Prevent compaction
-
     while (state.KeepRunning()) {
         CBlock block;
         stream >> block;
-        assert(stream.Rewind(sizeof(block_bench::block413567)));
+        assert(stream.Rewind(size));
     }
 }
 
 static void DeserializeAndCheckBlockTest(benchmark::State& state)
 {
-    CDataStream stream((const char*)block_bench::block413567,
+    auto stream = getTestBlockStream();
-            (const char*)&block_bench::block413567[sizeof(block_bench::block413567)],
+    const auto size = stream.size() - 1;
-            SER_NETWORK, PROTOCOL_VERSION);
+    const auto chainParams = CreateChainParams(CBaseChainParams::REGTEST);
-    char a = '\0';
-    stream.write(&a, 1); // Prevent compaction
-
-    const auto chainParams = CreateChainParams(CBaseChainParams::MAIN);
 
     while (state.KeepRunning()) {
         CBlock block; // Note that CBlock caches its checked state, so we need to recreate it here
         stream >> block;
-        assert(stream.Rewind(sizeof(block_bench::block413567)));
+        assert(stream.Rewind(size));
 
         CValidationState validationState;
         assert(CheckBlock(block, validationState, chainParams->GetConsensus()));

src/bench/merkle_root.cpp

@@ -4,11 +4,13 @@
 
 #include <bench/bench.h>
 
-#include <uint256.h>
+#include <claimtrie/hashes.h>
-#include <random.h>
 #include <consensus/merkle.h>
+#include <crypto/sha256.h>
+#include <random.h>
+#include <uint256.h>
 
-static void MerkleRoot(benchmark::State& state)
+static void MerkleRootUni(benchmark::State& state)
 {
     FastRandomContext rng(true);
     std::vector<uint256> leaves;
@@ -18,9 +20,18 @@ static void MerkleRoot(benchmark::State& state)
     }
     while (state.KeepRunning()) {
         bool mutation = false;
-        uint256 hash = ComputeMerkleRoot(std::vector<uint256>(leaves), &mutation);
+        uint256 hash = ComputeMerkleRoot(leaves, &mutation);
         leaves[mutation] = hash;
     }
 }
 
+static void MerkleRoot(benchmark::State& state)
+{
+    sha256n_way = [](std::vector<uint256>& hashes) {
+        SHA256D64(hashes[0].begin(), hashes[0].begin(), hashes.size() / 2);
+    };
+    MerkleRootUni(state);
+}
+
 BENCHMARK(MerkleRoot, 800);
+BENCHMARK(MerkleRootUni, 800);

src/chain.h

@@ -171,7 +171,7 @@ class CBlockIndex
 {
 public:
     //! pointer to the hash of the block, if any. Memory is owned by this CBlockIndex
-    const uint256* phashBlock;
+    const uint256 hash;
 
     //! pointer to the index of the predecessor of this block
     CBlockIndex* pprev;
@@ -222,7 +222,6 @@ public:
 
     void SetNull()
     {
-        phashBlock = nullptr;
         pprev = nullptr;
         pskip = nullptr;
         nHeight = 0;
@@ -244,12 +243,12 @@ public:
         nNonce         = 0;
     }
 
-    CBlockIndex()
+    CBlockIndex(const uint256& blockHash) : hash(blockHash)
     {
         SetNull();
     }
 
-    explicit CBlockIndex(const CBlockHeader& block)
+    explicit CBlockIndex(const CBlockHeader& block) : hash(block.GetHash())
     {
         SetNull();
 
@@ -293,9 +292,9 @@ public:
         return block;
     }
 
-    uint256 GetBlockHash() const
+    const uint256& GetBlockHash() const
     {
-        return *phashBlock;
+        return hash;
     }
 
     uint256 GetBlockPoWHash() const
@@ -376,73 +375,6 @@ int64_t GetBlockProofEquivalentTime(const CBlockIndex& to, const CBlockIndex& fr
 const CBlockIndex* LastCommonAncestor(const CBlockIndex* pa, const CBlockIndex* pb);
 
 
-/** Used to marshal pointers into hashes for db storage. */
-class CDiskBlockIndex : public CBlockIndex
-{
-public:
-    uint256 hashPrev;
-
-    CDiskBlockIndex() {
-        hashPrev = uint256();
-    }
-
-    explicit CDiskBlockIndex(const CBlockIndex* pindex) : CBlockIndex(*pindex) {
-        hashPrev = (pprev ? pprev->GetBlockHash() : uint256());
-    }
-
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action) {
-        int _nVersion = s.GetVersion();
-        if (!(s.GetType() & SER_GETHASH))
-            READWRITE(VARINT(_nVersion, VarIntMode::NONNEGATIVE_SIGNED));
-
-        READWRITE(VARINT(nHeight, VarIntMode::NONNEGATIVE_SIGNED));
-        READWRITE(VARINT(nStatus));
-        READWRITE(VARINT(nTx));
-        if (nStatus & (BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO))
-            READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED));
-        if (nStatus & BLOCK_HAVE_DATA)
-            READWRITE(VARINT(nDataPos));
-        if (nStatus & BLOCK_HAVE_UNDO)
-            READWRITE(VARINT(nUndoPos));
-
-        // block header
-        READWRITE(this->nVersion);
-        READWRITE(hashPrev);
-        READWRITE(hashMerkleRoot);
-        READWRITE(hashClaimTrie);
-        READWRITE(nTime);
-        READWRITE(nBits);
-        READWRITE(nNonce);
-    }
-
-    uint256 GetBlockHash() const
-    {
-        CBlockHeader block;
-        block.nVersion        = nVersion;
-        block.hashPrevBlock   = hashPrev;
-        block.hashMerkleRoot  = hashMerkleRoot;
-        block.hashClaimTrie   = hashClaimTrie;
-        block.nTime           = nTime;
-        block.nBits           = nBits;
-        block.nNonce          = nNonce;
-        return block.GetHash();
-    }
-
-    std::string ToString() const
-    {
-        std::string str = "CDiskBlockIndex(";
-        str += CBlockIndex::ToString();
-        str += strprintf("\n                hashBlock=%s, hashClaimTrie=%s, hashPrev=%s)",
-            GetBlockHash().ToString(),
-            hashClaimTrie.ToString(),
-            hashPrev.ToString());
-        return str;
-    }
-};
-
 /** An in-memory indexed chain of blocks. */
 class CChain {
 private:

src/chainparams.cpp

@@ -143,8 +143,8 @@ public:
         consensus.nAllowMinDiffMinHeight = -1;
         consensus.nAllowMinDiffMaxHeight = -1;
         consensus.nNormalizedNameForkHeight = 539940; // targeting 21 March 2019
-        consensus.nMinTakeoverWorkaroundHeight = 496850;
+        consensus.nMinRemovalWorkaroundHeight = 297706;
-        consensus.nMaxTakeoverWorkaroundHeight = 658300; // targeting 30 Oct 2019
+        consensus.nMaxRemovalWorkaroundHeight = 658300;
         consensus.nWitnessForkHeight = 680770; // targeting 11 Dec 2019
         consensus.nAllClaimsInMerkleForkHeight = 658310; // targeting 30 Oct 2019
         consensus.fPowAllowMinDifficultyBlocks = false;
@@ -166,10 +166,10 @@ public:
         consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout = 1548288000; // Jan 24, 2019
 
         // The best chain should have at least this much work.
-        consensus.nMinimumChainWork = uint256S("000000000000000000000000000000000000000000000499ed6684d1bf6f6fd3"); //946000
+        consensus.nMinimumChainWork = uint256S("0000000000000000000000000000000000000000000003253077412df5b49766"); //749k
 
         // By default assume that the signatures in ancestors of this block are valid.
-        consensus.defaultAssumeValid = uint256S("0d3b537afe49820e1c6efc555463f955251b1293c6e5130137e1e25744431172"); //946000
+        consensus.defaultAssumeValid = uint256S("b9676f45be594438a2011407c93bb530d817fa365846e7a6ecdf2790e4a0ad6b"); //749k
 
         /**
          * The message start string is designed to be unlikely to occur in normal data.
@@ -195,11 +195,9 @@ public:
         vSeeds.clear();
         vFixedSeeds.clear();
 
-        vSeeds.emplace_back("dnsseed1.lbry.io"); // LBRY Inc
+        vSeeds.emplace_back("dnsseed1.lbry.io"); // lbry.io
-        vSeeds.emplace_back("dnsseed2.lbry.io"); // LBRY Inc
+        vSeeds.emplace_back("dnsseed2.lbry.io"); // lbry.io
-        vSeeds.emplace_back("dnsseed3.lbry.io"); // LBRY Inc
+        vSeeds.emplace_back("dnsseed3.lbry.io"); // lbry.io
-        vSeeds.emplace_back("seed.lbry.grin.io"); // Grin
-        vSeeds.emplace_back("seed.allaboutlbc.com"); // Madiator2011
 
         base58Prefixes[PUBKEY_ADDRESS] = std::vector<unsigned char>(1, 0x55);
         base58Prefixes[SCRIPT_ADDRESS] = std::vector<unsigned char>(1, 0x7a);
@@ -207,6 +205,8 @@ public:
         base58Prefixes[EXT_PUBLIC_KEY] = {0x04, 0x88, 0xB2, 0x1E};
         base58Prefixes[EXT_SECRET_KEY] = {0x04, 0x88, 0xAD, 0xE4};
 
+        vFixedSeeds = std::vector<SeedSpec6>(pnSeed6_main, pnSeed6_main + ARRAYLEN(pnSeed6_main));
+
         bech32_hrp = "lbc";
 
         vFixedSeeds = std::vector<SeedSpec6>(pnSeed6_main, pnSeed6_main + ARRAYLEN(pnSeed6_main));
@@ -247,24 +247,27 @@ public:
         consensus.nMajorityRejectBlockOutdated = 75;
         consensus.nMajorityWindow = 100;
         consensus.BIP16Exception = uint256S("0x00000000dd30457c001f4095d208cc1296b0eed002427aa599874af7a432b105");
-        consensus.BIP34Height = 21111;
+        consensus.BIP34Height = 14;
-        consensus.BIP34Hash = uint256S("0x0000000023b3a96d3484e5abb3755c413e7d41500f8e2a5c3f0dd01299cd8ef8");
+        consensus.BIP34Hash = uint256S("0x079557d16edcd640c4057c9fddb81257263014fe384c4aa348c5b9d190650a46");
         // FIXME: adjust heights
-        consensus.BIP65Height = 1200000;
+        consensus.BIP65Height = 120;
-        consensus.BIP66Height = 1200000;
+        consensus.BIP66Height = 120;
         consensus.powLimit = uint256S("0000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff");
         consensus.nPowTargetTimespan = 150;
+//        consensus.CSVHeight = 0;
+//        consensus.SegwitHeight = 120;
+//        consensus.MinBIP9WarningHeight = 2136; // segwit activation height + miner confirmation window
         consensus.nPowTargetSpacing = 150;
         consensus.nOriginalClaimExpirationTime = 262974;
         consensus.nExtendedClaimExpirationTime = 2102400;
-        consensus.nExtendedClaimExpirationForkHeight = 278160;
+        consensus.nExtendedClaimExpirationForkHeight = 1;
-        consensus.nAllowMinDiffMinHeight = 277299;
+        consensus.nAllowMinDiffMinHeight = 1;
-        consensus.nAllowMinDiffMaxHeight = 1100000;
+        consensus.nAllowMinDiffMaxHeight = 2;
-        consensus.nNormalizedNameForkHeight = 993380;   // targeting, 21 Feb 2019
+        consensus.nNormalizedNameForkHeight = 1;   // targeting, 21 Feb 2019
-        consensus.nMinTakeoverWorkaroundHeight = 99;
+        consensus.nMinRemovalWorkaroundHeight = 99;
-        consensus.nMaxTakeoverWorkaroundHeight = 1198550; // targeting 30 Sep 2019
+        consensus.nMaxRemovalWorkaroundHeight = 100;
-        consensus.nWitnessForkHeight = 1198600;
+        consensus.nWitnessForkHeight = 120;
-        consensus.nAllClaimsInMerkleForkHeight = 1198560; // targeting 30 Sep 2019
+        consensus.nAllClaimsInMerkleForkHeight = 110; // targeting 30 Sep 2019
         consensus.fPowAllowMinDifficultyBlocks = true;
         consensus.fPowNoRetargeting = false;
         consensus.nRuleChangeActivationThreshold = 1512; // 75% for testchains
@@ -275,19 +278,19 @@ public:
 
         // Deployment of BIP68, BIP112, and BIP113.
         consensus.vDeployments[Consensus::DEPLOYMENT_CSV].bit = 0;
-        consensus.vDeployments[Consensus::DEPLOYMENT_CSV].nStartTime = 1456790400; // March 1st, 2016
+        consensus.vDeployments[Consensus::DEPLOYMENT_CSV].nStartTime = Consensus::BIP9Deployment::ALWAYS_ACTIVE;
-        consensus.vDeployments[Consensus::DEPLOYMENT_CSV].nTimeout = 1493596800; // May 1st, 2017
+        consensus.vDeployments[Consensus::DEPLOYMENT_CSV].nTimeout = Consensus::BIP9Deployment::NO_TIMEOUT;
 
         // Deployment of SegWit (BIP141, BIP143, and BIP147) -- Unused (see nWitnessForkHeight).
         consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].bit = 1;
         consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nStartTime = 1462060800; // May 1st 2016
-        consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout = 1493596800; // May 1st 2017
+        consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout = 1585849000; // Apr 2nd 2020
 
         // The best chain should have at least this much work.
-        consensus.nMinimumChainWork = uint256S("0x000000000000000000000000000000000000000000000000000a0c3931735170");
+        consensus.nMinimumChainWork = uint256S("00000000000000000000000000000000000000000000000000000054b7d280af"); // 8400
 
         // By default assume that the signatures in ancestors of this block are valid.
-        consensus.defaultAssumeValid = uint256S("9812b0bcb7e889e58d999c897e9eaddb2dab98122ff1cfb238ebeef5351bd48c"); // 1
+        consensus.defaultAssumeValid = uint256S("50b68b892f4e0f2ef649df37ef10b702e826c8913cc785c5e8ec16dd6be83f8b"); // 8400
 
         pchMessageStart[0] = 0xfa;
         pchMessageStart[1] = 0xe4;
@@ -308,8 +311,8 @@ public:
 
         vFixedSeeds.clear();
         vSeeds.clear();
-        vSeeds.emplace_back("testdnsseed1.lbry.io");
+        vSeeds.emplace_back("testdnsseed1.lbry.com");
-        vSeeds.emplace_back("testdnsseed2.lbry.io");
+        vSeeds.emplace_back("testdnsseed2.lbry.com");
 
         base58Prefixes[PUBKEY_ADDRESS] = std::vector<unsigned char>(1,111);
         base58Prefixes[SCRIPT_ADDRESS] = std::vector<unsigned char>(1,196);
@@ -368,8 +371,8 @@ public:
         consensus.nAllowMinDiffMinHeight = -1;
         consensus.nAllowMinDiffMaxHeight = -1;
         consensus.nNormalizedNameForkHeight = 250; // SDK depends upon this number
-        consensus.nMinTakeoverWorkaroundHeight = -1;
+        consensus.nMinRemovalWorkaroundHeight = -1;
-        consensus.nMaxTakeoverWorkaroundHeight = -1;
+        consensus.nMaxRemovalWorkaroundHeight = -1;
         consensus.nWitnessForkHeight = 150;
         consensus.nAllClaimsInMerkleForkHeight = 350;
         consensus.fPowAllowMinDifficultyBlocks = false;

src/claimscriptop.cpp

@@ -13,22 +13,30 @@ CClaimScriptAddOp::CClaimScriptAddOp(const COutPoint& point, CAmount nValue, int
 
 bool CClaimScriptAddOp::claimName(CClaimTrieCache& trieCache, const std::string& name)
 {
-    return addClaim(trieCache, name, ClaimIdHash(point.hash, point.n));
+    auto claimId = ClaimIdHash(point.hash, point.n);
+    LogPrint(BCLog::CLAIMS, "+++ Claim added: %s, c: %.6s, t: %.6s:%d, h: %.6d, a: %d\n",
+                            name, claimId.GetHex(), point.hash.GetHex(), point.n, nHeight, nValue);
+    return addClaim(trieCache, name, claimId, -1, -1);
 }
 
 bool CClaimScriptAddOp::updateClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId)
 {
-    return addClaim(trieCache, name, claimId);
+    LogPrint(BCLog::CLAIMS, "+++ Claim updated: %s, c: %.6s, t: %.6s:%d, h: %.6d, a: %d\n",
+             name, claimId.GetHex(), point.hash.GetHex(), point.n, nHeight, nValue);
+    return addClaim(trieCache, name, claimId, -1, -1);
 }
 
-bool CClaimScriptAddOp::addClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId)
+bool CClaimScriptAddOp::addClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId,
+                                 int takeoverHeight, int originalHeight)
 {
-    return trieCache.addClaim(name, point, claimId, nValue, nHeight);
+    return trieCache.addClaim(name, point, claimId, nValue, nHeight, takeoverHeight, originalHeight);
 }
 
 bool CClaimScriptAddOp::supportClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId)
 {
-    return trieCache.addSupport(name, point, nValue, claimId, nHeight);
+    LogPrint(BCLog::CLAIMS, "+++ Support added: %s, c: %.6s, t: %.6s:%d, h: %.6d, a: %d\n",
+             name, claimId.GetHex(), point.hash.GetHex(), point.n, nHeight, nValue);
+    return trieCache.addSupport(name, point, claimId, nValue, nHeight, -1);
 }
 
 CClaimScriptUndoAddOp::CClaimScriptUndoAddOp(const COutPoint& point, int nHeight) : point(point), nHeight(nHeight)
@@ -38,37 +46,39 @@ CClaimScriptUndoAddOp::CClaimScriptUndoAddOp(const COutPoint& point, int nHeight
 bool CClaimScriptUndoAddOp::claimName(CClaimTrieCache& trieCache, const std::string& name)
 {
     auto claimId = ClaimIdHash(point.hash, point.n);
-    LogPrint(BCLog::CLAIMS, "--- [%lu]: OP_CLAIM_NAME \"%s\" with claimId %s and tx prevout %s at index %d\n", nHeight, name, claimId.GetHex(), point.hash.ToString(), point.n);
+    LogPrint(BCLog::CLAIMS, "--- Undoing claim add: %s, c: %.6s, t: %.6s:%d, h: %.6d\n",
+             name, claimId.GetHex(), point.hash.GetHex(), point.n, nHeight);
     return undoAddClaim(trieCache, name, claimId);
 }
 
 bool CClaimScriptUndoAddOp::updateClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId)
 {
-    LogPrint(BCLog::CLAIMS, "--- [%lu]: OP_UPDATE_CLAIM \"%s\" with claimId %s and tx prevout %s at index %d\n", nHeight, name, claimId.GetHex(), point.hash.ToString(), point.n);
+    LogPrint(BCLog::CLAIMS, "--- Undoing claim update: %s, c: %.6s, t: %.6s:%d, h: %.6d\n",
+             name, claimId.GetHex(), point.hash.GetHex(), point.n, nHeight);
     return undoAddClaim(trieCache, name, claimId);
 }
 
 bool CClaimScriptUndoAddOp::undoAddClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId)
 {
-    LogPrint(BCLog::CLAIMS, "%s: (txid: %s, nOut: %d) Removing %s, claimId: %s, from the claim trie due to block disconnect\n", __func__, point.hash.ToString(), point.n, name, claimId.ToString());
+    std::string nodeName;
-    bool res = trieCache.undoAddClaim(name, point, nHeight);
+    int validHeight, originalHeight;
+    bool res = trieCache.removeClaim(claimId, point, nodeName, validHeight, originalHeight);
     if (!res)
-        LogPrint(BCLog::CLAIMS, "%s: Removing claim fails\n", __func__);
+        LogPrint(BCLog::CLAIMS, "Remove claim failed for %s (%s) with claimID %.6s, t:%.6s:%d\n", name, nodeName,
+                claimId.GetHex(), point.hash.GetHex(), point.n);
     return res;
 }
 
 bool CClaimScriptUndoAddOp::supportClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId)
 {
-    if (LogAcceptCategory(BCLog::CLAIMS)) {
+    LogPrint(BCLog::CLAIMS, "--- Undoing support add: %s, c: %.6s, t: %.6s:%d, h: %.6d\n",
-        LogPrintf("--- [%lu]: OP_SUPPORT_CLAIM \"%s\" with claimId %s and tx prevout %s at index %d\n", nHeight, name,
+             name, claimId.GetHex(), point.hash.GetHex(), point.n, nHeight);
-                  claimId.GetHex(), point.hash.ToString(), point.n);
+    std::string nodeName;
-        LogPrintf(
+    int validHeight;
-                "%s: (txid: %s, nOut: %d) Removing support for %s, claimId: %s, from the claim trie due to block disconnect\n",
+    bool res = trieCache.removeSupport(point, nodeName, validHeight);
-                __func__, point.hash.ToString(), point.n, name, claimId.ToString());
-    }
-    bool res = trieCache.undoAddSupport(name, point, nHeight);
     if (!res)
-        LogPrint(BCLog::CLAIMS, "%s: Removing support fails\n", __func__);
+        LogPrint(BCLog::CLAIMS, "Remove support failed for %s with claimID %.6s, t:%.6s:%d\n", name,
+                 claimId.GetHex(), point.hash.GetHex(), point.n);
     return res;
 }
 
@@ -80,64 +90,71 @@ CClaimScriptSpendOp::CClaimScriptSpendOp(const COutPoint& point, int nHeight, in
 bool CClaimScriptSpendOp::claimName(CClaimTrieCache& trieCache, const std::string& name)
 {
     auto claimId = ClaimIdHash(point.hash, point.n);
-    LogPrint(BCLog::CLAIMS, "+++ [%lu]: OP_CLAIM_NAME \"%s\" with claimId %s and tx prevout %s at index %d\n", nHeight, name, claimId.GetHex(), point.hash.ToString(), point.n);
+    auto ret = spendClaim(trieCache, name, claimId);
-    return spendClaim(trieCache, name, claimId);
+    LogPrint(BCLog::CLAIMS, "--- Spent original claim: %s, c: %.6s, t: %.6s:%d, h: %.6d, vh: %d\n",
+             name, claimId.GetHex(), point.hash.GetHex(), point.n, nHeight, nValidHeight);
+    return ret;
 }
 
 bool CClaimScriptSpendOp::updateClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId)
 {
-    LogPrint(BCLog::CLAIMS, "+++ [%lu]: OP_UPDATE_CLAIM \"%s\" with claimId %s and tx prevout %s at index %d\n", nHeight, name, claimId.GetHex(), point.hash.ToString(), point.n);
+    auto ret = spendClaim(trieCache, name, claimId);
-    return spendClaim(trieCache, name, claimId);
+    LogPrint(BCLog::CLAIMS, "--- Spent updated claim: %s, c: %.6s, t: %.6s:%d, h: %.6d, vh: %d\n",
+             name, claimId.GetHex(), point.hash.GetHex(), point.n, nHeight, nValidHeight);
+    return ret;
 }
 
 bool CClaimScriptSpendOp::spendClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId)
 {
-    LogPrint(BCLog::CLAIMS, "%s: (txid: %s, nOut: %d) Removing %s, claimId: %s, from the claim trie\n", __func__, point.hash.ToString(), point.n, name, claimId.ToString());
+    std::string nodeName;
-    bool res = trieCache.spendClaim(name, point, nHeight, nValidHeight);
+    bool res = trieCache.removeClaim(claimId, point, nodeName, nValidHeight, nOriginalHeight);
     if (!res)
-        LogPrint(BCLog::CLAIMS, "%s: Removing fails\n", __func__);
+        LogPrint(BCLog::CLAIMS, "Remove claim failed for %s with claimid %s\n", name, claimId.GetHex().substr(0, 6));
     return res;
 }
 
 bool CClaimScriptSpendOp::supportClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId)
 {
-    if (LogAcceptCategory(BCLog::CLAIMS)) {
+    std::string nodeName;
-        LogPrintf("+++ [%lu]: OP_SUPPORT_CLAIM \"%s\" with claimId %s and tx prevout %s at index %d\n", nHeight, name,
+    bool res = trieCache.removeSupport(point, nodeName, nValidHeight);
-                  claimId.GetHex(), point.hash.ToString(), point.n);
+    LogPrint(BCLog::CLAIMS, "--- Spent support: %s, c: %.6s, t: %.6s:%d, h: %.6d, vh: %d\n",
-        LogPrintf("%s: (txid: %s, nOut: %d) Restoring support for %s, claimId: %s, to the claim trie\n", __func__,
+             name, claimId.GetHex(), point.hash.GetHex(), point.n, nHeight, nValidHeight);
-                  point.hash.ToString(), point.n, name, claimId.ToString());
-    }
-    bool res = trieCache.spendSupport(name, point, nHeight, nValidHeight);
     if (!res)
-        LogPrint(BCLog::CLAIMS, "%s: Removing support fails\n", __func__);
+        LogPrint(BCLog::CLAIMS, "Remove support failed for %s with claimid %s\n", name, claimId.GetHex().substr(0, 6));
     return res;
 }
 
-CClaimScriptUndoSpendOp::CClaimScriptUndoSpendOp(const COutPoint& point, CAmount nValue, int nHeight, int nValidHeight)
+CClaimScriptUndoSpendOp::CClaimScriptUndoSpendOp(const COutPoint& point, CAmount nValue, int nHeight, int nValidHeight, int nOriginalHeight)
-    : point(point), nValue(nValue), nHeight(nHeight), nValidHeight(nValidHeight)
+    : point(point), nValue(nValue), nHeight(nHeight), nValidHeight(nValidHeight), nOriginalHeight(nOriginalHeight)
 {
 }
 
 bool CClaimScriptUndoSpendOp::claimName(CClaimTrieCache& trieCache, const std::string& name)
 {
-    return undoSpendClaim(trieCache, name, ClaimIdHash(point.hash, point.n));
+    auto claimId = ClaimIdHash(point.hash, point.n);
+    LogPrint(BCLog::CLAIMS, "+++ Undoing original claim spend: %s, c: %.6s, t: %.6s:%d, h: %.6d, vh: %d\n",
+             name, claimId.GetHex(), point.hash.GetHex(), point.n, nHeight, nValidHeight);
+    return undoSpendClaim(trieCache, name, claimId);
 }
 
 bool CClaimScriptUndoSpendOp::updateClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId)
 {
+    LogPrint(BCLog::CLAIMS, "+++ Undoing updated claim spend: %s, c: %.6s, t: %.6s:%d, h: %.6d, vh: %d\n",
+             name, claimId.GetHex(), point.hash.GetHex(), point.n, nHeight, nValidHeight);
     return undoSpendClaim(trieCache, name, claimId);
 }
 
 bool CClaimScriptUndoSpendOp::undoSpendClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId)
 {
-    LogPrint(BCLog::CLAIMS, "%s: (txid: %s, nOut: %d) Restoring %s, claimId: %s, to the claim trie due to block disconnect\n", __func__, point.hash.ToString(), point.n, name, claimId.ToString());
+    return trieCache.addClaim(name, point, claimId, nValue, nHeight,
-    return trieCache.undoSpendClaim(name, point, claimId, nValue, nHeight, nValidHeight);
+            nValidHeight, nOriginalHeight);
 }
 
 bool CClaimScriptUndoSpendOp::supportClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId)
 {
-    LogPrint(BCLog::CLAIMS, "%s: (txid: %s, nOut: %d) Restoring support for %s, claimId: %s, to the claim trie due to block disconnect\n", __func__, point.hash.ToString(), point.n, name, claimId.ToString());
+    LogPrint(BCLog::CLAIMS, "+++ Undoing support spend: %s, c: %.6s, t: %.6s:%d, h: %.6d, vh: %d\n",
-    return trieCache.undoSpendSupport(name, point, claimId, nValue, nHeight, nValidHeight);
+             name, claimId.GetHex(), point.hash.GetHex(), point.n, nHeight, nValidHeight);
+    return trieCache.addSupport(name, point, claimId, nValue, nHeight, nValidHeight);
 }
 
 static std::string vchToString(const std::vector<unsigned char>& name)
@@ -159,8 +176,9 @@ bool ProcessClaim(CClaimScriptOp& claimOp, CClaimTrieCache& trieCache, const CSc
             return claimOp.supportClaim(trieCache, vchToString(vvchParams[0]), uint160(vvchParams[1]));
         case OP_UPDATE_CLAIM:
             return claimOp.updateClaim(trieCache, vchToString(vvchParams[0]), uint160(vvchParams[1]));
+        default:
+            throw std::runtime_error("Unimplemented OP handler.");
     }
-    throw std::runtime_error("Unimplemented OP handler.");
 }
 
 void UpdateCache(const CTransaction& tx, CClaimTrieCache& trieCache, const CCoinsViewCache& view, int nHeight, const CUpdateCacheCallbacks& callbacks)
@@ -173,17 +191,18 @@ void UpdateCache(const CTransaction& tx, CClaimTrieCache& trieCache, const CCoin
         bool spendClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId) override
         {
             if (CClaimScriptSpendOp::spendClaim(trieCache, name, claimId)) {
-                callback(name, claimId);
+                assert(nOriginalHeight >= 0);
+                callback(name, claimId, nOriginalHeight);
                 return true;
             }
             return false;
         }
-        std::function<void(const std::string& name, const uint160& claimId)> callback;
+        std::function<void(const std::string& name, const uint160& claimId, int originalHeight)> callback;
     };
 
     spentClaimsType spentClaims;
 
-    for (std::size_t j = 0; j < tx.vin.size(); j++) {
+    for (uint32_t j = 0; j < tx.vin.size(); j++) {
         const CTxIn& txin = tx.vin[j];
         const Coin& coin = view.AccessCoin(txin.prevout);
 
@@ -199,13 +218,16 @@ void UpdateCache(const CTransaction& tx, CClaimTrieCache& trieCache, const CCoin
         if (scriptPubKey.empty())
             continue;
 
-        int nValidAtHeight;
+        int nValidAtHeight, nOriginalHeight = 0;
         CSpendClaimHistory spendClaim(COutPoint(txin.prevout.hash, txin.prevout.n), scriptHeight, nValidAtHeight);
-        spendClaim.callback = [&spentClaims](const std::string& name, const uint160& claimId) {
+        spendClaim.callback = [&spentClaims, &nOriginalHeight](const std::string& name, const uint160& claimId, int originalHeight) {
-            spentClaims.emplace_back(name, claimId);
+            spentClaims.push_back({name, claimId, originalHeight});
+            nOriginalHeight = originalHeight;
         };
-        if (ProcessClaim(spendClaim, trieCache, scriptPubKey) && callbacks.claimUndoHeights)
+        if (ProcessClaim(spendClaim, trieCache, scriptPubKey) && callbacks.claimUndoHeights) {
-            callbacks.claimUndoHeights(j, nValidAtHeight);
+            // assert(nValidAtHeight > 0 && nOriginalHeight > 0); // fails on tests
+            callbacks.claimUndoHeights(j, uint32_t(nValidAtHeight), uint32_t(nOriginalHeight));
+        }
     }
 
     class CAddSpendClaim : public CClaimScriptAddOp
@@ -215,11 +237,15 @@ void UpdateCache(const CTransaction& tx, CClaimTrieCache& trieCache, const CCoin
 
         bool updateClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId) override
         {
-            if (callback(name, claimId))
+            auto originalHeight = callback(name, claimId);
-                return CClaimScriptAddOp::updateClaim(trieCache, name, claimId);
+            if (originalHeight >= 0) {
+                LogPrint(BCLog::CLAIMS, "+++ Claim updated: %s, c: %.6s, t: %.6s:%d, h: %.6d, a: %d\n",
+                         name, claimId.GetHex(), point.hash.GetHex(), point.n, nHeight, nValue);
+                return CClaimScriptAddOp::addClaim(trieCache, name, claimId, -1, originalHeight);
+            }
             return false;
         }
-        std::function<bool(const std::string& name, const uint160& claimId)> callback;
+        std::function<int(const std::string& name, const uint160& claimId)> callback;
     };
 
     for (std::size_t j = 0; j < tx.vout.size(); j++) {
@@ -229,14 +255,15 @@ void UpdateCache(const CTransaction& tx, CClaimTrieCache& trieCache, const CCoin
             continue;
 
         CAddSpendClaim addClaim(COutPoint(tx.GetHash(), j), txout.nValue, nHeight);
-        addClaim.callback = [&trieCache, &spentClaims](const std::string& name, const uint160& claimId) -> bool {
+        addClaim.callback = [&trieCache, &spentClaims](const std::string& name, const uint160& claimId) -> int {
             for (auto itSpent = spentClaims.begin(); itSpent != spentClaims.end(); ++itSpent) {
-                if (itSpent->second == claimId && trieCache.normalizeClaimName(name) == trieCache.normalizeClaimName(itSpent->first)) {
+                if (itSpent->id == claimId && trieCache.normalizeClaimName(name) == trieCache.normalizeClaimName(itSpent->name)) {
                     spentClaims.erase(itSpent);
-                    return true;
+                    assert(itSpent->originalHeight >= 0);
+                    return itSpent->originalHeight;
                 }
             }
-            return false;
+            return -1;
         };
         ProcessClaim(addClaim, trieCache, txout.scriptPubKey);
     }

src/claimscriptop.h

@@ -6,7 +6,7 @@
 #define CLAIMSCRIPTOP_H
 
 #include "amount.h"
-#include "claimtrie.h"
+#include "claimtrie/forks.h"
 #include "hash.h"
 #include "primitives/transaction.h"
 #include "script/script.h"
@@ -22,7 +22,7 @@
 class CClaimScriptOp
 {
 public:
-    virtual ~CClaimScriptOp() {}
+    virtual ~CClaimScriptOp() = default;
     /**
      * Pure virtual, OP_CLAIM_NAME handler
      * @param[in] trieCache     trie to operate on
@@ -81,7 +81,8 @@ protected:
      * @param[in] name          name of the claim
      * @param[in] claimId       id of the claim
      */
-    virtual bool addClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId);
+    virtual bool addClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId,
+            int takeoverHeight, int originalHeight);
     const COutPoint point;
     const CAmount nValue;
     const int nHeight;
@@ -167,6 +168,7 @@ protected:
     const COutPoint point;
     const int nHeight;
     int& nValidHeight;
+    int nOriginalHeight;
 };
 
 /**
@@ -182,7 +184,7 @@ public:
      * @param[in] nHeight       entry height of the claim
      * @param[in] nValidHeight  valid height of the claim
      */
-    CClaimScriptUndoSpendOp(const COutPoint& point, CAmount nValue, int nHeight, int nValidHeight);
+    CClaimScriptUndoSpendOp(const COutPoint& point, CAmount nValue, int nHeight, int nValidHeight, int nOriginalHeight);
     /**
      * Implementation of OP_CLAIM_NAME handler
      * @see CClaimScriptOp::claimName
@@ -211,6 +213,7 @@ protected:
     const CAmount nValue;
     const int nHeight;
     const int nValidHeight;
+    const int nOriginalHeight;
 };
 
 /**
@@ -221,14 +224,13 @@ protected:
  */
 bool ProcessClaim(CClaimScriptOp& claimOp, CClaimTrieCache& trieCache, const CScript& scriptPubKey);
 
-typedef std::pair<std::string, uint160> spentClaimType;
+struct spentClaimType { std::string name; uint160 id; int originalHeight; };
-
 typedef std::vector<spentClaimType> spentClaimsType;
 
 struct CUpdateCacheCallbacks
 {
     std::function<CScript(const COutPoint& point)> findScriptKey;
-    std::function<void(int, int)> claimUndoHeights;
+    std::function<void(uint32_t, uint32_t, uint32_t)> claimUndoHeights;
 };
 
 /**

src/claimtrie.h

@@ -1,756 +0,0 @@
-#ifndef BITCOIN_CLAIMTRIE_H
-#define BITCOIN_CLAIMTRIE_H
-
-#include <amount.h>
-#include <chain.h>
-#include <chainparams.h>
-#include <dbwrapper.h>
-#include <prefixtrie.h>
-#include <primitives/transaction.h>
-#include <serialize.h>
-#include <uint256.h>
-#include <util.h>
-
-#include <map>
-#include <string>
-#include <vector>
-#include <unordered_map>
-#include <unordered_set>
-
-// leveldb keys
-#define TRIE_NODE 'n'
-#define TRIE_NODE_CHILDREN 'b'
-#define CLAIM_BY_ID 'i'
-#define CLAIM_QUEUE_ROW 'r'
-#define CLAIM_QUEUE_NAME_ROW 'm'
-#define CLAIM_EXP_QUEUE_ROW 'e'
-#define SUPPORT 's'
-#define SUPPORT_QUEUE_ROW 'u'
-#define SUPPORT_QUEUE_NAME_ROW 'p'
-#define SUPPORT_EXP_QUEUE_ROW 'x'
-
-uint256 getValueHash(const COutPoint& outPoint, int nHeightOfLastTakeover);
-
-struct CClaimValue
-{
-    COutPoint outPoint;
-    uint160 claimId;
-    CAmount nAmount = 0;
-    CAmount nEffectiveAmount = 0;
-    int nHeight = 0;
-    int nValidAtHeight = 0;
-
-    CClaimValue() = default;
-
-    CClaimValue(const COutPoint& outPoint, const uint160& claimId, CAmount nAmount, int nHeight, int nValidAtHeight)
-        : outPoint(outPoint), claimId(claimId), nAmount(nAmount), nEffectiveAmount(nAmount), nHeight(nHeight), nValidAtHeight(nValidAtHeight)
-    {
-    }
-
-    CClaimValue(CClaimValue&&) = default;
-    CClaimValue(const CClaimValue&) = default;
-    CClaimValue& operator=(CClaimValue&&) = default;
-    CClaimValue& operator=(const CClaimValue&) = default;
-
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action)
-    {
-        READWRITE(outPoint);
-        READWRITE(claimId);
-        READWRITE(nAmount);
-        READWRITE(nHeight);
-        READWRITE(nValidAtHeight);
-    }
-
-    bool operator<(const CClaimValue& other) const
-    {
-        if (nEffectiveAmount < other.nEffectiveAmount)
-            return true;
-        if (nEffectiveAmount != other.nEffectiveAmount)
-            return false;
-        if (nHeight > other.nHeight)
-            return true;
-        if (nHeight != other.nHeight)
-            return false;
-        return outPoint != other.outPoint && !(outPoint < other.outPoint);
-    }
-
-    bool operator==(const CClaimValue& other) const
-    {
-        return outPoint == other.outPoint && claimId == other.claimId && nAmount == other.nAmount && nHeight == other.nHeight && nValidAtHeight == other.nValidAtHeight;
-    }
-
-    bool operator!=(const CClaimValue& other) const
-    {
-        return !(*this == other);
-    }
-};
-
-struct CSupportValue
-{
-    COutPoint outPoint;
-    uint160 supportedClaimId;
-    CAmount nAmount = 0;
-    int nHeight = 0;
-    int nValidAtHeight = 0;
-
-    CSupportValue() = default;
-
-    CSupportValue(const COutPoint& outPoint, const uint160& supportedClaimId, CAmount nAmount, int nHeight, int nValidAtHeight)
-        : outPoint(outPoint), supportedClaimId(supportedClaimId), nAmount(nAmount), nHeight(nHeight), nValidAtHeight(nValidAtHeight)
-    {
-    }
-
-    CSupportValue(CSupportValue&&) = default;
-    CSupportValue(const CSupportValue&) = default;
-    CSupportValue& operator=(CSupportValue&&) = default;
-    CSupportValue& operator=(const CSupportValue&) = default;
-
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action)
-    {
-        READWRITE(outPoint);
-        READWRITE(supportedClaimId);
-        READWRITE(nAmount);
-        READWRITE(nHeight);
-        READWRITE(nValidAtHeight);
-    }
-
-    bool operator==(const CSupportValue& other) const
-    {
-        return outPoint == other.outPoint && supportedClaimId == other.supportedClaimId && nAmount == other.nAmount && nHeight == other.nHeight && nValidAtHeight == other.nValidAtHeight;
-    }
-
-    bool operator!=(const CSupportValue& other) const
-    {
-        return !(*this == other);
-    }
-};
-
-typedef std::vector<CClaimValue> claimEntryType;
-typedef std::vector<CSupportValue> supportEntryType;
-
-struct CClaimTrieData
-{
-    uint256 hash;
-    claimEntryType claims;
-    int nHeightOfLastTakeover = 0;
-
-    CClaimTrieData() = default;
-    CClaimTrieData(CClaimTrieData&&) = default;
-    CClaimTrieData(const CClaimTrieData&) = default;
-    CClaimTrieData& operator=(CClaimTrieData&&) = default;
-    CClaimTrieData& operator=(const CClaimTrieData& d) = default;
-
-    bool insertClaim(const CClaimValue& claim);
-    bool removeClaim(const COutPoint& outPoint, CClaimValue& claim);
-    bool getBestClaim(CClaimValue& claim) const;
-    bool haveClaim(const COutPoint& outPoint) const;
-    void reorderClaims(const supportEntryType& support);
-
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action)
-    {
-        READWRITE(hash);
-
-        if (ser_action.ForRead()) {
-            if (s.eof()) {
-                claims.clear();
-                nHeightOfLastTakeover = 0;
-                return;
-            }
-        }
-        else if (claims.empty())
-            return;
-
-        READWRITE(claims);
-        READWRITE(nHeightOfLastTakeover);
-    }
-
-    bool operator==(const CClaimTrieData& other) const
-    {
-        return hash == other.hash && nHeightOfLastTakeover == other.nHeightOfLastTakeover && claims == other.claims;
-    }
-
-    bool operator!=(const CClaimTrieData& other) const
-    {
-        return !(*this == other);
-    }
-
-    bool empty() const
-    {
-        return claims.empty();
-    }
-};
-
-struct COutPointHeightType
-{
-    COutPoint outPoint;
-    int nHeight = 0;
-
-    COutPointHeightType() = default;
-
-    COutPointHeightType(const COutPoint& outPoint, int nHeight)
-        : outPoint(outPoint), nHeight(nHeight)
-    {
-    }
-
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action)
-    {
-        READWRITE(outPoint);
-        READWRITE(nHeight);
-    }
-};
-
-struct CNameOutPointHeightType
-{
-    std::string name;
-    COutPoint outPoint;
-    int nHeight = 0;
-
-    CNameOutPointHeightType() = default;
-
-    CNameOutPointHeightType(std::string name, const COutPoint& outPoint, int nHeight)
-        : name(std::move(name)), outPoint(outPoint), nHeight(nHeight)
-    {
-    }
-
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action)
-    {
-        READWRITE(name);
-        READWRITE(outPoint);
-        READWRITE(nHeight);
-    }
-};
-
-struct CNameOutPointType
-{
-    std::string name;
-    COutPoint outPoint;
-
-    CNameOutPointType() = default;
-
-    CNameOutPointType(std::string name, const COutPoint& outPoint)
-        : name(std::move(name)), outPoint(outPoint)
-    {
-    }
-
-    bool operator==(const CNameOutPointType& other) const
-    {
-        return name == other.name && outPoint == other.outPoint;
-    }
-
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action)
-    {
-        READWRITE(name);
-        READWRITE(outPoint);
-    }
-};
-
-struct CClaimIndexElement
-{
-    CClaimIndexElement() = default;
-
-    CClaimIndexElement(std::string name, CClaimValue claim)
-        : name(std::move(name)), claim(std::move(claim))
-    {
-    }
-
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action)
-    {
-        READWRITE(name);
-        READWRITE(claim);
-    }
-
-    std::string name;
-    CClaimValue claim;
-};
-
-struct CClaimNsupports
-{
-    CClaimNsupports() = default;
-    CClaimNsupports(CClaimNsupports&&) = default;
-    CClaimNsupports(const CClaimNsupports&) = default;
-
-    CClaimNsupports& operator=(CClaimNsupports&&) = default;
-    CClaimNsupports& operator=(const CClaimNsupports&) = default;
-
-    CClaimNsupports(const CClaimValue& claim, CAmount effectiveAmount, const std::vector<CSupportValue>& supports = {})
-        : claim(claim), effectiveAmount(effectiveAmount), supports(supports)
-    {
-    }
-
-    bool IsNull() const
-    {
-        return claim.claimId.IsNull();
-    }
-
-    CClaimValue claim;
-    CAmount effectiveAmount = 0;
-    std::vector<CSupportValue> supports;
-};
-
-static const CClaimNsupports invalid;
-
-struct CClaimSupportToName
-{
-    CClaimSupportToName(const std::string& name, int nLastTakeoverHeight, std::vector<CClaimNsupports> claimsNsupports, std::vector<CSupportValue> unmatchedSupports)
-        : name(name), nLastTakeoverHeight(nLastTakeoverHeight), claimsNsupports(std::move(claimsNsupports)), unmatchedSupports(std::move(unmatchedSupports))
-    {
-    }
-
-    const CClaimNsupports& find(const uint160& claimId) const
-    {
-        auto it = std::find_if(claimsNsupports.begin(), claimsNsupports.end(), [&claimId](const CClaimNsupports& value) {
-            return claimId == value.claim.claimId;
-        });
-        return it != claimsNsupports.end() ? *it : invalid;
-    }
-
-    const CClaimNsupports& find(const std::string& partialId) const
-    {
-        std::string lowered(partialId);
-        for (auto& c: lowered)
-            c = std::tolower(c);
-
-        auto it = std::find_if(claimsNsupports.begin(), claimsNsupports.end(), [&lowered](const CClaimNsupports& value) {
-            return value.claim.claimId.GetHex().find(lowered) == 0;
-        });
-        return it != claimsNsupports.end() ? *it : invalid;
-    }
-
-    const std::string name;
-    const int nLastTakeoverHeight;
-    const std::vector<CClaimNsupports> claimsNsupports;
-    const std::vector<CSupportValue> unmatchedSupports;
-};
-
-class CClaimTrie : public CPrefixTrie<std::string, CClaimTrieData>
-{
-public:
-    CClaimTrie() = default;
-    virtual ~CClaimTrie() = default;
-    CClaimTrie(CClaimTrie&&) = delete;
-    CClaimTrie(const CClaimTrie&) = delete;
-    CClaimTrie(bool fMemory, bool fWipe, int proportionalDelayFactor = 32, std::size_t cacheMB=200);
-
-    CClaimTrie& operator=(CClaimTrie&&) = delete;
-    CClaimTrie& operator=(const CClaimTrie&) = delete;
-
-    bool SyncToDisk();
-
-    friend class CClaimTrieCacheBase;
-    friend struct ClaimTrieChainFixture;
-    friend class CClaimTrieCacheExpirationFork;
-    friend class CClaimTrieCacheNormalizationFork;
-    friend bool getClaimById(const uint160&, std::string&, CClaimValue*);
-    friend bool getClaimById(const std::string&, std::string&, CClaimValue*);
-
-    std::size_t getTotalNamesInTrie() const;
-    std::size_t getTotalClaimsInTrie() const;
-    CAmount getTotalValueOfClaimsInTrie(bool fControllingOnly) const;
-
-protected:
-    int nNextHeight = 0;
-    int nProportionalDelayFactor = 0;
-    std::unique_ptr<CDBWrapper> db;
-};
-
-struct CClaimTrieProofNode
-{
-    CClaimTrieProofNode(std::vector<std::pair<unsigned char, uint256>> children, bool hasValue, const uint256& valHash)
-        : children(std::move(children)), hasValue(hasValue), valHash(valHash)
-    {
-    }
-
-    CClaimTrieProofNode(CClaimTrieProofNode&&) = default;
-    CClaimTrieProofNode(const CClaimTrieProofNode&) = default;
-    CClaimTrieProofNode& operator=(CClaimTrieProofNode&&) = default;
-    CClaimTrieProofNode& operator=(const CClaimTrieProofNode&) = default;
-
-    std::vector<std::pair<unsigned char, uint256>> children;
-    bool hasValue;
-    uint256 valHash;
-};
-
-struct CClaimTrieProof
-{
-    CClaimTrieProof() = default;
-    CClaimTrieProof(CClaimTrieProof&&) = default;
-    CClaimTrieProof(const CClaimTrieProof&) = default;
-    CClaimTrieProof& operator=(CClaimTrieProof&&) = default;
-    CClaimTrieProof& operator=(const CClaimTrieProof&) = default;
-
-    std::vector<std::pair<bool, uint256>> pairs;
-    std::vector<CClaimTrieProofNode> nodes;
-    int nHeightOfLastTakeover = 0;
-    bool hasValue = false;
-    COutPoint outPoint;
-};
-
-template <typename T>
-class COptional
-{
-    bool own;
-    T* value;
-public:
-    COptional(T* value = nullptr) : own(false), value(value) {}
-    COptional(COptional&& o)
-    {
-        own = o.own;
-        value = o.value;
-        o.own = false;
-        o.value = nullptr;
-    }
-    COptional(T&& o) : own(true)
-    {
-        value = new T(std::move(o));
-    }
-    ~COptional()
-    {
-        if (own)
-            delete value;
-    }
-    COptional& operator=(COptional&&) = delete;
-    bool unique() const
-    {
-        return own;
-    }
-    operator bool() const
-    {
-        return value;
-    }
-    operator T*() const
-    {
-        return value;
-    }
-    T* operator->() const
-    {
-        return value;
-    }
-    operator T&() const
-    {
-        return *value;
-    }
-    T& operator*() const
-    {
-        return *value;
-    }
-};
-
-template <typename T>
-using queueEntryType = std::pair<std::string, T>;
-
-typedef std::vector<queueEntryType<CClaimValue>> claimQueueRowType;
-typedef std::map<int, claimQueueRowType> claimQueueType;
-
-typedef std::vector<queueEntryType<CSupportValue>> supportQueueRowType;
-typedef std::map<int, supportQueueRowType> supportQueueType;
-
-typedef std::vector<COutPointHeightType> queueNameRowType;
-typedef std::map<std::string, queueNameRowType> queueNameType;
-
-typedef std::vector<CNameOutPointHeightType> insertUndoType;
-
-typedef std::vector<CNameOutPointType> expirationQueueRowType;
-typedef std::map<int, expirationQueueRowType> expirationQueueType;
-
-typedef std::set<CClaimValue> claimIndexClaimListType;
-typedef std::vector<CClaimIndexElement> claimIndexElementListType;
-
-class CClaimTrieCacheBase
-{
-public:
-    explicit CClaimTrieCacheBase(CClaimTrie* base);
-    virtual ~CClaimTrieCacheBase() = default;
-
-    uint256 getMerkleHash();
-
-    bool flush();
-    bool empty() const;
-    bool checkConsistency() const;
-    bool ReadFromDisk(const CBlockIndex* tip);
-
-    bool haveClaim(const std::string& name, const COutPoint& outPoint) const;
-    bool haveClaimInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const;
-
-    bool haveSupport(const std::string& name, const COutPoint& outPoint) const;
-    bool haveSupportInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const;
-
-    bool addClaim(const std::string& name, const COutPoint& outPoint, const uint160& claimId, CAmount nAmount, int nHeight);
-    bool undoAddClaim(const std::string& name, const COutPoint& outPoint, int nHeight);
-
-    bool spendClaim(const std::string& name, const COutPoint& outPoint, int nHeight, int& nValidAtHeight);
-    bool undoSpendClaim(const std::string& name, const COutPoint& outPoint, const uint160& claimId, CAmount nAmount, int nHeight, int nValidAtHeight);
-
-    bool addSupport(const std::string& name, const COutPoint& outPoint, CAmount nAmount, const uint160& supportedClaimId, int nHeight);
-    bool undoAddSupport(const std::string& name, const COutPoint& outPoint, int nHeight);
-
-    bool spendSupport(const std::string& name, const COutPoint& outPoint, int nHeight, int& nValidAtHeight);
-    bool undoSpendSupport(const std::string& name, const COutPoint& outPoint, const uint160& supportedClaimId, CAmount nAmount, int nHeight, int nValidAtHeight);
-
-    virtual bool incrementBlock(insertUndoType& insertUndo,
-        claimQueueRowType& expireUndo,
-        insertUndoType& insertSupportUndo,
-        supportQueueRowType& expireSupportUndo,
-        std::vector<std::pair<std::string, int>>& takeoverHeightUndo);
-
-    virtual bool decrementBlock(insertUndoType& insertUndo,
-        claimQueueRowType& expireUndo,
-        insertUndoType& insertSupportUndo,
-        supportQueueRowType& expireSupportUndo);
-
-    virtual bool getProofForName(const std::string& name, CClaimTrieProof& proof);
-    virtual bool getInfoForName(const std::string& name, CClaimValue& claim) const;
-
-    virtual int expirationTime() const;
-
-    virtual bool finalizeDecrement(std::vector<std::pair<std::string, int>>& takeoverHeightUndo);
-
-    virtual CClaimSupportToName getClaimsForName(const std::string& name) const;
-
-    CClaimTrie::const_iterator find(const std::string& name) const;
-    void iterate(std::function<void(const std::string&, const CClaimTrieData&)> callback) const;
-
-    void dumpToLog(CClaimTrie::const_iterator it, bool diffFromBase = true) const;
-    virtual std::string adjustNameForValidHeight(const std::string& name, int validHeight) const;
-
-protected:
-    CClaimTrie* base;
-    CClaimTrie nodesToAddOrUpdate; // nodes pulled in from base (and possibly modified thereafter), written to base on flush
-    std::unordered_set<std::string> nodesAlreadyCached; // set of nodes already pulled into cache from base
-    std::unordered_set<std::string> namesToCheckForTakeover; // takeover numbers are updated on increment
-
-    virtual uint256 recursiveComputeMerkleHash(CClaimTrie::iterator& it);
-    virtual bool recursiveCheckConsistency(CClaimTrie::const_iterator& it, std::string& failed) const;
-
-    virtual bool insertClaimIntoTrie(const std::string& name, const CClaimValue& claim, bool fCheckTakeover);
-    virtual bool removeClaimFromTrie(const std::string& name, const COutPoint& outPoint, CClaimValue& claim, bool fCheckTakeover);
-
-    virtual bool insertSupportIntoMap(const std::string& name, const CSupportValue& support, bool fCheckTakeover);
-    virtual bool removeSupportFromMap(const std::string& name, const COutPoint& outPoint, CSupportValue& support, bool fCheckTakeover);
-
-    supportEntryType getSupportsForName(const std::string& name) const;
-
-    int getDelayForName(const std::string& name) const;
-    virtual int getDelayForName(const std::string& name, const uint160& claimId) const;
-
-    CClaimTrie::iterator cacheData(const std::string& name, bool create = true);
-
-    bool getLastTakeoverForName(const std::string& name, uint160& claimId, int& takeoverHeight) const;
-
-    int getNumBlocksOfContinuousOwnership(const std::string& name) const;
-
-    void reactivateClaim(const expirationQueueRowType& row, int height, bool increment);
-    void reactivateSupport(const expirationQueueRowType& row, int height, bool increment);
-
-    expirationQueueType expirationQueueCache;
-    expirationQueueType supportExpirationQueueCache;
-
-    int nNextHeight; // Height of the block that is being worked on, which is
-                     // one greater than the height of the chain's tip
-
-private:
-    uint256 hashBlock;
-
-    std::unordered_map<std::string, std::pair<uint160, int>> takeoverCache;
-
-    claimQueueType claimQueueCache; // claims not active yet: to be written to disk on flush
-    queueNameType claimQueueNameCache;
-    supportQueueType supportQueueCache; // supports not active yet: to be written to disk on flush
-    queueNameType supportQueueNameCache;
-    claimIndexElementListType claimsToAddToByIdIndex; // written to index on flush
-    claimIndexClaimListType claimsToDeleteFromByIdIndex;
-
-    std::unordered_map<std::string, supportEntryType> supportCache;  // to be added/updated to base (and disk) on flush
-    std::unordered_set<std::string> nodesToDelete; // to be removed from base (and disk) on flush
-    std::unordered_map<std::string, bool> takeoverWorkaround;
-    std::unordered_set<std::string> removalWorkaround;
-
-    bool shouldUseTakeoverWorkaround(const std::string& key) const;
-    void addTakeoverWorkaroundPotential(const std::string& key);
-    void confirmTakeoverWorkaroundNeeded(const std::string& key);
-
-    bool clear();
-
-    void markAsDirty(const std::string& name, bool fCheckTakeover);
-    bool removeSupport(const std::string& name, const COutPoint& outPoint, int nHeight, int& nValidAtHeight, bool fCheckTakeover);
-    bool removeClaim(const std::string& name, const COutPoint& outPoint, int nHeight, int& nValidAtHeight, bool fCheckTakeover);
-
-    template <typename T>
-    void insertRowsFromQueue(std::vector<T>& result, const std::string& name) const;
-
-    template <typename T>
-    std::vector<queueEntryType<T>>* getQueueCacheRow(int nHeight, bool createIfNotExists);
-
-    template <typename T>
-    COptional<const std::vector<queueEntryType<T>>> getQueueCacheRow(int nHeight) const;
-
-    template <typename T>
-    queueNameRowType* getQueueCacheNameRow(const std::string& name, bool createIfNotExists);
-
-    template <typename T>
-    COptional<const queueNameRowType> getQueueCacheNameRow(const std::string& name) const;
-
-    template <typename T>
-    expirationQueueRowType* getExpirationQueueCacheRow(int nHeight, bool createIfNotExists);
-
-    template <typename T>
-    bool haveInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const;
-
-    template <typename T>
-    T add(const std::string& name, const COutPoint& outPoint, const uint160& claimId, CAmount nAmount, int nHeight);
-
-    template <typename T>
-    bool remove(T& value, const std::string& name, const COutPoint& outPoint, int nHeight, int& nValidAtHeight, bool fCheckTakeover = false);
-
-    template <typename T>
-    bool addToQueue(const std::string& name, const T& value);
-
-    template <typename T>
-    bool removeFromQueue(const std::string& name, const COutPoint& outPoint, T& value);
-
-    template <typename T>
-    bool addToCache(const std::string& name, const T& value, bool fCheckTakeover = false);
-
-    template <typename T>
-    bool removeFromCache(const std::string& name, const COutPoint& outPoint, T& value, bool fCheckTakeover = false);
-
-    template <typename T>
-    bool undoSpend(const std::string& name, const T& value, int nValidAtHeight);
-
-    template <typename T>
-    void undoIncrement(insertUndoType& insertUndo, std::vector<queueEntryType<T>>& expireUndo, std::set<T>* deleted = nullptr);
-
-    template <typename T>
-    void undoDecrement(insertUndoType& insertUndo, std::vector<queueEntryType<T>>& expireUndo, std::vector<CClaimIndexElement>* added = nullptr, std::set<T>* deleted = nullptr);
-
-    template <typename T>
-    void undoIncrement(const std::string& name, insertUndoType& insertUndo, std::vector<queueEntryType<T>>& expireUndo);
-
-    template <typename T>
-    void reactivate(const expirationQueueRowType& row, int height, bool increment);
-
-    // for unit test
-    friend struct ClaimTrieChainFixture;
-    friend class CClaimTrieCacheTest;
-};
-
-class CClaimTrieCacheExpirationFork : public CClaimTrieCacheBase
-{
-public:
-    explicit CClaimTrieCacheExpirationFork(CClaimTrie* base);
-
-    void setExpirationTime(int time);
-    int expirationTime() const override;
-
-    virtual void initializeIncrement();
-    bool finalizeDecrement(std::vector<std::pair<std::string, int>>& takeoverHeightUndo) override;
-
-    bool incrementBlock(insertUndoType& insertUndo,
-        claimQueueRowType& expireUndo,
-        insertUndoType& insertSupportUndo,
-        supportQueueRowType& expireSupportUndo,
-        std::vector<std::pair<std::string, int>>& takeoverHeightUndo) override;
-
-    bool decrementBlock(insertUndoType& insertUndo,
-        claimQueueRowType& expireUndo,
-        insertUndoType& insertSupportUndo,
-        supportQueueRowType& expireSupportUndo) override;
-
-private:
-    int nExpirationTime;
-    bool forkForExpirationChange(bool increment);
-};
-
-class CClaimTrieCacheNormalizationFork : public CClaimTrieCacheExpirationFork
-{
-public:
-    explicit CClaimTrieCacheNormalizationFork(CClaimTrie* base)
-        : CClaimTrieCacheExpirationFork(base), overrideInsertNormalization(false), overrideRemoveNormalization(false)
-    {
-    }
-
-    bool shouldNormalize() const;
-
-    // lower-case and normalize any input string name
-    // see: https://unicode.org/reports/tr15/#Norm_Forms
-    std::string normalizeClaimName(const std::string& name, bool force = false) const; // public only for validating name field on update op
-
-    bool incrementBlock(insertUndoType& insertUndo,
-        claimQueueRowType& expireUndo,
-        insertUndoType& insertSupportUndo,
-        supportQueueRowType& expireSupportUndo,
-        std::vector<std::pair<std::string, int>>& takeoverHeightUndo) override;
-
-    bool decrementBlock(insertUndoType& insertUndo,
-        claimQueueRowType& expireUndo,
-        insertUndoType& insertSupportUndo,
-        supportQueueRowType& expireSupportUndo) override;
-
-    bool getProofForName(const std::string& name, CClaimTrieProof& proof) override;
-    bool getInfoForName(const std::string& name, CClaimValue& claim) const override;
-    CClaimSupportToName getClaimsForName(const std::string& name) const override;
-    std::string adjustNameForValidHeight(const std::string& name, int validHeight) const override;
-
-protected:
-    bool insertClaimIntoTrie(const std::string& name, const CClaimValue& claim, bool fCheckTakeover) override;
-    bool removeClaimFromTrie(const std::string& name, const COutPoint& outPoint, CClaimValue& claim, bool fCheckTakeover) override;
-
-    bool insertSupportIntoMap(const std::string& name, const CSupportValue& support, bool fCheckTakeover) override;
-    bool removeSupportFromMap(const std::string& name, const COutPoint& outPoint, CSupportValue& support, bool fCheckTakeover) override;
-
-    int getDelayForName(const std::string& name, const uint160& claimId) const override;
-
-private:
-    bool overrideInsertNormalization;
-    bool overrideRemoveNormalization;
-
-    bool normalizeAllNamesInTrieIfNecessary(insertUndoType& insertUndo,
-        claimQueueRowType& removeUndo,
-        insertUndoType& insertSupportUndo,
-        supportQueueRowType& expireSupportUndo,
-        std::vector<std::pair<std::string, int>>& takeoverHeightUndo);
-};
-
-class CClaimTrieCacheHashFork : public CClaimTrieCacheNormalizationFork
-{
-public:
-    explicit CClaimTrieCacheHashFork(CClaimTrie* base);
-
-    bool getProofForName(const std::string& name, CClaimTrieProof& proof) override;
-    bool getProofForName(const std::string& name, CClaimTrieProof& proof, const std::function<bool(const CClaimValue&)>& comp);
-    void initializeIncrement() override;
-    bool finalizeDecrement(std::vector<std::pair<std::string, int>>& takeoverHeightUndo) override;
-
-    bool allowSupportMetadata() const;
-
-protected:
-    uint256 recursiveComputeMerkleHash(CClaimTrie::iterator& it) override;
-    bool recursiveCheckConsistency(CClaimTrie::const_iterator& it, std::string& failed) const override;
-
-private:
-    void copyAllBaseToCache();
-};
-
-typedef CClaimTrieCacheHashFork CClaimTrieCache;
-
-#endif // BITCOIN_CLAIMTRIE_H

src/claimtrie/CMakeLists.txt

@@ -0,0 +1,158 @@
+cmake_minimum_required(VERSION 3.10)
+
+project(libclaimtrie)
+set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+include(../../contrib/cmake/cmake/CPM.cmake)
+include(ExternalProject)
+
+set(CLAIMTRIE_SRC
+    blob.cpp
+    data.cpp
+    forks.cpp
+    hashes.cpp
+    log.cpp
+    trie.cpp
+    txoutpoint.cpp
+    uints.cpp
+    ./sqlite/sqlite3.c
+)
+
+if(BIND)
+    find_program(SWIG NAMES swig)
+    string(TOLOWER ${BIND} BIND)
+    if(${BIND} STREQUAL "python")
+        find_package(PythonInterp 3.6 REQUIRED)
+        find_package(PythonLibs 3.6 REQUIRED)
+        set(BIND_INCLUDE_DIRS ${PYTHON_INCLUDE_DIRS})
+        set(SWIG_OPTIONS -python;-py3)
+        set(CMAKE_SHARED_LIBRARY_PREFIX _lib)
+    elseif(${BIND} STREQUAL "go")
+        find_program(GOLANG NAMES go)
+        if(NOT GOLANG)
+            message(FATAL_ERROR "Golang was not found in system path")
+        endif()
+        execute_process(COMMAND ${GOLANG} "version" OUTPUT_VARIABLE GOLANG_VERSION)
+        STRING(REGEX MATCH "[0-9]+.[0-9]+.[0-9]" GOLANG_VERSION "${GOLANG_VERSION}")
+        if(GOLANG_VERSION VERSION_LESS 1.5)
+            message(FATAL_ERROR "Update Golang to at least 1.5, now " ${GOLANG_VERSION})
+        endif()
+        set(SWIG_OPTIONS -go;-cgo;-intgosize;32)
+        set(POST_BUILD_COMMAND go)
+        set(POST_BUILD_ARGS install;-x;${CMAKE_PROJECT_NAME}.go)
+        # CGO_LDFLAGS is buggy so we should inject linker flags in generated go file
+        set(POST_BUILD_PATCH -i "\"s/import \\\"C\\\"/\\/\\/ #cgo LDFLAGS: -L\\\$$\\{SRCDIR\\} -lclaimtrie\\nimport \\\"C\\\"/\"" ${CMAKE_PROJECT_NAME}.go)
+    else()
+        message(FATAL_ERROR "Implement a handler for ${BIND}")
+    endif()
+    add_custom_command(OUTPUT ${CMAKE_PROJECT_NAME}_wrap.cxx
+        COMMAND ${SWIG}
+        ARGS -c++ ${SWIG_OPTIONS} -outcurrentdir ${CMAKE_CURRENT_SOURCE_DIR}/${CMAKE_PROJECT_NAME}.i
+        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    )
+    set(CLAIMTRIE_SRC ${CLAIMTRIE_SRC}
+        ${CMAKE_PROJECT_NAME}_wrap.cxx
+    )
+endif()
+
+add_library(claimtrie SHARED ${CLAIMTRIE_SRC})
+if (POST_BUILD_PATCH)
+    add_custom_command(TARGET claimtrie POST_BUILD
+        COMMAND sed
+        ARGS ${POST_BUILD_PATCH}
+        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    )
+endif()
+if(POST_BUILD_COMMAND)
+    add_custom_command(TARGET claimtrie POST_BUILD
+        COMMAND ${POST_BUILD_COMMAND}
+        ARGS ${POST_BUILD_ARGS}
+        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    )
+endif()
+
+target_include_directories(claimtrie PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+
+if(BIND_INCLUDE_DIRS)
+    target_include_directories(claimtrie PRIVATE ${BIND_INCLUDE_DIRS})
+endif()
+
+CPMAddPackage(
+    NAME OpenSSL
+    GITHUB_REPOSITORY openssl/openssl
+    VERSION 1.0.2
+    GIT_TAG OpenSSL_1_0_2r
+    DOWNLOAD_ONLY TRUE
+)
+
+if(OpenSSL_ADDED)
+    string(TOLOWER ${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR} ARCH)
+    ExternalProject_Add(OpenSSL
+        PREFIX openssl
+        SOURCE_DIR ${OpenSSL_SOURCE_DIR}
+        CONFIGURE_COMMAND ${OpenSSL_SOURCE_DIR}/Configure ${ARCH} no-shared no-dso no-engines -fPIC --prefix=<INSTALL_DIR>
+        BUILD_IN_SOURCE 1
+    )
+    add_dependencies(claimtrie OpenSSL)
+    ExternalProject_Get_Property(OpenSSL INSTALL_DIR)
+    target_link_directories(claimtrie PRIVATE ${INSTALL_DIR}/lib)
+    target_include_directories(claimtrie PRIVATE ${INSTALL_DIR}/include)
+endif(OpenSSL_ADDED)
+
+target_link_libraries(claimtrie PRIVATE ssl)
+
+set(BOOST_LIBS filesystem,locale,system,chrono,thread,test)
+
+set(BOOST_COMPONENTS filesystem;locale;system;chrono;thread;unit_test_framework)
+
+CPMAddPackage(
+    NAME Boost
+    GITHUB_REPOSITORY boostorg/boost
+    VERSION 1.64.0
+    COMPONENTS ${BOOST_COMPONENTS}
+    GIT_TAG boost-1.69.0
+    GIT_SUBMODULES libs/* tools/*
+    DOWNLOAD_ONLY TRUE
+)
+
+# if boost is found system wide we expect to be compiled against icu, so we can skip it
+if(Boost_ADDED)
+    CPMAddPackage(
+        NAME ICU
+        GITHUB_REPOSITORY unicode-org/icu
+        VERSION 63.2
+        GIT_TAG release-63-2
+        DOWNLOAD_ONLY TRUE
+    )
+
+    if(ICU_ADDED)
+        ExternalProject_Add(ICU
+            PREFIX icu
+            SOURCE_DIR ${ICU_SOURCE_DIR}
+            CONFIGURE_COMMAND ${ICU_SOURCE_DIR}/icu4c/source/configure --disable-extras --disable-strict --enable-static
+                --disable-shared --disable-tests --disable-samples --disable-dyload --disable-layoutex CFLAGS=-fPIC CPPFLAGS=-fPIC --prefix=<INSTALL_DIR>
+        )
+        ExternalProject_Get_Property(ICU INSTALL_DIR)
+        set(ICU_PATH ${INSTALL_DIR})
+        target_link_directories(claimtrie PRIVATE ${ICU_PATH}/lib)
+        target_include_directories(claimtrie PRIVATE ${ICU_PATH}/include)
+    endif(ICU_ADDED)
+
+    ExternalProject_Add(Boost
+        PREFIX boost
+        DEPENDS ICU
+        SOURCE_DIR ${Boost_SOURCE_DIR}
+        CONFIGURE_COMMAND ${Boost_SOURCE_DIR}/bootstrap.sh --with-icu=${ICU_PATH} --with-libraries=${BOOST_LIBS} && ${Boost_SOURCE_DIR}/b2 headers
+        BUILD_COMMAND ${Boost_SOURCE_DIR}/b2 install threading=multi -sNO_BZIP2=1 -sNO_ZLIB=1 link=static linkflags="-L${ICU_PATH}/lib -licuio -licuuc -licudata -licui18n" cxxflags=-fPIC boost.locale.iconv=off boost.locale.posix=off boost.locale.icu=on boost.locale.std=off -sICU_PATH=${ICU_PATH} --prefix=<INSTALL_DIR>
+        INSTALL_COMMAND ""
+        BUILD_IN_SOURCE 1
+    )
+    add_dependencies(claimtrie Boost)
+    ExternalProject_Get_Property(Boost INSTALL_DIR)
+    target_link_directories(claimtrie PRIVATE ${INSTALL_DIR}/lib)
+    target_include_directories(claimtrie PRIVATE ${INSTALL_DIR}/include)
+    set_property(DIRECTORY PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${Boost_SOURCE_DIR}/bin.v2)
+endif(Boost_ADDED)
+
+target_link_libraries(claimtrie PRIVATE boost_filesystem boost_locale)

src/claimtrie/blob.cpp

@@ -0,0 +1,170 @@
+
+#include <blob.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cstring>
+#include <iomanip>
+#include <sstream>
+
+/** Template base class for fixed-sized opaque blobs. */
+template<uint32_t BITS>
+CBaseBlob<BITS>::CBaseBlob() noexcept : data{}
+{
+}
+
+template<uint32_t BITS>
+CBaseBlob<BITS>::CBaseBlob(const std::vector<uint8_t>& vec)
+{
+    assert(vec.size() == size());
+    std::copy(vec.begin(), vec.end(), begin());
+}
+
+template<uint32_t BITS>
+CBaseBlob<BITS>::CBaseBlob(const CBaseBlob& o) noexcept
+{
+    *this = o;
+}
+
+template<uint32_t BITS>
+CBaseBlob<BITS>& CBaseBlob<BITS>::operator=(const CBaseBlob& o) noexcept
+{
+    data = o.data;
+    return *this;
+}
+
+template<uint32_t BITS>
+int CBaseBlob<BITS>::Compare(const CBaseBlob& b) const
+{
+    return std::memcmp(begin(), b.begin(), size());
+}
+
+template<uint32_t BITS>
+bool CBaseBlob<BITS>::operator<(const CBaseBlob& b) const
+{
+    return Compare(b) < 0;
+}
+
+template<uint32_t BITS>
+bool CBaseBlob<BITS>::operator==(const CBaseBlob& b) const
+{
+    return Compare(b) == 0;
+}
+
+template<uint32_t BITS>
+bool CBaseBlob<BITS>::operator!=(const CBaseBlob& b) const
+{
+    return Compare(b) != 0;
+}
+
+template<uint32_t BITS>
+bool CBaseBlob<BITS>::IsNull() const
+{
+    return std::all_of(begin(), end(), [](uint8_t e) { return e == 0; });
+}
+
+template<uint32_t BITS>
+void CBaseBlob<BITS>::SetNull()
+{
+    data.fill(0);
+}
+
+template<uint32_t BITS>
+std::string CBaseBlob<BITS>::GetHex(bool reverse) const
+{
+    std::stringstream ss;
+    ss << std::hex;
+    if (reverse) {
+        for (auto it = data.rbegin(); it != data.rend(); ++it)
+            ss << std::setw(2) << std::setfill('0') << uint32_t(*it);
+    }
+    else {
+        for (auto it = data.begin(); it != data.end(); ++it)
+            ss << std::setw(2) << std::setfill('0') << uint32_t(*it);
+    }
+    return ss.str();
+}
+
+template<uint32_t BITS>
+void CBaseBlob<BITS>::SetHex(const char* psz)
+{
+    SetNull();
+
+    // skip leading spaces
+    while (isspace(*psz))
+        psz++;
+
+    // skip 0x
+    if (psz[0] == '0' && tolower(psz[1]) == 'x')
+        psz += 2;
+
+    auto b = psz;
+    // advance to end
+    while (isxdigit(*psz))
+        psz++;
+
+    --psz;
+    char buf[3] = {};
+    auto it = begin();
+    while (psz >= b && it != end()) {
+        buf[1] = *psz--;
+        buf[0] = psz >= b ? *psz-- : '0';
+        *it++ = std::strtoul(buf, nullptr, 16);
+    }
+}
+
+template<uint32_t BITS>
+uint64_t CBaseBlob<BITS>::GetUint64(int pos) const
+{
+    assert((pos + 1) * 8 <= size());
+    const uint8_t* ptr = begin() + pos * 8;
+    uint64_t res = 0;
+    for (int i = 0; i < 8; ++i)
+        res |= (uint64_t(ptr[i]) << (8 * i));
+    return res;
+}
+
+template<uint32_t BITS>
+void CBaseBlob<BITS>::SetHex(const std::string& str)
+{
+    SetHex(str.c_str());
+}
+
+template<uint32_t BITS>
+std::string CBaseBlob<BITS>::ToString() const
+{
+    return GetHex();
+}
+
+template<uint32_t BITS>
+uint8_t* CBaseBlob<BITS>::begin() noexcept
+{
+    return data.data();
+}
+
+template<uint32_t BITS>
+const uint8_t* CBaseBlob<BITS>::begin() const noexcept
+{
+    return data.data();
+}
+
+template<uint32_t BITS>
+uint8_t* CBaseBlob<BITS>::end() noexcept
+{
+    return data.data() + data.size();
+}
+
+template<uint32_t BITS>
+const uint8_t* CBaseBlob<BITS>::end() const noexcept
+{
+    return data.data() + data.size();
+}
+
+template<uint32_t BITS>
+std::size_t CBaseBlob<BITS>::size() const
+{
+    return data.size();
+}
+
+template class CBaseBlob<160>;
+template class CBaseBlob<256>;

src/claimtrie/blob.h

@@ -0,0 +1,50 @@
+
+#ifndef CLAIMTRIE_BLOB_H
+#define CLAIMTRIE_BLOB_H
+
+#include <array>
+#include <string>
+#include <vector>
+
+/** Template base class for fixed-sized opaque blobs. */
+template<uint32_t BITS>
+class CBaseBlob
+{
+    std::array<uint8_t, BITS / 8> data;
+public:
+    CBaseBlob() noexcept;
+
+    explicit CBaseBlob(const std::vector<uint8_t>& vec);
+
+    CBaseBlob(CBaseBlob&&) = default;
+    CBaseBlob& operator=(CBaseBlob&&) = default;
+
+    CBaseBlob(const CBaseBlob& o) noexcept;
+    CBaseBlob& operator=(const CBaseBlob& o) noexcept;
+
+    int Compare(const CBaseBlob& b) const;
+    bool operator<(const CBaseBlob& b) const;
+    bool operator==(const CBaseBlob& b) const;
+    bool operator!=(const CBaseBlob& b) const;
+
+    uint8_t* begin() noexcept;
+    const uint8_t* begin() const noexcept;
+
+    uint8_t* end() noexcept;
+    const uint8_t* end() const noexcept;
+
+    std::size_t size() const;
+
+    bool IsNull() const;
+    void SetNull();
+
+    std::string GetHex(bool reverse=true) const;
+    void SetHex(const char* psz);
+    void SetHex(const std::string& str);
+
+    std::string ToString() const;
+
+    uint64_t GetUint64(int pos) const;
+};
+
+#endif // CLAIMTRIE_BLOB_H

src/claimtrie/data.cpp

@@ -0,0 +1,123 @@
+
+#include <data.h>
+
+#include <algorithm>
+#include <sstream>
+
+CClaimValue::CClaimValue(COutPoint outPoint, uint160 claimId, int64_t nAmount, int nHeight, int nValidAtHeight)
+    : outPoint(std::move(outPoint)), claimId(std::move(claimId)), nAmount(nAmount), nEffectiveAmount(nAmount), nHeight(nHeight), nValidAtHeight(nValidAtHeight)
+{
+}
+
+bool CClaimValue::operator<(const CClaimValue& other) const
+{
+    if (nEffectiveAmount < other.nEffectiveAmount)
+        return true;
+    if (nEffectiveAmount != other.nEffectiveAmount)
+        return false;
+    if (nHeight > other.nHeight)
+        return true;
+    if (nHeight != other.nHeight)
+        return false;
+    return outPoint != other.outPoint && !(outPoint < other.outPoint);
+}
+
+bool CClaimValue::operator==(const CClaimValue& other) const
+{
+    return outPoint == other.outPoint && claimId == other.claimId && nAmount == other.nAmount && nHeight == other.nHeight && nValidAtHeight == other.nValidAtHeight;
+}
+
+bool CClaimValue::operator!=(const CClaimValue& other) const
+{
+    return !(*this == other);
+}
+
+std::string CClaimValue::ToString() const
+{
+    std::stringstream ss;
+    ss  << "CClaimValue(" << outPoint.ToString()
+        << ", " << claimId.ToString()
+        << ", " << nAmount
+        << ", " << nEffectiveAmount
+        << ", " << nHeight
+        << ", " << nValidAtHeight << ')';
+    return ss.str();
+}
+
+CSupportValue::CSupportValue(COutPoint outPoint, uint160 supportedClaimId, int64_t nAmount, int nHeight, int nValidAtHeight)
+    : outPoint(std::move(outPoint)), supportedClaimId(std::move(supportedClaimId)), nAmount(nAmount), nHeight(nHeight), nValidAtHeight(nValidAtHeight)
+{
+}
+
+bool CSupportValue::operator==(const CSupportValue& other) const
+{
+    return outPoint == other.outPoint && supportedClaimId == other.supportedClaimId && nAmount == other.nAmount && nHeight == other.nHeight && nValidAtHeight == other.nValidAtHeight;
+}
+
+bool CSupportValue::operator!=(const CSupportValue& other) const
+{
+    return !(*this == other);
+}
+
+std::string CSupportValue::ToString() const
+{
+    std::stringstream ss;
+    ss  << "CSupportValue(" << outPoint.ToString()
+        << ", " << supportedClaimId.ToString()
+        << ", " << nAmount
+        << ", " << nHeight
+        << ", " << nValidAtHeight << ')';
+    return ss.str();
+}
+
+CNameOutPointHeightType::CNameOutPointHeightType(std::string name, COutPoint outPoint, int nValidHeight)
+    : name(std::move(name)), outPoint(std::move(outPoint)), nValidHeight(nValidHeight)
+{
+}
+
+CClaimNsupports::CClaimNsupports(CClaimValue claim, int64_t effectiveAmount, int originalHeight, std::vector<CSupportValue> supports)
+    : claim(std::move(claim)), effectiveAmount(effectiveAmount), originalHeight(originalHeight), supports(std::move(supports))
+{
+}
+
+bool CClaimNsupports::IsNull() const
+{
+    return claim.claimId.IsNull();
+}
+
+CClaimSupportToName::CClaimSupportToName(std::string name, int nLastTakeoverHeight, std::vector<CClaimNsupports> claimsNsupports, std::vector<CSupportValue> unmatchedSupports)
+    : name(std::move(name)), nLastTakeoverHeight(nLastTakeoverHeight), claimsNsupports(std::move(claimsNsupports)), unmatchedSupports(std::move(unmatchedSupports))
+{
+}
+
+static const CClaimNsupports invalid;
+
+const CClaimNsupports& CClaimSupportToName::find(const uint160& claimId) const
+{
+    auto it = std::find_if(claimsNsupports.begin(), claimsNsupports.end(), [&claimId](const CClaimNsupports& value) {
+        return claimId == value.claim.claimId;
+    });
+    return it != claimsNsupports.end() ? *it : invalid;
+}
+
+const CClaimNsupports& CClaimSupportToName::find(const std::string& partialId) const
+{
+    std::string lowered(partialId);
+    for (auto& c: lowered)
+        c = std::tolower(c);
+
+    auto it = std::find_if(claimsNsupports.begin(), claimsNsupports.end(), [&lowered](const CClaimNsupports& value) {
+        return value.claim.claimId.GetHex().find(lowered) == 0;
+    });
+    return it != claimsNsupports.end() ? *it : invalid;
+}
+
+bool CClaimNsupports::operator<(const CClaimNsupports& other) const
+{
+    return claim < other.claim;
+}
+
+CClaimTrieProofNode::CClaimTrieProofNode(std::vector<std::pair<unsigned char, uint256>> children, bool hasValue, uint256 valHash)
+    : children(std::move(children)), hasValue(hasValue), valHash(std::move(valHash))
+{
+}

src/claimtrie/data.h

@@ -0,0 +1,135 @@
+
+#ifndef CLAIMTRIE_DATA_H
+#define CLAIMTRIE_DATA_H
+
+#include <txoutpoint.h>
+#include <uints.h>
+
+#include <cstring>
+#include <string>
+#include <vector>
+
+struct CClaimValue
+{
+    COutPoint outPoint;
+    uint160 claimId;
+    int64_t nAmount = 0;
+    int64_t nEffectiveAmount = 0;
+    int nHeight = 0;
+    int nValidAtHeight = 0;
+
+    CClaimValue() = default;
+    CClaimValue(COutPoint outPoint, uint160 claimId, int64_t nAmount, int nHeight, int nValidAtHeight);
+
+    CClaimValue(CClaimValue&&) = default;
+    CClaimValue(const CClaimValue&) = default;
+    CClaimValue& operator=(CClaimValue&&) = default;
+    CClaimValue& operator=(const CClaimValue&) = default;
+
+    bool operator<(const CClaimValue& other) const;
+    bool operator==(const CClaimValue& other) const;
+    bool operator!=(const CClaimValue& other) const;
+
+    std::string ToString() const;
+};
+
+struct CSupportValue
+{
+    COutPoint outPoint;
+    uint160 supportedClaimId;
+    int64_t nAmount = 0;
+    int nHeight = 0;
+    int nValidAtHeight = 0;
+
+    CSupportValue() = default;
+    CSupportValue(COutPoint outPoint, uint160 supportedClaimId, int64_t nAmount, int nHeight, int nValidAtHeight);
+
+    CSupportValue(CSupportValue&&) = default;
+    CSupportValue(const CSupportValue&) = default;
+    CSupportValue& operator=(CSupportValue&&) = default;
+    CSupportValue& operator=(const CSupportValue&) = default;
+
+    bool operator==(const CSupportValue& other) const;
+    bool operator!=(const CSupportValue& other) const;
+
+    std::string ToString() const;
+};
+
+typedef std::vector<CClaimValue> claimEntryType;
+typedef std::vector<CSupportValue> supportEntryType;
+
+struct CNameOutPointHeightType
+{
+    std::string name;
+    COutPoint outPoint;
+    int nValidHeight = 0;
+
+    CNameOutPointHeightType() = default;
+    CNameOutPointHeightType(std::string name, COutPoint outPoint, int nValidHeight);
+};
+
+struct CClaimNsupports
+{
+    CClaimNsupports() = default;
+    CClaimNsupports(CClaimNsupports&&) = default;
+    CClaimNsupports(const CClaimNsupports&) = default;
+
+    bool operator<(const CClaimNsupports& other) const;
+    CClaimNsupports& operator=(CClaimNsupports&&) = default;
+    CClaimNsupports& operator=(const CClaimNsupports&) = default;
+
+    CClaimNsupports(CClaimValue claim, int64_t effectiveAmount, int originalHeight,
+            std::vector<CSupportValue> supports = {});
+
+    bool IsNull() const;
+
+    CClaimValue claim;
+    int64_t effectiveAmount = 0;
+    int originalHeight = -1;
+    std::vector<CSupportValue> supports;
+};
+
+struct CClaimSupportToName
+{
+    CClaimSupportToName(std::string name, int nLastTakeoverHeight, std::vector<CClaimNsupports> claimsNsupports, std::vector<CSupportValue> unmatchedSupports);
+
+    const CClaimNsupports& find(const uint160& claimId) const;
+    const CClaimNsupports& find(const std::string& partialId) const;
+
+    const std::string name;
+    const int nLastTakeoverHeight;
+    const std::vector<CClaimNsupports> claimsNsupports;
+    const std::vector<CSupportValue> unmatchedSupports;
+};
+
+struct CClaimTrieProofNode
+{
+    CClaimTrieProofNode(std::vector<std::pair<unsigned char, uint256>> children, bool hasValue, uint256 valHash);
+
+    CClaimTrieProofNode() = default;
+    CClaimTrieProofNode(CClaimTrieProofNode&&) = default;
+    CClaimTrieProofNode(const CClaimTrieProofNode&) = default;
+    CClaimTrieProofNode& operator=(CClaimTrieProofNode&&) = default;
+    CClaimTrieProofNode& operator=(const CClaimTrieProofNode&) = default;
+
+    std::vector<std::pair<unsigned char, uint256>> children;
+    bool hasValue;
+    uint256 valHash;
+};
+
+struct CClaimTrieProof
+{
+    CClaimTrieProof() = default;
+    CClaimTrieProof(CClaimTrieProof&&) = default;
+    CClaimTrieProof(const CClaimTrieProof&) = default;
+    CClaimTrieProof& operator=(CClaimTrieProof&&) = default;
+    CClaimTrieProof& operator=(const CClaimTrieProof&) = default;
+
+    std::vector<std::pair<bool, uint256>> pairs;
+    std::vector<CClaimTrieProofNode> nodes;
+    int nHeightOfLastTakeover = 0;
+    bool hasValue = false;
+    COutPoint outPoint;
+};
+
+#endif // CLAIMTRIE_DATA_H

src/claimtrie/forks.cpp

@@ -0,0 +1,421 @@
+
+#include <forks.h>
+#include <hashes.h>
+#include <log.h>
+#include <trie.h>
+
+#include <boost/locale.hpp>
+#include <boost/locale/conversion.hpp>
+#include <boost/locale/localization_backend.hpp>
+
+#define logPrint CLogPrint::global()
+
+CClaimTrieCacheExpirationFork::CClaimTrieCacheExpirationFork(CClaimTrie* base) : CClaimTrieCacheBase(base)
+{
+    expirationHeight = nNextHeight;
+}
+
+int CClaimTrieCacheExpirationFork::expirationTime() const
+{
+    if (expirationHeight < base->nExtendedClaimExpirationForkHeight)
+        return CClaimTrieCacheBase::expirationTime();
+    return base->nExtendedClaimExpirationTime;
+}
+
+bool CClaimTrieCacheExpirationFork::incrementBlock()
+{
+    if (CClaimTrieCacheBase::incrementBlock()) {
+        expirationHeight = nNextHeight;
+        return true;
+    }
+    return false;
+}
+
+bool CClaimTrieCacheExpirationFork::decrementBlock()
+{
+    if (CClaimTrieCacheBase::decrementBlock()) {
+        expirationHeight = nNextHeight;
+        return true;
+    }
+    return false;
+}
+
+void CClaimTrieCacheExpirationFork::initializeIncrement()
+{
+    // we could do this in the constructor, but that would not allow for multiple increments in a row (as done in unit tests)
+    if (nNextHeight != base->nExtendedClaimExpirationForkHeight)
+        return;
+
+    forkForExpirationChange(true);
+}
+
+bool CClaimTrieCacheExpirationFork::finalizeDecrement()
+{
+    auto ret = CClaimTrieCacheBase::finalizeDecrement();
+    if (ret && nNextHeight == base->nExtendedClaimExpirationForkHeight)
+       forkForExpirationChange(false);
+    return ret;
+}
+
+bool CClaimTrieCacheExpirationFork::forkForExpirationChange(bool increment)
+{
+    ensureTransacting();
+
+    /*
+    If increment is True, we have forked to extend the expiration time, thus items in the expiration queue
+    will have their expiration extended by "new expiration time - original expiration time"
+
+    If increment is False, we are decremented a block to reverse the fork. Thus items in the expiration queue
+    will have their expiration extension removed.
+    */
+
+    auto height = nNextHeight;
+    int extension = base->nExtendedClaimExpirationTime - base->nOriginalClaimExpirationTime;
+    if (!increment) {
+        height += extension;
+        extension *= -1;
+    }
+
+    db << "UPDATE claim SET expirationHeight = expirationHeight + ? WHERE expirationHeight >= ?" << extension << height;
+    db << "UPDATE support SET expirationHeight = expirationHeight + ? WHERE expirationHeight >= ?" << extension << height;
+
+    return true;
+}
+
+CClaimTrieCacheNormalizationFork::CClaimTrieCacheNormalizationFork(CClaimTrie* base) : CClaimTrieCacheExpirationFork(base)
+{
+    db.define("NORMALIZED", [this](const std::string& str) { return normalizeClaimName(str, true); });
+}
+
+bool CClaimTrieCacheNormalizationFork::shouldNormalize() const
+{
+    return nNextHeight > base->nNormalizedNameForkHeight;
+}
+
+std::string CClaimTrieCacheNormalizationFork::normalizeClaimName(const std::string& name, bool force) const
+{
+    if (!force && !shouldNormalize())
+        return name;
+
+    static std::locale utf8;
+    static bool initialized = false;
+    if (!initialized) {
+        static boost::locale::localization_backend_manager manager =
+            boost::locale::localization_backend_manager::global();
+        manager.select("icu");
+
+        static boost::locale::generator curLocale(manager);
+        utf8 = curLocale("en_US.UTF8");
+        initialized = true;
+    }
+
+    std::string normalized;
+    try {
+        // Check if it is a valid utf-8 string. If not, it will throw a
+        // boost::locale::conv::conversion_error exception which we catch later
+        normalized = boost::locale::conv::to_utf<char>(name, "UTF-8", boost::locale::conv::stop);
+        if (normalized.empty())
+            return name;
+
+        // these methods supposedly only use the "UTF8" portion of the locale object:
+        normalized = boost::locale::normalize(normalized, boost::locale::norm_nfd, utf8);
+        normalized = boost::locale::fold_case(normalized, utf8);
+    } catch (const boost::locale::conv::conversion_error& e) {
+        return name;
+    } catch (const std::bad_cast& e) {
+        logPrint << "CClaimTrieCacheNormalizationFork::" << __func__ << "() is invalid or dependencies are missing: " << e.what() << Clog::endl;
+        throw;
+    } catch (const std::exception& e) { // TODO: change to use ... with current_exception() in c++11
+        logPrint << "CClaimTrieCacheNormalizationFork::" << __func__ << "() had an unexpected exception: " << e.what() << Clog::endl;
+        return name;
+    }
+
+    return normalized;
+}
+
+bool CClaimTrieCacheNormalizationFork::normalizeAllNamesInTrieIfNecessary()
+{
+    ensureTransacting();
+
+    // make the new nodes
+    db << "INSERT INTO node(name) SELECT NORMALIZED(name) AS nn FROM claim WHERE nn != nodeName "
+          "AND activationHeight <= ?1 AND expirationHeight > ?1 ON CONFLICT(name) DO UPDATE SET hash = NULL, claimsHash = NULL" << nNextHeight;
+
+    // there's a subtlety here: names in supports don't make new nodes
+    db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name IN "
+          "(SELECT NORMALIZED(name) AS nn FROM support WHERE nn != nodeName "
+          "AND activationHeight <= ?1 AND expirationHeight > ?1)" << nNextHeight;
+
+    // update the claims and supports
+    db << "UPDATE claim SET nodeName = NORMALIZED(name) WHERE activationHeight <= ?1 AND expirationHeight > ?1" << nNextHeight;
+    db << "UPDATE support SET nodeName = NORMALIZED(name) WHERE activationHeight <= ?1 AND expirationHeight > ?1" << nNextHeight;
+
+    // remove the old nodes
+    db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name NOT IN "
+          "(SELECT nodeName FROM claim WHERE activationHeight <= ?1 AND expirationHeight > ?1 "
+          "UNION SELECT nodeName FROM support WHERE activationHeight <= ?1 AND expirationHeight > ?1)" << nNextHeight;
+
+    // work around a bug in the old implementation:
+    db << "UPDATE claim SET activationHeight = ?1 " // force a takeover on these
+          "WHERE updateHeight < ?1 AND activationHeight > ?1 AND nodeName != name" << nNextHeight;
+
+    return true;
+}
+
+bool CClaimTrieCacheNormalizationFork::unnormalizeAllNamesInTrieIfNecessary()
+{
+    ensureTransacting();
+
+    db << "INSERT INTO node(name) SELECT name FROM claim WHERE name != nodeName "
+          "AND activationHeight < ?1 AND expirationHeight > ?1 ON CONFLICT(name) DO UPDATE SET hash = NULL, claimsHash = NULL" << nNextHeight;
+
+    db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name IN "
+          "(SELECT nodeName FROM support WHERE name != nodeName "
+          "UNION SELECT nodeName FROM claim WHERE name != nodeName)";
+
+    db << "UPDATE claim SET nodeName = name";
+    db << "UPDATE support SET nodeName = name";
+
+    // we need to let the tree structure method do the actual node delete
+    db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name NOT IN "
+          "(SELECT DISTINCT name FROM claim)";
+
+    return true;
+}
+
+bool CClaimTrieCacheNormalizationFork::incrementBlock()
+{
+    if (nNextHeight == base->nNormalizedNameForkHeight)
+        normalizeAllNamesInTrieIfNecessary();
+    return CClaimTrieCacheExpirationFork::incrementBlock();
+}
+
+bool CClaimTrieCacheNormalizationFork::decrementBlock()
+{
+    auto ret = CClaimTrieCacheExpirationFork::decrementBlock();
+    if (ret && nNextHeight == base->nNormalizedNameForkHeight)
+        unnormalizeAllNamesInTrieIfNecessary();
+    return ret;
+}
+
+bool CClaimTrieCacheNormalizationFork::getProofForName(const std::string& name, const uint160& claim, CClaimTrieProof& proof)
+{
+    return CClaimTrieCacheExpirationFork::getProofForName(normalizeClaimName(name), claim, proof);
+}
+
+bool CClaimTrieCacheNormalizationFork::getInfoForName(const std::string& name, CClaimValue& claim, int offsetHeight) const
+{
+    return CClaimTrieCacheExpirationFork::getInfoForName(normalizeClaimName(name), claim, offsetHeight);
+}
+
+CClaimSupportToName CClaimTrieCacheNormalizationFork::getClaimsForName(const std::string& name) const
+{
+    return CClaimTrieCacheExpirationFork::getClaimsForName(normalizeClaimName(name));
+}
+
+int CClaimTrieCacheNormalizationFork::getDelayForName(const std::string& name, const uint160& claimId) const
+{
+    return CClaimTrieCacheExpirationFork::getDelayForName(normalizeClaimName(name), claimId);
+}
+
+std::string CClaimTrieCacheNormalizationFork::adjustNameForValidHeight(const std::string& name, int validHeight) const
+{
+    return normalizeClaimName(name, validHeight > base->nNormalizedNameForkHeight);
+}
+
+CClaimTrieCacheHashFork::CClaimTrieCacheHashFork(CClaimTrie* base) : CClaimTrieCacheNormalizationFork(base)
+{
+}
+
+static const auto leafHash = uint256S("0000000000000000000000000000000000000000000000000000000000000002");
+static const auto emptyHash = uint256S("0000000000000000000000000000000000000000000000000000000000000003");
+
+uint256 ComputeMerkleRoot(std::vector<uint256> hashes)
+{
+    while (hashes.size() > 1) {
+        if (hashes.size() & 1)
+            hashes.push_back(hashes.back());
+
+        sha256n_way(hashes);
+        hashes.resize(hashes.size() / 2);
+    }
+    return hashes.empty() ? uint256{} : hashes[0];
+}
+
+uint256 CClaimTrieCacheHashFork::computeNodeHash(const std::string& name, uint256& claimsHash, int takeoverHeight)
+{
+    if (nNextHeight < base->nAllClaimsInMerkleForkHeight)
+        return CClaimTrieCacheNormalizationFork::computeNodeHash(name, claimsHash, takeoverHeight);
+
+    std::vector<uint256> childHashes;
+    childHashQuery << name >> [&childHashes](std::string, uint256 hash) {
+        childHashes.push_back(std::move(hash));
+    };
+    childHashQuery++;
+
+    if (takeoverHeight > 0) {
+        if (claimsHash.IsNull()) {
+            COutPoint p;
+            std::vector<uint256> hashes;
+            for (auto&& row: claimHashQuery << nNextHeight << name) {
+                row >> p.hash >> p.n;
+                hashes.push_back(getValueHash(p, takeoverHeight));
+            }
+            claimHashQuery++;
+            claimsHash = hashes.empty() ? emptyHash : ComputeMerkleRoot(std::move(hashes));
+        }
+    } else {
+        claimsHash = emptyHash;
+    }
+
+    const auto& childrenHash = childHashes.empty() ? leafHash : ComputeMerkleRoot(std::move(childHashes));
+
+    return Hash(childrenHash.begin(), childrenHash.end(), claimsHash.begin(), claimsHash.end());
+}
+
+std::vector<uint256> ComputeMerklePath(const std::vector<uint256>& hashes, uint32_t idx)
+{
+    uint32_t count = 0;
+    int matchlevel = -1;
+    bool matchh = false;
+    uint256 inner[32], h;
+    const uint32_t one = 1;
+    std::vector<uint256> res;
+
+    const auto iterateInner = [&](int& level) {
+        for (; !(count & (one << level)); level++) {
+            const auto& ihash = inner[level];
+            if (matchh) {
+                res.push_back(ihash);
+            } else if (matchlevel == level) {
+                res.push_back(h);
+                matchh = true;
+            }
+            h = Hash(ihash.begin(), ihash.end(), h.begin(), h.end());
+        }
+    };
+
+    while (count < hashes.size()) {
+        h = hashes[count];
+        matchh = count == idx;
+        count++;
+        int level = 0;
+        iterateInner(level);
+        // Store the resulting hash at inner position level.
+        inner[level] = h;
+        if (matchh)
+            matchlevel = level;
+    }
+
+    int level = 0;
+    while (!(count & (one << level)))
+        level++;
+
+    h = inner[level];
+    matchh = matchlevel == level;
+
+    while (count != (one << level)) {
+        // If we reach this point, h is an inner value that is not the top.
+        if (matchh)
+            res.push_back(h);
+
+        h = Hash(h.begin(), h.end(), h.begin(), h.end());
+        // Increment count to the value it would have if two entries at this
+        count += (one << level);
+        level++;
+        iterateInner(level);
+    }
+    return res;
+}
+
+extern const std::string proofClaimQuery_s;
+
+bool CClaimTrieCacheHashFork::getProofForName(const std::string& name, const uint160& claim, CClaimTrieProof& proof)
+{
+    if (nNextHeight < base->nAllClaimsInMerkleForkHeight)
+        return CClaimTrieCacheNormalizationFork::getProofForName(name, claim, proof);
+
+    auto fillPairs = [&proof](const std::vector<uint256>& hashes, uint32_t idx) {
+        auto partials = ComputeMerklePath(hashes, idx);
+        for (int i = partials.size() - 1; i >= 0; --i)
+            proof.pairs.emplace_back((idx >> i) & 1, partials[i]);
+    };
+
+    // cache the parent nodes
+    getMerkleHash();
+    proof = CClaimTrieProof();
+    for (auto&& row: db << proofClaimQuery_s << name) {
+        std::string key;
+        int takeoverHeight;
+        row >> key >> takeoverHeight;
+        uint32_t nextCurrentIdx = 0;
+        std::vector<uint256> childHashes;
+        for (auto&& child : childHashQuery << key) {
+            std::string childKey;
+            uint256 childHash;
+            child >> childKey >> childHash;
+            if (name.find(childKey) == 0)
+                nextCurrentIdx = uint32_t(childHashes.size());
+            childHashes.push_back(childHash);
+        }
+        childHashQuery++;
+
+        std::vector<uint256> claimHashes;
+        uint32_t claimIdx = 0;
+        for (auto&& child: claimHashQuery << nNextHeight << key) {
+            COutPoint childOutPoint;
+            uint160 childClaimID;
+            child >> childOutPoint.hash >> childOutPoint.n >> childClaimID;
+            if (childClaimID == claim && key == name) {
+                claimIdx = uint32_t(claimHashes.size());
+                proof.outPoint = childOutPoint;
+                proof.hasValue = true;
+            }
+            claimHashes.push_back(getValueHash(childOutPoint, takeoverHeight));
+        }
+        claimHashQuery++;
+
+        // I am on a node; I need a hash(children, claims)
+        // if I am the last node on the list, it will be hash(children, x)
+        // else it will be hash(x, claims)
+        if (key == name) {
+            proof.nHeightOfLastTakeover = takeoverHeight;
+            auto hash = childHashes.empty() ? leafHash : ComputeMerkleRoot(std::move(childHashes));
+            proof.pairs.emplace_back(true, hash);
+            if (!claimHashes.empty())
+                fillPairs(claimHashes, claimIdx);
+        } else {
+            auto hash = claimHashes.empty() ? emptyHash : ComputeMerkleRoot(std::move(claimHashes));
+            proof.pairs.emplace_back(false, hash);
+            if (!childHashes.empty())
+                fillPairs(childHashes, nextCurrentIdx);
+        }
+    }
+    std::reverse(proof.pairs.begin(), proof.pairs.end());
+    return true;
+}
+
+void CClaimTrieCacheHashFork::initializeIncrement()
+{
+    CClaimTrieCacheNormalizationFork::initializeIncrement();
+    // we could do this in the constructor, but that would not allow for multiple increments in a row (as done in unit tests)
+    if (nNextHeight == base->nAllClaimsInMerkleForkHeight - 1) {
+        ensureTransacting();
+        db << "UPDATE node SET hash = NULL, claimsHash = NULL";
+    }
+}
+
+bool CClaimTrieCacheHashFork::finalizeDecrement()
+{
+    auto ret = CClaimTrieCacheNormalizationFork::finalizeDecrement();
+    if (ret && nNextHeight == base->nAllClaimsInMerkleForkHeight - 1) {
+        ensureTransacting();
+        db << "UPDATE node SET hash = NULL, claimsHash = NULL";
+    }
+    return ret;
+}
+
+bool CClaimTrieCacheHashFork::allowSupportMetadata() const
+{
+    return nNextHeight >= base->nAllClaimsInMerkleForkHeight;
+}

src/claimtrie/forks.h

@@ -0,0 +1,70 @@
+
+#ifndef CLAIMTRIE_FORKS_H
+#define CLAIMTRIE_FORKS_H
+
+#include <trie.h>
+
+class CClaimTrieCacheExpirationFork : public CClaimTrieCacheBase
+{
+public:
+    explicit CClaimTrieCacheExpirationFork(CClaimTrie* base);
+
+    int expirationTime() const override;
+
+    virtual void initializeIncrement();
+    bool incrementBlock() override;
+    bool decrementBlock() override;
+    bool finalizeDecrement() override;
+
+protected:
+    int expirationHeight;
+
+private:
+    bool forkForExpirationChange(bool increment);
+};
+
+class CClaimTrieCacheNormalizationFork : public CClaimTrieCacheExpirationFork
+{
+public:
+    explicit CClaimTrieCacheNormalizationFork(CClaimTrie* base);
+
+    bool shouldNormalize() const;
+
+    // lower-case and normalize any input string name
+    // see: https://unicode.org/reports/tr15/#Norm_Forms
+    std::string normalizeClaimName(const std::string& name, bool force = false) const; // public only for validating name field on update op
+
+    bool incrementBlock() override;
+    bool decrementBlock() override;
+
+    bool getProofForName(const std::string& name, const uint160& claim, CClaimTrieProof& proof) override;
+    bool getInfoForName(const std::string& name, CClaimValue& claim, int heightOffset = 0) const override;
+    CClaimSupportToName getClaimsForName(const std::string& name) const override;
+    std::string adjustNameForValidHeight(const std::string& name, int validHeight) const override;
+
+protected:
+    int getDelayForName(const std::string& name, const uint160& claimId) const override;
+
+private:
+    bool normalizeAllNamesInTrieIfNecessary();
+    bool unnormalizeAllNamesInTrieIfNecessary();
+};
+
+class CClaimTrieCacheHashFork : public CClaimTrieCacheNormalizationFork
+{
+public:
+    explicit CClaimTrieCacheHashFork(CClaimTrie* base);
+
+    bool getProofForName(const std::string& name, const uint160& claim, CClaimTrieProof& proof) override;
+    void initializeIncrement() override;
+    bool finalizeDecrement() override;
+
+    bool allowSupportMetadata() const;
+
+protected:
+    uint256 computeNodeHash(const std::string& name, uint256& claimsHash, int takeoverHeight) override;
+};
+
+typedef CClaimTrieCacheHashFork CClaimTrieCache;
+
+#endif // CLAIMTRIE_FORKS_H

src/claimtrie/hashes.cpp

@@ -0,0 +1,21 @@
+
+#include <hashes.h>
+
+// Bitcoin doubles hash
+uint256 CalcHash(SHA256_CTX* sha)
+{
+    uint256 result;
+    SHA256_Final(result.begin(), sha);
+    SHA256_Init(sha);
+    SHA256_Update(sha, result.begin(), result.size());
+    SHA256_Final(result.begin(), sha);
+    return result;
+}
+
+// universal N way hash function
+std::function<void(std::vector<uint256>&)> sha256n_way =
+    [](std::vector<uint256>& hashes) {
+        for (std::size_t i = 0, j = 0; i < hashes.size(); i += 2)
+            hashes[j++] = Hash(hashes[i].begin(), hashes[i].end(),
+                               hashes[i+1].begin(), hashes[i+1].end());
+    };

src/claimtrie/hashes.h

@@ -0,0 +1,33 @@
+
+#ifndef CLAIMTRIE_HASHES_H
+#define CLAIMTRIE_HASHES_H
+
+#include <openssl/sha.h>
+
+#include <functional>
+#include <vector>
+
+#include <uints.h>
+
+uint256 CalcHash(SHA256_CTX* sha);
+
+template<typename TIterator, typename... Args>
+uint256 CalcHash(SHA256_CTX* sha, TIterator begin, TIterator end, Args... args)
+{
+    static uint8_t blank;
+    SHA256_Update(sha, begin == end ? &blank : (uint8_t*)&begin[0], std::distance(begin, end) * sizeof(begin[0]));
+    return CalcHash(sha, args...);
+}
+
+template<typename TIterator, typename... Args>
+uint256 Hash(TIterator begin, TIterator end, Args... args)
+{
+    static_assert((sizeof...(args) & 1) != 1, "Parameters should be even number");
+    SHA256_CTX sha;
+    SHA256_Init(&sha);
+    return CalcHash(&sha, begin, end, args...);
+}
+
+extern std::function<void(std::vector<uint256>&)> sha256n_way;
+
+#endif // CLAIMTRIE_HASHES_H

src/claimtrie/libclaimtrie.i

@@ -0,0 +1,75 @@
+
+%module(directors="1") libclaimtrie
+%{
+#include "blob.h"
+#include "uints.h"
+#include "txoutpoint.h"
+#include "data.h"
+#include "trie.h"
+#include "forks.h"
+%}
+
+%feature("flatnested", 1);
+%feature("director") CIterateCallback;
+
+%include stl.i
+%include stdint.i
+%include typemaps.i
+
+%apply int& OUTPUT { int& nValidAtHeight };
+
+%ignore CBaseBlob(CBaseBlob &&);
+%ignore CClaimNsupports(CClaimNsupports &&);
+%ignore CClaimTrieProof(CClaimTrieProof &&);
+%ignore CClaimTrieProofNode(CClaimTrieProofNode &&);
+%ignore CClaimValue(CClaimValue &&);
+%ignore COutPoint(COutPoint &&);
+%ignore CSupportValue(CSupportValue &&);
+%ignore uint160(uint160 &&);
+%ignore uint256(uint256 &&);
+
+%template(vecUint8) std::vector<uint8_t>;
+
+%include "blob.h"
+
+%template(blob160) CBaseBlob<160>;
+%template(blob256) CBaseBlob<256>;
+
+%include "uints.h"
+%include "txoutpoint.h"
+%include "data.h"
+
+%rename(CClaimTrieCache) CClaimTrieCacheHashFork;
+
+%include "trie.h"
+%include "forks.h"
+
+%template(claimEntryType) std::vector<CClaimValue>;
+%template(supportEntryType) std::vector<CSupportValue>;
+%template(claimsNsupports) std::vector<CClaimNsupports>;
+
+%template(proofPair) std::pair<bool, uint256>;
+%template(proofNodePair) std::pair<unsigned char, uint256>;
+
+%template(proofNodes) std::vector<CClaimTrieProofNode>;
+%template(proofPairs) std::vector<std::pair<bool, uint256>>;
+%template(proofNodeChildren) std::vector<std::pair<unsigned char, uint256>>;
+
+%inline %{
+struct CIterateCallback {
+    CIterateCallback() = default;
+    virtual ~CIterateCallback() = default;
+    virtual void apply(const std::string&) = 0;
+};
+
+void getNamesInTrie(const CClaimTrieCache& cache, CIterateCallback* cb)
+{
+    cache.getNamesInTrie([cb](const std::string& name) {
+        cb->apply(name);
+    });
+}
+%}
+
+%typemap(in,numinputs=0) CClaimValue&, CClaimTrieProof& %{
+    $1 = &$input;
+%}

src/claimtrie/libclaimtrie_test.go

@@ -0,0 +1,120 @@
+
+package main
+
+import(
+    "testing"
+    ."libclaimtrie"
+)
+
+func assertEqual(t *testing.T, a interface{}, b interface{}, msg string) {
+    if a != b {
+        t.Fatalf("%s != %s, %s", a, b, msg)
+    }
+}
+
+func assertNotEqual(t *testing.T, a interface{}, b interface{}, msg string) {
+    if a == b {
+        t.Fatalf("%s == %s, %s", a, b, msg)
+    }
+}
+
+func assertFalse(t *testing.T, a interface{}, msg string) {
+    if a != false {
+        t.Fatalf("%s != false, %s", a, msg)
+    }
+}
+
+func assertTrue(t *testing.T, a interface{}, msg string) {
+    if a != true {
+        t.Fatalf("%s != true, %s", a, msg)
+    }
+}
+
+func assertClaimEqual(t *testing.T, claim CClaimValue, txo COutPoint, cid Uint160, amount int64, effe int64, height int, validHeight int,  msg string) {
+    assertEqual(t, claim.GetOutPoint(), txo, msg)
+    assertEqual(t, claim.GetClaimId(), cid, msg)
+    assertEqual(t, claim.GetNAmount(), amount, msg)
+    assertEqual(t, claim.GetNEffectiveAmount(), effe, msg)
+    assertEqual(t, claim.GetNHeight(), height, msg)
+    assertEqual(t, claim.GetNValidAtHeight(), validHeight, msg)
+}
+
+func assertSupportEqual(t *testing.T, support CSupportValue, txo COutPoint, cid Uint160, amount int64, height int, validHeight int,  msg string) {
+    assertEqual(t, support.GetOutPoint(), txo, msg)
+    assertEqual(t, support.GetSupportedClaimId(), cid, msg)
+    assertEqual(t, support.GetNAmount(), amount, msg)
+    assertEqual(t, support.GetNHeight(), height, msg)
+    assertEqual(t, support.GetNValidAtHeight(), validHeight, msg)
+}
+
+var uint256s = "1234567890987654321012345678909876543210123456789098765432101234"
+var uint160 = Uint160S("1234567890987654321012345678909876543210")
+var uint256 = Uint256S(uint256s)
+var txp = NewCOutPoint(uint256, uint(1))
+
+func Test_uint256(t *testing.T) {
+    assertFalse(t, uint256.IsNull(), "incorrect uint256S or CBaseBlob::IsNull")
+    assertEqual(t, uint256.GetHex(), uint256s, "incorrect CBaseBlob::GetHex")
+    assertEqual(t, uint256.GetHex(), uint256.ToString(), "incorrect CBaseBlob::ToString")
+    assertEqual(t, uint256.Size(), 32, "incorrect CBaseBlob::size")
+    uint256c := NewUint256()
+    assertNotEqual(t, uint256c, uint256, "incorrect CBaseBlob::operator!=")
+    assertTrue(t, uint256c.IsNull(), "incorrect CBaseBlob::IsNull")
+    uint256c = NewUint256(uint256)
+    assertEqual(t, uint256c, uint256, "incorrect CBaseBlob::operator==")
+    uint256c.SetNull()
+    assertTrue(t, uint256c.IsNull(), "incorrect CBaseBlob::SetNull")
+}
+
+func Test_txoupoint(t *testing.T) {
+    assertEqual(t, txp.GetHash(), uint256, "incorrect COutPoint::COutPoint")
+    assertEqual(t, txp.GetN(), 1, "incorrect COutPoint::COutPoint")
+    assertFalse(t, txp.IsNull(), "incorrect COutPoint::IsNull")
+    pcopy := NewCOutPoint()
+    assertTrue(t, pcopy.IsNull(), "incorrect COutPoint::IsNull")
+    assertEqual(t, pcopy.GetHash(), NewUint256(), "incorrect COutPoint::COutPoint")
+    assertNotEqual(t, pcopy, txp, "incorrect COutPoint::operator!=")
+}
+
+func Test_claim(t *testing.T) {
+    assertEqual(t, uint160.Size(), 20, "incorrect CBaseBlob::size")
+    claim := NewCClaimValue(txp, uint160, 20, 1, 10)
+    assertClaimEqual(t, claim, txp, uint160, 20, 20, 1, 10, "incorrect CClaimValue::CClaimValue")
+}
+
+func Test_support(t *testing.T) {
+    claim := NewCClaimValue(txp, uint160, 20, 1, 10)
+    support := NewCSupportValue(txp, uint160, 20, 1, 10)
+    assertSupportEqual(t, support, claim.GetOutPoint(), claim.GetClaimId(), claim.GetNAmount(), claim.GetNHeight(), claim.GetNValidAtHeight(), "incorrect CSupportValue::CSupportValue")
+}
+
+func Test_claimtrie(t *testing.T) {
+    claim := NewCClaimValue(txp, uint160, 20, 0, 0)
+    wipe := true; height := 1; data_dir := "."
+    trie := NewCClaimTrie(10*1024*1024, wipe, height, data_dir)
+    cache := NewCClaimTrieCache(trie)
+    assertTrue(t, trie.Empty(), "incorrect CClaimtrieCache::empty")
+    assertTrue(t, cache.AddClaim("test", txp, uint160, 20, 0, 0), "incorrect CClaimtrieCache::addClaim")
+    assertTrue(t, cache.HaveClaim("test", txp), "incorrect CClaimtrieCache::haveClaim")
+    assertEqual(t, cache.GetTotalNamesInTrie(), 1, "incorrect CClaimtrieCache::getTotalNamesInTrie")
+    assertEqual(t, cache.GetTotalClaimsInTrie(), 1, "incorrect CClaimtrieCache::getTotalClaimsInTrie")
+    var nValidAtHeight []int
+    // add second claim
+    txp.SetN(2)
+    uint1601 := Uint160S("1234567890987654321012345678909876543211")
+    assertTrue(t, cache.AddClaim("test", txp, uint1601, 20, 1, 1), "incorrect CClaimtrieCache::addClaim")
+    result := cache.HaveClaimInQueue("test", txp, nValidAtHeight)
+    assertTrue(t, result, "incorrect CClaimTrieCache::haveClaimInQueue")
+    assertEqual(t, nValidAtHeight[0], 1, "incorrect CClaimTrieCache::haveClaimInQueue, nValidAtHeight")
+    claim1 := NewCClaimValue()
+    assertTrue(t, cache.GetInfoForName("test", claim1), "incorrect CClaimTrieCache::getInfoForName")
+    assertEqual(t, claim, claim1, "incorrect CClaimtrieCache::getInfoForName")
+    proof := NewCClaimTrieProof()
+    assertTrue(t, cache.GetProofForName("test", uint160, proof), "incorrect CacheProofCallback")
+    assertTrue(t, proof.GetHasValue(), "incorrect CClaimTrieCache::getProofForName")
+    claimsToName := cache.GetClaimsForName("test")
+    claims := claimsToName.GetClaimsNsupports()
+    assertEqual(t, claims.Size(), 2, "incorrect CClaimTrieCache::getClaimsForName")
+    assertFalse(t, claims.Get(0).IsNull(), "incorrect CClaimNsupports::IsNull")
+    assertFalse(t, claims.Get(1).IsNull(), "incorrect CClaimNsupports::IsNull")
+}

src/claimtrie/libclaimtrie_test.py

@@ -0,0 +1,108 @@
+
+from libclaimtrie import *
+import unittest
+
+class CacheIterateCallback(CIterateCallback):
+    def __init__(self, names):
+        CIterateCallback.__init__(self)
+        self.names = names
+
+    def apply(self, name):
+        assert(name in self.names), "Incorrect trie names"
+
+class TestClaimTrieTypes(unittest.TestCase):
+    def setUp(self):
+        self.uint256s = "1234567890987654321012345678909876543210123456789098765432101234"
+        self.uint160 = uint160S("1234567890987654321012345678909876543210")
+        self.uint = uint256S(self.uint256s)
+        self.txp = COutPoint(self.uint, 1)
+
+    def assertClaimEqual(self, claim, txo, cid, amount, effe, height, validHeight, msg):
+        self.assertEqual(claim.outPoint, txo, msg)
+        self.assertEqual(claim.claimId, cid, msg)
+        self.assertEqual(claim.nAmount, amount, msg)
+        self.assertEqual(claim.nEffectiveAmount, effe, msg)
+        self.assertEqual(claim.nHeight, height, msg)
+        self.assertEqual(claim.nValidAtHeight, validHeight, msg)
+
+    def assertSupportEqual(self, support, txo, cid, amount, height, validHeight, msg):
+        self.assertEqual(support.outPoint, txo, msg)
+        self.assertEqual(support.supportedClaimId, cid, msg)
+        self.assertEqual(support.nAmount, amount, msg)
+        self.assertEqual(support.nHeight, height, msg)
+        self.assertEqual(support.nValidAtHeight, validHeight, msg)
+
+    def test_uint256(self):
+        uint = self.uint
+        self.assertFalse(uint.IsNull(), "incorrect uint256S or CBaseBlob::IsNull")
+        self.assertEqual(uint.GetHex(), self.uint256s, "incorrect CBaseBlob::GetHex")
+        self.assertEqual(uint.GetHex(), uint.ToString(), "incorrect CBaseBlob::ToString")
+        self.assertEqual(uint.size(), 32, "incorrect CBaseBlob::size")
+        copy = uint256()
+        self.assertNotEqual(copy, uint, "incorrect CBaseBlob::operator!=")
+        self.assertTrue(copy.IsNull(), "incorrect CBaseBlob::IsNull")
+        copy = uint256(uint)
+        self.assertEqual(copy, uint, "incorrect CBaseBlob::operator==")
+        copy.SetNull()
+        self.assertTrue(copy.IsNull()), "incorrect CBaseBlob::SetNull"
+
+    def test_txoupoint(self):
+        txp = self.txp
+        uint = self.uint
+        self.assertEqual(txp.hash, uint, "incorrect COutPoint::COutPoint")
+        self.assertEqual(txp.n, 1, "incorrect COutPoint::COutPoint")
+        self.assertFalse(txp.IsNull(), "incorrect COutPoint::IsNull")
+        pcopy = COutPoint()
+        self.assertTrue(pcopy.IsNull(), "incorrect COutPoint::IsNull")
+        self.assertEqual(pcopy.hash, uint256(), "incorrect COutPoint::COutPoint")
+        self.assertNotEqual(pcopy, txp, "incorrect COutPoint::operator!=")
+        self.assertIn(uint.ToString()[:10], txp.ToString(), "incorrect COutPoint::ToString")
+
+    def test_claim(self):
+        txp = self.txp
+        uint160 = self.uint160
+        self.assertEqual(uint160.size(), 20, "incorrect CBaseBlob::size")
+        claim = CClaimValue(txp, uint160, 20, 1, 10)
+        self.assertClaimEqual(claim, txp, uint160, 20, 20, 1, 10, "incorrect CClaimValue::CClaimValue")
+
+    def test_support(self):
+        txp = self.txp
+        uint160 = self.uint160
+        claim = CClaimValue(txp, uint160, 20, 1, 10)
+        support = CSupportValue(txp, uint160, 20, 1, 10)
+        self.assertSupportEqual(support, claim.outPoint, claim.claimId, claim.nAmount, claim.nHeight, claim.nValidAtHeight, "incorrect CSupportValue::CSupportValue")
+
+    def test_claimtrie(self):
+        txp = self.txp
+        uint160 = self.uint160
+        claim = CClaimValue(txp, uint160, 20, 0, 0)
+        wipe = True; height = 1; data_dir = "."
+        trie = CClaimTrie(10*1024*1024, wipe, height, data_dir)
+        cache = CClaimTrieCache(trie)
+        self.assertTrue(trie.empty(), "incorrect CClaimtrieCache::empty")
+        self.assertTrue(cache.addClaim("test", txp, uint160, 20, 0, 0), "incorrect CClaimtrieCache::addClaim")
+        self.assertTrue(cache.haveClaim("test", txp), "incorrect CClaimtrieCache::haveClaim")
+        self.assertEqual(cache.getTotalNamesInTrie(), 1, "incorrect CClaimtrieCache::getTotalNamesInTrie")
+        self.assertEqual(cache.getTotalClaimsInTrie(), 1, "incorrect CClaimtrieCache::getTotalClaimsInTrie")
+        getNamesInTrie(cache, CacheIterateCallback(["test"]))
+        nValidAtHeight = -1
+        # add second claim
+        txp.n = 2
+        uint1601 = uint160S("1234567890987654321012345678909876543211")
+        self.assertTrue(cache.addClaim("test", txp, uint1601, 20, 1, 1), "incorrect CClaimtrieCache::addClaim")
+        result, nValidAtHeight = cache.haveClaimInQueue("test", txp)
+        self.assertTrue(result, "incorrect CClaimTrieCache::haveClaimInQueue")
+        self.assertEqual(nValidAtHeight, 1, "incorrect CClaimTrieCache::haveClaimInQueue, nValidAtHeight")
+        claim1 = CClaimValue()
+        self.assertTrue(cache.getInfoForName("test", claim1), "incorrect CClaimTrieCache::getInfoForName")
+        self.assertEqual(claim, claim1, "incorrect CClaimtrieCache::getInfoForName")
+        proof = CClaimTrieProof()
+        self.assertTrue(cache.getProofForName("test", uint160, proof), "incorrect CacheProofCallback")
+        self.assertTrue(proof.hasValue, "incorrect CClaimTrieCache::getProofForName")
+        claimsToName = cache.getClaimsForName("test")
+        claims = claimsToName.claimsNsupports
+        self.assertEqual(claims.size(), 2, "incorrect CClaimTrieCache::getClaimsForName")
+        self.assertFalse(claims[0].IsNull(), "incorrect CClaimNsupports::IsNull")
+        self.assertFalse(claims[1].IsNull(), "incorrect CClaimNsupports::IsNull")
+
+unittest.main()

src/claimtrie/log.cpp

@@ -0,0 +1,30 @@
+
+#include <log.h>
+
+void CLogPrint::setLogger(ClogBase* log)
+{
+    ss.str({});
+    logger = log;
+}
+
+CLogPrint& CLogPrint::global()
+{
+    static CLogPrint logger;
+    return logger;
+}
+
+CLogPrint& CLogPrint::operator<<(const Clog& cl)
+{
+    if (logger) {
+        switch(cl) {
+        case Clog::endl:
+            ss << '\n';
+            // fallthrough
+        case Clog::flush:
+            logger->LogPrintStr(ss.str());
+            ss.str({});
+            break;
+        }
+    }
+    return *this;
+}

src/claimtrie/log.h

@@ -0,0 +1,42 @@
+
+#ifndef CLAIMTRIE_LOG_H
+#define CLAIMTRIE_LOG_H
+
+#include <string>
+#include <sstream>
+
+struct ClogBase
+{
+    ClogBase() = default;
+    virtual ~ClogBase() = default;
+    virtual void LogPrintStr(const std::string&) = 0;
+};
+
+enum struct Clog
+{
+    endl = 0,
+    flush = 1,
+};
+
+struct CLogPrint
+{
+    template <typename T>
+    CLogPrint& operator<<(const T& a)
+    {
+        if (logger)
+            ss << a;
+        return *this;
+    }
+
+    CLogPrint& operator<<(const Clog& cl);
+
+    void setLogger(ClogBase* log);
+    static CLogPrint& global();
+
+private:
+    CLogPrint() = default;
+    std::stringstream ss;
+    ClogBase* logger = nullptr;
+};
+
+#endif // CLAIMTRIE_LOG_H

src/claimtrie/sqlite.h

@@ -0,0 +1,125 @@
+
+#ifndef SQLITE_H
+#define SQLITE_H
+
+#include <sqlite/sqlite3.h>
+#include <uints.h>
+
+#include <chrono>
+#include <thread>
+
+ namespace sqlite
+{
+    inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, const uint160& val) {
+        return sqlite3_bind_blob(stmt, inx, val.begin(), int(val.size()), SQLITE_STATIC);
+    }
+
+    inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, uint160&& val) {
+        return sqlite3_bind_blob(stmt, inx, val.begin(), int(val.size()), SQLITE_TRANSIENT);
+    }
+
+    inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, const uint256& val) {
+        return sqlite3_bind_blob(stmt, inx, val.begin(), int(val.size()), SQLITE_STATIC);
+    }
+
+    inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, uint256&& val) {
+        return sqlite3_bind_blob(stmt, inx, val.begin(), int(val.size()), SQLITE_TRANSIENT);
+    }
+
+    inline void store_result_in_db(sqlite3_context* db, const uint160& val) {
+        sqlite3_result_blob(db, val.begin(), int(val.size()), SQLITE_STATIC);
+    }
+
+    inline void store_result_in_db(sqlite3_context* db, uint160&& val) {
+        sqlite3_result_blob(db, val.begin(), int(val.size()), SQLITE_TRANSIENT);
+    }
+
+    inline void store_result_in_db(sqlite3_context* db, const uint256& val) {
+        sqlite3_result_blob(db, val.begin(), int(val.size()), SQLITE_STATIC);
+    }
+
+    inline void store_result_in_db(sqlite3_context* db, uint256&& val) {
+        sqlite3_result_blob(db, val.begin(), int(val.size()), SQLITE_TRANSIENT);
+    }
+}
+
+#include <sqlite/hdr/sqlite_modern_cpp.h>
+
+namespace sqlite
+{
+    template<>
+    struct has_sqlite_type<uint256, SQLITE_BLOB, void> : std::true_type {};
+
+    template<>
+    struct has_sqlite_type<uint160, SQLITE_BLOB, void> : std::true_type {};
+
+    inline uint160 get_col_from_db(sqlite3_stmt* stmt, int inx, result_type<uint160>) {
+        uint160 ret;
+        auto ptr = sqlite3_column_blob(stmt, inx);
+        if (!ptr) return ret;
+        int bytes = sqlite3_column_bytes(stmt, inx);
+        assert(bytes > 0 && bytes <= int(ret.size()));
+        std::memcpy(ret.begin(), ptr, bytes);
+        return ret;
+    }
+
+    inline uint256 get_col_from_db(sqlite3_stmt* stmt, int inx, result_type<uint256>) {
+        uint256 ret;
+
+// I think we need this, but I lost my specific test case:
+//        auto type = sqlite3_column_type(stmt, inx);
+//        if (type == SQLITE_NULL)
+//            return ret;
+//
+//        if (type == SQLITE_INTEGER) {
+//            auto integer = sqlite3_column_int64(stmt, inx);
+//            return uint256(integer);
+//        }
+//        assert(type == SQLITE_BLOB);
+
+        auto ptr = sqlite3_column_blob(stmt, inx);
+        if (!ptr) return ret;
+        int bytes = sqlite3_column_bytes(stmt, inx);
+        assert(bytes <= ret.size());
+        std::memcpy(ret.begin(), ptr, bytes);
+        return ret;
+    }
+
+    inline int commit(database& db, std::size_t attempts = 200)
+    {
+        int code = SQLITE_OK;
+        for (auto i = 0u; i < attempts; ++i) {
+            try {
+                db << "COMMIT";
+            } catch (const sqlite_exception& e) {
+                code = e.get_code();
+                if (code == SQLITE_LOCKED || code == SQLITE_BUSY) {
+                    using namespace std::chrono_literals;
+                    std::this_thread::sleep_for(100ms);
+                    continue;
+                }
+                return code;
+            }
+            return SQLITE_OK;
+        }
+        return code;
+    }
+
+    inline int sync(database& db, std::size_t attempts = 20)
+    {
+        int code = SQLITE_OK;
+        for (auto i = 0u; i < attempts; ++i) {
+            code = sqlite3_wal_checkpoint_v2(db.connection().get(), nullptr, SQLITE_CHECKPOINT_FULL, nullptr, nullptr);
+            if (code != SQLITE_OK) {
+                sqlite3_wal_checkpoint_v2(db.connection().get(), nullptr, SQLITE_CHECKPOINT_PASSIVE, nullptr, nullptr);
+                using namespace std::chrono_literals;
+                std::this_thread::sleep_for(200ms);
+                continue;
+            }
+            break;
+        }
+        return code;
+    }
+}
+
+#endif // SQLITE_H

src/claimtrie/sqlite/hdr/sqlite_modern_cpp.h

@@ -0,0 +1,682 @@
+#pragma once
+
+#include <algorithm>
+#include <cctype>
+#include <string>
+#include <functional>
+#include <tuple>
+#include <memory>
+
+#define MODERN_SQLITE_VERSION 3002008
+
+#include "../sqlite3.h"
+
+#include "sqlite_modern_cpp/type_wrapper.h"
+#include "sqlite_modern_cpp/errors.h"
+#include "sqlite_modern_cpp/utility/function_traits.h"
+#include "sqlite_modern_cpp/utility/uncaught_exceptions.h"
+#include "sqlite_modern_cpp/utility/utf16_utf8.h"
+
+namespace sqlite {
+
+    class database;
+    class database_binder;
+
+    template<std::size_t> class binder;
+
+    typedef std::shared_ptr<sqlite3> connection_type;
+
+    template<class T, bool Name = false>
+    struct index_binding_helper {
+        index_binding_helper(const index_binding_helper &) = delete;
+#if __cplusplus < 201703
+        index_binding_helper(index_binding_helper &&) = default;
+#endif
+        typename std::conditional<Name, const char *, int>::type index;
+        T value;
+    };
+
+    template<class T>
+    auto named_parameter(const char *name, T &&arg) {
+        return index_binding_helper<decltype(arg), true>{name, std::forward<decltype(arg)>(arg)};
+    }
+    template<class T>
+    auto indexed_parameter(int index, T &&arg) {
+        return index_binding_helper<decltype(arg)>{index, std::forward<decltype(arg)>(arg)};
+    }
+
+    class row_iterator;
+    class database_binder {
+
+    public:
+        // database_binder is not copyable
+        database_binder() = delete;
+        database_binder(const database_binder& other) = delete;
+        database_binder& operator=(const database_binder&) = delete;
+
+        database_binder(database_binder&& other) :
+                _db(std::move(other._db)),
+                _stmt(std::move(other._stmt)),
+                _inx(other._inx), execution_started(other.execution_started) { }
+
+        void execute();
+
+        std::string sql() {
+#if SQLITE_VERSION_NUMBER >= 3014000
+            auto sqlite_deleter = [](void *ptr) {sqlite3_free(ptr);};
+            std::unique_ptr<char, decltype(sqlite_deleter)> str(sqlite3_expanded_sql(_stmt.get()), sqlite_deleter);
+            return str ? str.get() : original_sql();
+#else
+            return original_sql();
+#endif
+        }
+
+        std::string original_sql() {
+            return sqlite3_sql(_stmt.get());
+        }
+
+        void used(bool state) {
+            if(!state) {
+                // We may have to reset first if we haven't done so already:
+                _next_index();
+                --_inx;
+            }
+            execution_started = state;
+        }
+        bool used() const { return execution_started; }
+        row_iterator begin();
+        row_iterator end();
+
+    private:
+        std::shared_ptr<sqlite3> _db;
+        std::unique_ptr<sqlite3_stmt, decltype(&sqlite3_finalize)> _stmt;
+        utility::UncaughtExceptionDetector _has_uncaught_exception;
+
+        int _inx;
+
+        bool execution_started = false;
+
+        int _next_index() {
+            if(execution_started && !_inx) {
+                sqlite3_reset(_stmt.get());
+                sqlite3_clear_bindings(_stmt.get());
+            }
+            return ++_inx;
+        }
+
+        sqlite3_stmt* _prepare(u16str_ref sql) {
+            return _prepare(utility::utf16_to_utf8(sql));
+        }
+
+        sqlite3_stmt* _prepare(str_ref sql) {
+            int hresult;
+            sqlite3_stmt* tmp = nullptr;
+            const char *remaining;
+            hresult = sqlite3_prepare_v2(_db.get(), sql.data(), sql.length(), &tmp, &remaining);
+            if(hresult != SQLITE_OK) errors::throw_sqlite_error(hresult, sql, sqlite3_errmsg(_db.get()));
+            if(!std::all_of(remaining, sql.data() + sql.size(), [](char ch) {return std::isspace(ch);}))
+                throw errors::more_statements("Multiple semicolon separated statements are unsupported", sql);
+            return tmp;
+        }
+
+        template<typename T> friend database_binder& operator<<(database_binder& db, T&&);
+        template<typename T> friend database_binder& operator<<(database_binder& db, index_binding_helper<T>);
+        template<typename T> friend database_binder& operator<<(database_binder& db, index_binding_helper<T, true>);
+        friend void operator++(database_binder& db, int);
+
+    public:
+
+        database_binder(std::shared_ptr<sqlite3> db, u16str_ref sql):
+                _db(db),
+                _stmt(_prepare(sql), sqlite3_finalize),
+                _inx(0) {
+        }
+
+        database_binder(std::shared_ptr<sqlite3> db, str_ref sql):
+                _db(db),
+                _stmt(_prepare(sql), sqlite3_finalize),
+                _inx(0) {
+        }
+
+        ~database_binder() noexcept(false) {
+            /* Will be executed if no >>op is found, but not if an exception
+            is in mid flight */
+            if(!used() && !_has_uncaught_exception && _stmt) {
+                execute();
+            }
+        }
+
+        friend class row_iterator;
+    };
+
+    class row_iterator {
+    public:
+        class value_type {
+        public:
+            value_type(database_binder *_binder): _binder(_binder) {};
+            template<class T>
+            typename std::enable_if<is_sqlite_value<T>::value, value_type &>::type operator >>(T &result) {
+                result = get_col_from_db(_binder->_stmt.get(), next_index++, result_type<T>());
+                return *this;
+            }
+            template<class ...Types>
+            value_type &operator >>(std::tuple<Types...>& values) {
+                values = handle_tuple<std::tuple<typename std::decay<Types>::type...>>(std::index_sequence_for<Types...>());
+                next_index += sizeof...(Types);
+                return *this;
+            }
+            template<class ...Types>
+            value_type &operator >>(std::tuple<Types...>&& values) {
+                return *this >> values;
+            }
+            template<class ...Types>
+            operator std::tuple<Types...>() {
+                std::tuple<Types...> value;
+                *this >> value;
+                return value;
+            }
+            explicit operator bool() {
+                return sqlite3_column_count(_binder->_stmt.get()) >= next_index;
+            }
+            int& index() { return next_index; }
+        private:
+            template<class Tuple, std::size_t ...Index>
+            Tuple handle_tuple(std::index_sequence<Index...>) {
+                return Tuple(
+                        get_col_from_db(
+                                _binder->_stmt.get(),
+                                next_index + Index,
+                                result_type<typename std::tuple_element<Index, Tuple>::type>())...);
+            }
+            database_binder *_binder;
+            int next_index = 0;
+        };
+        using difference_type = std::ptrdiff_t;
+        using pointer = value_type*;
+        using reference = value_type&;
+        using iterator_category = std::input_iterator_tag;
+
+        row_iterator() = default;
+        explicit row_iterator(database_binder &binder): _binder(&binder) {
+            _binder->_next_index();
+            _binder->_inx = 0;
+            _binder->used(true);
+            ++*this;
+        }
+
+        reference operator*() const { return value;}
+        pointer operator->() const { return std::addressof(**this); }
+        row_iterator &operator++() {
+            switch(int result = sqlite3_step(_binder->_stmt.get())) {
+                case SQLITE_ROW:
+                    value = {_binder};
+                    break;
+                case SQLITE_DONE:
+                    _binder = nullptr;
+                    break;
+                default:
+                    exceptions::throw_sqlite_error(result, _binder->sql(), sqlite3_errmsg(_binder->_db.get()));
+            }
+            return *this;
+        }
+
+        friend inline bool operator ==(const row_iterator &a, const row_iterator &b) {
+            return a._binder == b._binder;
+        }
+        friend inline bool operator !=(const row_iterator &a, const row_iterator &b) {
+            return !(a==b);
+        }
+
+    private:
+        database_binder *_binder = nullptr;
+        mutable value_type value{_binder}; // mutable, because `changing` the value is just reading it
+    };
+
+    inline row_iterator database_binder::begin() {
+        return row_iterator(*this);
+    }
+
+    inline row_iterator database_binder::end() {
+        return row_iterator();
+    }
+
+    namespace detail {
+        template<class Callback>
+        void _extract_single_value(database_binder &binder, Callback call_back) {
+            auto iter = binder.begin();
+            if(iter == binder.end())
+                throw errors::no_rows("no rows to extract: exactly 1 row expected", binder.sql(), SQLITE_DONE);
+
+            call_back(*iter);
+
+            if(++iter != binder.end())
+                throw errors::more_rows("not all rows extracted", binder.sql(), SQLITE_ROW);
+        }
+    }
+    inline void database_binder::execute() {
+        for(auto &&row : *this)
+            (void)row;
+    }
+    namespace detail {
+        template<class T> using void_t = void;
+        template<class T, class = void>
+        struct sqlite_direct_result : std::false_type {};
+        template<class T>
+        struct sqlite_direct_result<
+                T,
+                void_t<decltype(std::declval<row_iterator::value_type&>() >> std::declval<T&&>())>
+        > : std::true_type {};
+    }
+    template <typename Result>
+    inline typename std::enable_if<detail::sqlite_direct_result<Result>::value>::type operator>>(database_binder &binder, Result&& value) {
+        detail::_extract_single_value(binder, [&value] (row_iterator::value_type &row) {
+            row >> std::forward<Result>(value);
+        });
+    }
+
+    template <typename Function>
+    inline typename std::enable_if<!detail::sqlite_direct_result<Function>::value>::type operator>>(database_binder &db_binder, Function&& func) {
+        using traits = utility::function_traits<Function>;
+
+        for(auto &&row : db_binder) {
+            binder<traits::arity>::run(row, func);
+        }
+    }
+
+    template <typename Result>
+    inline decltype(auto) operator>>(database_binder &&binder, Result&& value) {
+        return binder >> std::forward<Result>(value);
+    }
+
+    namespace sql_function_binder {
+        template<
+                typename    ContextType,
+                std::size_t Count,
+                typename    Functions
+        >
+        inline void step(
+                sqlite3_context* db,
+                int              count,
+                sqlite3_value**  vals
+        );
+
+        template<
+                std::size_t Count,
+                typename    Functions,
+                typename... Values
+        >
+        inline typename std::enable_if<(sizeof...(Values) && sizeof...(Values) < Count), void>::type step(
+                sqlite3_context* db,
+                int              count,
+                sqlite3_value**  vals,
+                Values&&...      values
+        );
+
+        template<
+                std::size_t Count,
+                typename    Functions,
+                typename... Values
+        >
+        inline typename std::enable_if<(sizeof...(Values) == Count), void>::type step(
+                sqlite3_context* db,
+                int,
+                sqlite3_value**,
+                Values&&...      values
+        );
+
+        template<
+                typename    ContextType,
+                typename    Functions
+        >
+        inline void final(sqlite3_context* db);
+
+        template<
+                std::size_t Count,
+                typename    Function,
+                typename... Values
+        >
+        inline typename std::enable_if<(sizeof...(Values) < Count), void>::type scalar(
+                sqlite3_context* db,
+                int              count,
+                sqlite3_value**  vals,
+                Values&&...      values
+        );
+
+        template<
+                std::size_t Count,
+                typename    Function,
+                typename... Values
+        >
+        inline typename std::enable_if<(sizeof...(Values) == Count), void>::type scalar(
+                sqlite3_context* db,
+                int,
+                sqlite3_value**,
+                Values&&...      values
+        );
+    }
+
+    enum class OpenFlags {
+        READONLY = SQLITE_OPEN_READONLY,
+        READWRITE = SQLITE_OPEN_READWRITE,
+        CREATE = SQLITE_OPEN_CREATE,
+        NOMUTEX = SQLITE_OPEN_NOMUTEX,
+        FULLMUTEX = SQLITE_OPEN_FULLMUTEX,
+        SHAREDCACHE = SQLITE_OPEN_SHAREDCACHE,
+        PRIVATECACH = SQLITE_OPEN_PRIVATECACHE,
+        URI = SQLITE_OPEN_URI
+    };
+    inline OpenFlags operator|(const OpenFlags& a, const OpenFlags& b) {
+        return static_cast<OpenFlags>(static_cast<int>(a) | static_cast<int>(b));
+    }
+    enum class Encoding {
+        ANY = SQLITE_ANY,
+        UTF8 = SQLITE_UTF8,
+        UTF16 = SQLITE_UTF16
+    };
+    struct sqlite_config {
+        OpenFlags flags = OpenFlags::READWRITE | OpenFlags::CREATE;
+        const char *zVfs = nullptr;
+        Encoding encoding = Encoding::ANY;
+    };
+
+    class database {
+    protected:
+        std::shared_ptr<sqlite3> _db;
+
+    public:
+        database(const std::string &db_name, const sqlite_config &config = {}): _db(nullptr) {
+            sqlite3* tmp = nullptr;
+            auto ret = sqlite3_open_v2(db_name.data(), &tmp, static_cast<int>(config.flags), config.zVfs);
+            _db = std::shared_ptr<sqlite3>(tmp, [=](sqlite3* ptr) { sqlite3_close_v2(ptr); }); // this will close the connection eventually when no longer needed.
+            if(ret != SQLITE_OK) errors::throw_sqlite_error(_db ? sqlite3_extended_errcode(_db.get()) : ret, {}, sqlite3_errmsg(_db.get()));
+            sqlite3_extended_result_codes(_db.get(), true);
+            if(config.encoding == Encoding::UTF16)
+                *this << R"(PRAGMA encoding = "UTF-16";)";
+        }
+
+        database(const std::u16string &db_name, const sqlite_config &config = {}): database(utility::utf16_to_utf8(db_name), config) {
+            if (config.encoding == Encoding::ANY)
+                *this << R"(PRAGMA encoding = "UTF-16";)";
+        }
+
+        database(std::shared_ptr<sqlite3> db):
+                _db(db) {}
+
+        database_binder operator<<(str_ref sql) {
+            return database_binder(_db, sql);
+        }
+
+        database_binder operator<<(u16str_ref sql) {
+            return database_binder(_db, sql);
+        }
+
+        connection_type connection() const { return _db; }
+
+        sqlite3_int64 last_insert_rowid() const {
+            return sqlite3_last_insert_rowid(_db.get());
+        }
+
+        int rows_modified() const {
+            return sqlite3_changes(_db.get());
+        }
+
+        template <typename Function>
+        void define(const std::string &name, Function&& func, bool deterministic = true) {
+            typedef utility::function_traits<Function> traits;
+
+            auto funcPtr = new auto(std::forward<Function>(func));
+            if(int result = sqlite3_create_function_v2(
+                    _db.get(), name.data(), traits::arity, SQLITE_UTF8 | (deterministic ? SQLITE_DETERMINISTIC : 0), funcPtr,
+                    sql_function_binder::scalar<traits::arity, typename std::remove_reference<Function>::type>,
+                    nullptr, nullptr, [](void* ptr){
+                        delete static_cast<decltype(funcPtr)>(ptr);
+                    }))
+                errors::throw_sqlite_error(result, {}, sqlite3_errmsg(_db.get()));
+        }
+
+        template <typename StepFunction, typename FinalFunction>
+        void define(const std::string &name, StepFunction&& step, FinalFunction&& final, bool deterministic = true) {
+            typedef utility::function_traits<StepFunction> traits;
+            using ContextType = typename std::remove_reference<typename traits::template argument<0>>::type;
+
+            auto funcPtr = new auto(std::make_pair(std::forward<StepFunction>(step), std::forward<FinalFunction>(final)));
+            if(int result = sqlite3_create_function_v2(
+                    _db.get(), name.c_str(), traits::arity - 1, SQLITE_UTF8 | (deterministic ? SQLITE_DETERMINISTIC : 0), funcPtr, nullptr,
+                    sql_function_binder::step<ContextType, traits::arity, typename std::remove_reference<decltype(*funcPtr)>::type>,
+                    sql_function_binder::final<ContextType, typename std::remove_reference<decltype(*funcPtr)>::type>,
+                    [](void* ptr){
+                        delete static_cast<decltype(funcPtr)>(ptr);
+                    }))
+                errors::throw_sqlite_error(result, {}, sqlite3_errmsg(_db.get()));
+        }
+
+    };
+
+    template<std::size_t Count>
+    class binder {
+    private:
+        template <
+                typename    Function,
+                std::size_t Index
+        >
+        using nth_argument_type = typename utility::function_traits<
+                Function
+        >::template argument<Index>;
+
+    public:
+        // `Boundary` needs to be defaulted to `Count` so that the `run` function
+        // template is not implicitly instantiated on class template instantiation.
+        // Look up section 14.7.1 _Implicit instantiation_ of the ISO C++14 Standard
+        // and the [dicussion](https://github.com/aminroosta/sqlite_modern_cpp/issues/8)
+        // on Github.
+
+        template<
+                typename    Function,
+                typename... Values,
+                std::size_t Boundary = Count
+        >
+        static typename std::enable_if<(sizeof...(Values) < Boundary), void>::type run(
+                row_iterator::value_type& row,
+                Function&&                function,
+                Values&&...               values
+        ) {
+            typename std::decay<nth_argument_type<Function, sizeof...(Values)>>::type value;
+            row >> value;
+            run<Function>(row, function, std::forward<Values>(values)..., std::move(value));
+        }
+
+        template<
+                typename    Function,
+                typename... Values,
+                std::size_t Boundary = Count
+        >
+        static typename std::enable_if<(sizeof...(Values) == Boundary), void>::type run(
+                row_iterator::value_type&,
+                Function&&                function,
+                Values&&...               values
+        ) {
+            function(std::move(values)...);
+        }
+    };
+
+    // Some ppl are lazy so we have a operator for proper prep. statemant handling.
+    void inline operator++(database_binder& db, int) { db.execute(); }
+
+    template<typename T> database_binder &operator<<(database_binder& db, index_binding_helper<T> val) {
+        db._next_index(); --db._inx;
+        int result = bind_col_in_db(db._stmt.get(), val.index, std::forward<T>(val.value));
+        if(result != SQLITE_OK)
+            exceptions::throw_sqlite_error(result, db.sql(), sqlite3_errmsg(db._db.get()));
+        return db;
+    }
+
+    template<typename T> database_binder &operator<<(database_binder& db, index_binding_helper<T, true> val) {
+        db._next_index(); --db._inx;
+        int index = sqlite3_bind_parameter_index(db._stmt.get(), val.index);
+        if(!index)
+            throw errors::unknown_binding("The given binding name is not valid for this statement", db.sql());
+        int result = bind_col_in_db(db._stmt.get(), index, std::forward<T>(val.value));
+        if(result != SQLITE_OK)
+            exceptions::throw_sqlite_error(result, db.sql(), sqlite3_errmsg(db._db.get()));
+        return db;
+    }
+
+    template<typename T> database_binder &operator<<(database_binder& db, T&& val) {
+        int result = bind_col_in_db(db._stmt.get(), db._next_index(), std::forward<T>(val));
+        if(result != SQLITE_OK)
+            exceptions::throw_sqlite_error(result, db.sql(), sqlite3_errmsg(db._db.get()));
+        return db;
+    }
+    // Convert the rValue binder to a reference and call first op<<, its needed for the call that creates the binder (be carefull of recursion here!)
+    template<typename T> database_binder operator << (database_binder&& db, const T& val) { db << val; return std::move(db); }
+    template<typename T, bool Name> database_binder operator << (database_binder&& db, index_binding_helper<T, Name> val) { db << index_binding_helper<T, Name>{val.index, std::forward<T>(val.value)}; return std::move(db); }
+
+    namespace sql_function_binder {
+        template<class T>
+        struct AggregateCtxt {
+            T obj;
+            bool constructed = true;
+        };
+
+        template<
+                typename ContextType,
+                std::size_t Count,
+                typename    Functions
+        >
+        inline void step(
+                sqlite3_context* db,
+                int              count,
+                sqlite3_value**  vals
+        ) {
+            auto ctxt = static_cast<AggregateCtxt<ContextType>*>(sqlite3_aggregate_context(db, sizeof(AggregateCtxt<ContextType>)));
+            if(!ctxt) return;
+            try {
+                if(!ctxt->constructed) new(ctxt) AggregateCtxt<ContextType>();
+                step<Count, Functions>(db, count, vals, ctxt->obj);
+                return;
+            } catch(const sqlite_exception &e) {
+                sqlite3_result_error_code(db, e.get_code());
+                sqlite3_result_error(db, e.what(), -1);
+            } catch(const std::exception &e) {
+                sqlite3_result_error(db, e.what(), -1);
+            } catch(...) {
+                sqlite3_result_error(db, "Unknown error", -1);
+            }
+            if(ctxt && ctxt->constructed)
+                ctxt->~AggregateCtxt();
+        }
+
+        template<
+                std::size_t Count,
+                typename    Functions,
+                typename... Values
+        >
+        inline typename std::enable_if<(sizeof...(Values) && sizeof...(Values) < Count), void>::type step(
+                sqlite3_context* db,
+                int              count,
+                sqlite3_value**  vals,
+                Values&&...      values
+        ) {
+            using arg_type = typename std::remove_cv<
+                    typename std::remove_reference<
+                            typename utility::function_traits<
+                                    typename Functions::first_type
+                            >::template argument<sizeof...(Values)>
+                    >::type
+            >::type;
+
+            step<Count, Functions>(
+                    db,
+                    count,
+                    vals,
+                    std::forward<Values>(values)...,
+                    get_val_from_db(vals[sizeof...(Values) - 1], result_type<arg_type>()));
+        }
+
+        template<
+                std::size_t Count,
+                typename    Functions,
+                typename... Values
+        >
+        inline typename std::enable_if<(sizeof...(Values) == Count), void>::type step(
+                sqlite3_context* db,
+                int,
+                sqlite3_value**,
+                Values&&...      values
+        ) {
+            static_cast<Functions*>(sqlite3_user_data(db))->first(std::forward<Values>(values)...);
+        }
+
+        template<
+                typename    ContextType,
+                typename    Functions
+        >
+        inline void final(sqlite3_context* db) {
+            auto ctxt = static_cast<AggregateCtxt<ContextType>*>(sqlite3_aggregate_context(db, sizeof(AggregateCtxt<ContextType>)));
+            try {
+                if(!ctxt) return;
+                if(!ctxt->constructed) new(ctxt) AggregateCtxt<ContextType>();
+                store_result_in_db(db,
+                                   static_cast<Functions*>(sqlite3_user_data(db))->second(ctxt->obj));
+            } catch(const sqlite_exception &e) {
+                sqlite3_result_error_code(db, e.get_code());
+                sqlite3_result_error(db, e.what(), -1);
+            } catch(const std::exception &e) {
+                sqlite3_result_error(db, e.what(), -1);
+            } catch(...) {
+                sqlite3_result_error(db, "Unknown error", -1);
+            }
+            if(ctxt && ctxt->constructed)
+                ctxt->~AggregateCtxt();
+        }
+
+        template<
+                std::size_t Count,
+                typename    Function,
+                typename... Values
+        >
+        inline typename std::enable_if<(sizeof...(Values) < Count), void>::type scalar(
+                sqlite3_context* db,
+                int              count,
+                sqlite3_value**  vals,
+                Values&&...      values
+        ) {
+            using arg_type = typename std::remove_cv<
+                    typename std::remove_reference<
+                            typename utility::function_traits<Function>::template argument<sizeof...(Values)>
+                    >::type
+            >::type;
+
+            scalar<Count, Function>(
+                    db,
+                    count,
+                    vals,
+                    std::forward<Values>(values)...,
+                    get_val_from_db(vals[sizeof...(Values)], result_type<arg_type>()));
+        }
+
+        template<
+                std::size_t Count,
+                typename    Function,
+                typename... Values
+        >
+        inline typename std::enable_if<(sizeof...(Values) == Count), void>::type scalar(
+                sqlite3_context* db,
+                int,
+                sqlite3_value**,
+                Values&&...      values
+        ) {
+            try {
+                store_result_in_db(db,
+                                   (*static_cast<Function*>(sqlite3_user_data(db)))(std::forward<Values>(values)...));
+            } catch(const sqlite_exception &e) {
+                sqlite3_result_error_code(db, e.get_code());
+                sqlite3_result_error(db, e.what(), -1);
+            } catch(const std::exception &e) {
+                sqlite3_result_error(db, e.what(), -1);
+            } catch(...) {
+                sqlite3_result_error(db, "Unknown error", -1);
+            }
+        }
+    }
+}

src/claimtrie/sqlite/hdr/sqlite_modern_cpp/errors.h

@@ -0,0 +1,70 @@
+#pragma once
+
+#include <string>
+#include <stdexcept>
+
+#include "../../sqlite3.h"
+
+namespace sqlite {
+
+    class sqlite_exception: public std::runtime_error {
+    public:
+        sqlite_exception(const char* msg, str_ref sql, int code = -1): runtime_error(msg + std::string(": ") + sql), code(code), sql(sql) {}
+        sqlite_exception(int code, str_ref sql, const char *msg = nullptr): runtime_error((msg ? msg : sqlite3_errstr(code)) + std::string(": ") + sql), code(code), sql(sql) {}
+        int get_code() const {return code & 0xFF;}
+        int get_extended_code() const {return code;}
+        std::string get_sql() const {return sql;}
+        const char *errstr() const {return code == -1 ? "Unknown error" : sqlite3_errstr(code);}
+    private:
+        int code;
+        std::string sql;
+    };
+
+    namespace errors {
+        //One more or less trivial derived error class for each SQLITE error.
+        //Note the following are not errors so have no classes:
+        //SQLITE_OK, SQLITE_NOTICE, SQLITE_WARNING, SQLITE_ROW, SQLITE_DONE
+        //
+        //Note these names are exact matches to the names of the SQLITE error codes.
+#define SQLITE_MODERN_CPP_ERROR_CODE(NAME,name,derived) \
+		class name: public sqlite_exception { using sqlite_exception::sqlite_exception; };\
+		derived
+#define SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(BASE,SUB,base,sub) \
+		class base ## _ ## sub: public base { using base::base; };
+#include "lists/error_codes.h"
+#undef SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED
+#undef SQLITE_MODERN_CPP_ERROR_CODE
+
+        //Some additional errors are here for the C++ interface
+        class more_rows: public sqlite_exception { using sqlite_exception::sqlite_exception; };
+        class no_rows: public sqlite_exception { using sqlite_exception::sqlite_exception; };
+        class more_statements: public sqlite_exception { using sqlite_exception::sqlite_exception; }; // Prepared statements can only contain one statement
+        class invalid_utf16: public sqlite_exception { using sqlite_exception::sqlite_exception; };
+        class unknown_binding: public sqlite_exception { using sqlite_exception::sqlite_exception; };
+
+        static void throw_sqlite_error(const int& error_code, str_ref sql = "", const char *errmsg = nullptr) {
+            switch(error_code & 0xFF) {
+#define SQLITE_MODERN_CPP_ERROR_CODE(NAME,name,derived)     \
+				case SQLITE_ ## NAME: switch(error_code) {          \
+					derived                                           \
+					case SQLITE_ ## NAME:                             \
+					default: throw name(error_code, sql);             \
+				}
+
+#if SQLITE_VERSION_NUMBER < 3010000
+                #define SQLITE_IOERR_VNODE (SQLITE_IOERR | (27<<8))
+#define SQLITE_IOERR_AUTH (SQLITE_IOERR | (28<<8))
+#define SQLITE_AUTH_USER (SQLITE_AUTH | (1<<8))
+#endif
+
+#define SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(BASE,SUB,base,sub) \
+					case SQLITE_ ## BASE ## _ ## SUB: throw base ## _ ## sub(error_code, sql, errmsg);
+#include "lists/error_codes.h"
+#undef SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED
+#undef SQLITE_MODERN_CPP_ERROR_CODE
+                default: throw sqlite_exception(error_code, sql, errmsg);
+            }
+        }
+    }
+    namespace exceptions = errors;
+}

src/claimtrie/sqlite/hdr/sqlite_modern_cpp/lists/error_codes.h

@@ -0,0 +1,88 @@
+SQLITE_MODERN_CPP_ERROR_CODE(ERROR,error,)
+SQLITE_MODERN_CPP_ERROR_CODE(INTERNAL,internal,)
+SQLITE_MODERN_CPP_ERROR_CODE(PERM,perm,)
+SQLITE_MODERN_CPP_ERROR_CODE(ABORT,abort,
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(ABORT,ROLLBACK,abort,rollback)
+)
+SQLITE_MODERN_CPP_ERROR_CODE(BUSY,busy,
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(BUSY,RECOVERY,busy,recovery)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(BUSY,SNAPSHOT,busy,snapshot)
+)
+SQLITE_MODERN_CPP_ERROR_CODE(LOCKED,locked,
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(LOCKED,SHAREDCACHE,locked,sharedcache)
+)
+SQLITE_MODERN_CPP_ERROR_CODE(NOMEM,nomem,)
+SQLITE_MODERN_CPP_ERROR_CODE(READONLY,readonly,)
+SQLITE_MODERN_CPP_ERROR_CODE(INTERRUPT,interrupt,)
+SQLITE_MODERN_CPP_ERROR_CODE(IOERR,ioerr,
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,READ,ioerr,read)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,SHORT_READ,ioerr,short_read)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,WRITE,ioerr,write)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,FSYNC,ioerr,fsync)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,DIR_FSYNC,ioerr,dir_fsync)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,TRUNCATE,ioerr,truncate)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,FSTAT,ioerr,fstat)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,UNLOCK,ioerr,unlock)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,RDLOCK,ioerr,rdlock)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,DELETE,ioerr,delete)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,BLOCKED,ioerr,blocked)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,NOMEM,ioerr,nomem)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,ACCESS,ioerr,access)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,CHECKRESERVEDLOCK,ioerr,checkreservedlock)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,LOCK,ioerr,lock)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,CLOSE,ioerr,close)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,DIR_CLOSE,ioerr,dir_close)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,SHMOPEN,ioerr,shmopen)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,SHMSIZE,ioerr,shmsize)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,SHMLOCK,ioerr,shmlock)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,SHMMAP,ioerr,shmmap)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,SEEK,ioerr,seek)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,DELETE_NOENT,ioerr,delete_noent)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,MMAP,ioerr,mmap)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,GETTEMPPATH,ioerr,gettemppath)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,CONVPATH,ioerr,convpath)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,VNODE,ioerr,vnode)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(IOERR,AUTH,ioerr,auth)
+)
+SQLITE_MODERN_CPP_ERROR_CODE(CORRUPT,corrupt,
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(CORRUPT,VTAB,corrupt,vtab)
+)
+SQLITE_MODERN_CPP_ERROR_CODE(NOTFOUND,notfound,)
+SQLITE_MODERN_CPP_ERROR_CODE(FULL,full,)
+SQLITE_MODERN_CPP_ERROR_CODE(CANTOPEN,cantopen,
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(CANTOPEN,NOTEMPDIR,cantopen,notempdir)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(CANTOPEN,ISDIR,cantopen,isdir)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(CANTOPEN,FULLPATH,cantopen,fullpath)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(CANTOPEN,CONVPATH,cantopen,convpath)
+)
+SQLITE_MODERN_CPP_ERROR_CODE(PROTOCOL,protocol,)
+SQLITE_MODERN_CPP_ERROR_CODE(EMPTY,empty,)
+SQLITE_MODERN_CPP_ERROR_CODE(SCHEMA,schema,)
+SQLITE_MODERN_CPP_ERROR_CODE(TOOBIG,toobig,)
+SQLITE_MODERN_CPP_ERROR_CODE(CONSTRAINT,constraint,
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(CONSTRAINT,CHECK,constraint,check)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(CONSTRAINT,COMMITHOOK,constraint,commithook)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(CONSTRAINT,FOREIGNKEY,constraint,foreignkey)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(CONSTRAINT,FUNCTION,constraint,function)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(CONSTRAINT,NOTNULL,constraint,notnull)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(CONSTRAINT,PRIMARYKEY,constraint,primarykey)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(CONSTRAINT,TRIGGER,constraint,trigger)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(CONSTRAINT,UNIQUE,constraint,unique)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(CONSTRAINT,VTAB,constraint,vtab)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(CONSTRAINT,ROWID,constraint,rowid)
+)
+SQLITE_MODERN_CPP_ERROR_CODE(MISMATCH,mismatch,)
+SQLITE_MODERN_CPP_ERROR_CODE(MISUSE,misuse,)
+SQLITE_MODERN_CPP_ERROR_CODE(NOLFS,nolfs,)
+SQLITE_MODERN_CPP_ERROR_CODE(AUTH,auth,
+)
+SQLITE_MODERN_CPP_ERROR_CODE(FORMAT,format,)
+SQLITE_MODERN_CPP_ERROR_CODE(RANGE,range,)
+SQLITE_MODERN_CPP_ERROR_CODE(NOTADB,notadb,)
+SQLITE_MODERN_CPP_ERROR_CODE(NOTICE,notice,
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(NOTICE,RECOVER_WAL,notice,recover_wal)
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(NOTICE,RECOVER_ROLLBACK,notice,recover_rollback)
+)
+SQLITE_MODERN_CPP_ERROR_CODE(WARNING,warning,
+  SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(WARNING,AUTOINDEX,warning,autoindex)
+)

src/claimtrie/sqlite/hdr/sqlite_modern_cpp/log.h

@@ -0,0 +1,101 @@
+#include "errors.h"
+
+#include <sqlite/sqlite3.h>
+
+#include <utility>
+#include <tuple>
+#include <type_traits>
+
+namespace sqlite {
+	namespace detail {
+		template<class>
+		using void_t = void;
+		template<class T, class = void>
+		struct is_callable : std::false_type {};
+		template<class Functor, class ...Arguments>
+		struct is_callable<Functor(Arguments...), void_t<decltype(std::declval<Functor>()(std::declval<Arguments>()...))>> : std::true_type {};
+		template<class Functor, class ...Functors>
+		class FunctorOverload: public Functor, public FunctorOverload<Functors...> {
+			public:
+				template<class Functor1, class ...Remaining>
+				FunctorOverload(Functor1 &&functor, Remaining &&... remaining):
+					Functor(std::forward<Functor1>(functor)),
+					FunctorOverload<Functors...>(std::forward<Remaining>(remaining)...) {}
+				using Functor::operator();
+				using FunctorOverload<Functors...>::operator();
+		};
+		template<class Functor>
+		class FunctorOverload<Functor>: public Functor {
+			public:
+				template<class Functor1>
+				FunctorOverload(Functor1 &&functor):
+					Functor(std::forward<Functor1>(functor)) {}
+				using Functor::operator();
+		};
+		template<class Functor>
+		class WrapIntoFunctor: public Functor {
+			public:
+				template<class Functor1>
+				WrapIntoFunctor(Functor1 &&functor):
+					Functor(std::forward<Functor1>(functor)) {}
+				using Functor::operator();
+		};
+		template<class ReturnType, class ...Arguments>
+		class WrapIntoFunctor<ReturnType(*)(Arguments...)> {
+			ReturnType(*ptr)(Arguments...);
+			public:
+				WrapIntoFunctor(ReturnType(*ptr)(Arguments...)): ptr(ptr) {}
+				ReturnType operator()(Arguments... arguments) { return (*ptr)(std::forward<Arguments>(arguments)...); }
+		};
+		inline void store_error_log_data_pointer(std::shared_ptr<void> ptr) {
+			static std::shared_ptr<void> stored;
+			stored = std::move(ptr);
+		}
+		template<class T>
+		std::shared_ptr<typename std::decay<T>::type> make_shared_inferred(T &&t) {
+			return std::make_shared<typename std::decay<T>::type>(std::forward<T>(t));
+		}
+	}
+	template<class Handler>
+	typename std::enable_if<!detail::is_callable<Handler(const sqlite_exception&)>::value>::type
+	error_log(Handler &&handler);
+	template<class Handler>
+	typename std::enable_if<detail::is_callable<Handler(const sqlite_exception&)>::value>::type
+	error_log(Handler &&handler);
+	template<class ...Handler>
+	typename std::enable_if<sizeof...(Handler)>=2>::type
+	error_log(Handler &&...handler) {
+		return error_log(detail::FunctorOverload<detail::WrapIntoFunctor<typename std::decay<Handler>::type>...>(std::forward<Handler>(handler)...));
+	}
+	template<class Handler>
+	typename std::enable_if<!detail::is_callable<Handler(const sqlite_exception&)>::value>::type
+	error_log(Handler &&handler) {
+		return error_log(std::forward<Handler>(handler), [](const sqlite_exception&) {});
+	}
+	template<class Handler>
+	typename std::enable_if<detail::is_callable<Handler(const sqlite_exception&)>::value>::type
+	error_log(Handler &&handler) {
+		auto ptr = detail::make_shared_inferred([handler = std::forward<Handler>(handler)](int error_code, const char *errstr) mutable {
+			  switch(error_code & 0xFF) {
+#define SQLITE_MODERN_CPP_ERROR_CODE(NAME,name,derived)             \
+				  case SQLITE_ ## NAME: switch(error_code) {                \
+					  derived                                                 \
+				    default: handler(errors::name(errstr, "", error_code)); \
+				  };break;
+#define SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED(BASE,SUB,base,sub)                    \
+					  case SQLITE_ ## BASE ## _ ## SUB:                                       \
+				      handler(errors::base ## _ ## sub(errstr, "", error_code)); \
+				      break;
+#include "lists/error_codes.h"
+#undef SQLITE_MODERN_CPP_ERROR_CODE_EXTENDED
+#undef SQLITE_MODERN_CPP_ERROR_CODE
+				  default: handler(sqlite_exception(errstr, "", error_code)); \
+			  }
+			});
+
+		sqlite3_config(SQLITE_CONFIG_LOG, static_cast<void(*)(void*,int,const char*)>([](void *functor, int error_code, const char *errstr) {
+				(*static_cast<decltype(ptr.get())>(functor))(error_code, errstr);
+			}), ptr.get());
+		detail::store_error_log_data_pointer(std::move(ptr));
+	}
+}

src/claimtrie/sqlite/hdr/sqlite_modern_cpp/sqlcipher.h

@@ -0,0 +1,44 @@
+#pragma once
+
+#ifndef SQLITE_HAS_CODEC
+#define SQLITE_HAS_CODEC
+#endif
+
+#include "../sqlite_modern_cpp.h"
+
+namespace sqlite {
+	struct sqlcipher_config : public sqlite_config {
+		std::string key;
+	};
+
+	class sqlcipher_database : public database {
+	public:
+		sqlcipher_database(std::string db, const sqlcipher_config &config): database(db, config) {
+			set_key(config.key);
+		}
+		
+		sqlcipher_database(std::u16string db, const sqlcipher_config &config): database(db, config) {
+			set_key(config.key);
+		}
+		
+		void set_key(const std::string &key) {
+			if(auto ret = sqlite3_key(_db.get(), key.data(), key.size()))
+				errors::throw_sqlite_error(ret);
+		}
+
+		void set_key(const std::string &key, const std::string &db_name) {
+			if(auto ret = sqlite3_key_v2(_db.get(), db_name.c_str(), key.data(), key.size()))
+				errors::throw_sqlite_error(ret);
+		}
+
+		void rekey(const std::string &new_key) {
+			if(auto ret = sqlite3_rekey(_db.get(), new_key.data(), new_key.size()))
+				errors::throw_sqlite_error(ret);
+		}
+
+		void rekey(const std::string &new_key, const std::string &db_name) {
+			if(auto ret = sqlite3_rekey_v2(_db.get(), db_name.c_str(), new_key.data(), new_key.size()))
+				errors::throw_sqlite_error(ret);
+		}
+	};
+}

src/claimtrie/sqlite/hdr/sqlite_modern_cpp/type_wrapper.h

@@ -0,0 +1,443 @@
+#pragma once
+
+#include <type_traits>
+#include <string>
+#include <memory>
+#include <vector>
+#ifdef __has_include
+#if __cplusplus >= 201703 && __has_include(<string_view>)
+#define MODERN_SQLITE_STRINGVIEW_SUPPORT
+#endif
+#endif
+#ifdef __has_include
+#if __cplusplus > 201402 && __has_include(<optional>)
+#define MODERN_SQLITE_STD_OPTIONAL_SUPPORT
+#endif
+#endif
+
+#ifdef __has_include
+#if __cplusplus > 201402 && __has_include(<variant>)
+#define MODERN_SQLITE_STD_VARIANT_SUPPORT
+#endif
+#endif
+
+#ifdef MODERN_SQLITE_STD_OPTIONAL_SUPPORT
+#include <optional>
+#endif
+
+#ifdef MODERN_SQLITE_STD_VARIANT_SUPPORT
+#include <variant>
+#endif
+#ifdef MODERN_SQLITE_STRINGVIEW_SUPPORT
+#include <string_view>
+namespace sqlite
+{
+	typedef const std::string_view& str_ref;
+    typedef std::string_view&& str_uni_ref;
+	typedef const std::u16string_view& u16str_ref;
+	typedef std::u16string_view&& u16str_uni_ref;
+}
+#else
+namespace sqlite
+{
+    typedef const std::string& str_ref;
+    typedef std::string&& str_uni_ref;
+    typedef const std::u16string& u16str_ref;
+    typedef std::u16string&& u16str_uni_ref;
+}
+#endif
+#include "../../sqlite3.h"
+#include "errors.h"
+
+namespace sqlite {
+    template<class T, int Type, class = void>
+    struct has_sqlite_type : std::false_type {};
+
+    template<class T>
+    using is_sqlite_value = std::integral_constant<bool, false
+                                                         || has_sqlite_type<T, SQLITE_NULL>::value
+                                                         || has_sqlite_type<T, SQLITE_INTEGER>::value
+                                                         || has_sqlite_type<T, SQLITE_FLOAT>::value
+                                                         || has_sqlite_type<T, SQLITE_TEXT>::value
+                                                         || has_sqlite_type<T, SQLITE_BLOB>::value
+    >;
+
+    template<class T, int Type>
+    struct has_sqlite_type<T&, Type> : has_sqlite_type<T, Type> {};
+    template<class T, int Type>
+    struct has_sqlite_type<const T, Type> : has_sqlite_type<T, Type> {};
+    template<class T, int Type>
+    struct has_sqlite_type<volatile T, Type> : has_sqlite_type<T, Type> {};
+
+    template<class T>
+    struct result_type {
+        using type = T;
+        constexpr result_type() = default;
+        template<class U, class = typename std::enable_if<std::is_assignable<U, T>::value>>
+        constexpr result_type(result_type<U>) { }
+    };
+
+    // int
+    template<>
+    struct has_sqlite_type<int, SQLITE_INTEGER> : std::true_type {};
+
+    inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, const int& val) {
+        return sqlite3_bind_int(stmt, inx, val);
+    }
+    inline void store_result_in_db(sqlite3_context* db, const int& val) {
+        sqlite3_result_int(db, val);
+    }
+    inline int get_col_from_db(sqlite3_stmt* stmt, int inx, result_type<int>) {
+        return sqlite3_column_type(stmt, inx) == SQLITE_NULL ? 0 :
+               sqlite3_column_int(stmt, inx);
+    }
+    inline int get_val_from_db(sqlite3_value *value, result_type<int>) {
+        return sqlite3_value_type(value) == SQLITE_NULL ? 0 :
+               sqlite3_value_int(value);
+    }
+
+    // sqlite_int64
+    template<>
+    struct has_sqlite_type<sqlite_int64, SQLITE_INTEGER, void> : std::true_type {};
+
+    inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, const sqlite_int64& val) {
+        return sqlite3_bind_int64(stmt, inx, val);
+    }
+    inline void store_result_in_db(sqlite3_context* db, const sqlite_int64& val) {
+        sqlite3_result_int64(db, val);
+    }
+    inline sqlite_int64 get_col_from_db(sqlite3_stmt* stmt, int inx, result_type<sqlite_int64 >) {
+        return sqlite3_column_type(stmt, inx) == SQLITE_NULL ? 0 :
+               sqlite3_column_int64(stmt, inx);
+    }
+    inline sqlite3_int64 get_val_from_db(sqlite3_value *value, result_type<sqlite3_int64>) {
+        return sqlite3_value_type(value) == SQLITE_NULL ? 0 :
+               sqlite3_value_int64(value);
+    }
+
+    // float
+    template<>
+    struct has_sqlite_type<float, SQLITE_FLOAT, void> : std::true_type {};
+
+    inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, const float& val) {
+        return sqlite3_bind_double(stmt, inx, double(val));
+    }
+    inline void store_result_in_db(sqlite3_context* db, const float& val) {
+        sqlite3_result_double(db, val);
+    }
+    inline float get_col_from_db(sqlite3_stmt* stmt, int inx, result_type<float>) {
+        return sqlite3_column_type(stmt, inx) == SQLITE_NULL ? 0 :
+               sqlite3_column_double(stmt, inx);
+    }
+    inline float get_val_from_db(sqlite3_value *value, result_type<float>) {
+        return sqlite3_value_type(value) == SQLITE_NULL ? 0 :
+               sqlite3_value_double(value);
+    }
+
+    // double
+    template<>
+    struct has_sqlite_type<double, SQLITE_FLOAT, void> : std::true_type {};
+
+    inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, const double& val) {
+        return sqlite3_bind_double(stmt, inx, val);
+    }
+    inline void store_result_in_db(sqlite3_context* db, const double& val) {
+        sqlite3_result_double(db, val);
+    }
+    inline double get_col_from_db(sqlite3_stmt* stmt, int inx, result_type<double>) {
+        return sqlite3_column_type(stmt, inx) == SQLITE_NULL ? 0 :
+               sqlite3_column_double(stmt, inx);
+    }
+    inline double get_val_from_db(sqlite3_value *value, result_type<double>) {
+        return sqlite3_value_type(value) == SQLITE_NULL ? 0 :
+               sqlite3_value_double(value);
+    }
+
+    /* for nullptr support */
+    template<>
+    struct has_sqlite_type<std::nullptr_t, SQLITE_NULL, void> : std::true_type {};
+
+    inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, std::nullptr_t) {
+        return sqlite3_bind_null(stmt, inx);
+    }
+    inline void store_result_in_db(sqlite3_context* db, std::nullptr_t) {
+        sqlite3_result_null(db);
+    }
+
+#ifdef MODERN_SQLITE_STD_VARIANT_SUPPORT
+    template<>
+	struct has_sqlite_type<std::monostate, SQLITE_NULL, void> : std::true_type {};
+
+	inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, std::monostate) {
+		return sqlite3_bind_null(stmt, inx);
+	}
+	inline void store_result_in_db(sqlite3_context* db, std::monostate) {
+		sqlite3_result_null(db);
+	}
+	inline std::monostate get_col_from_db(sqlite3_stmt* stmt, int inx, result_type<std::monostate>) {
+		return std::monostate();
+	}
+	inline std::monostate get_val_from_db(sqlite3_value *value, result_type<std::monostate>) {
+		return std::monostate();
+	}
+#endif
+
+    // str_ref
+    template<>
+    struct has_sqlite_type<std::string, SQLITE_BLOB, void> : std::true_type {}; //
+    inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, str_ref val) {
+        return sqlite3_bind_blob(stmt, inx, val.data(), val.length(), SQLITE_STATIC);
+    }
+
+    // str_uni_ref
+    inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, str_uni_ref val) {
+        return sqlite3_bind_blob(stmt, inx, val.data(), val.length(), SQLITE_TRANSIENT);
+    }
+
+    // Convert char* to string_view to trigger op<<(..., const str_ref )
+    template<std::size_t N> inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, const char(&STR)[N]) {
+        return sqlite3_bind_blob(stmt, inx, &STR[0], N-1, SQLITE_STATIC);
+    }
+
+    inline std::string get_col_from_db(sqlite3_stmt* stmt, int inx, result_type<std::string>) {
+        auto type = sqlite3_column_type(stmt, inx);
+        switch (type) {
+            case SQLITE_INTEGER:
+                return std::to_string(sqlite3_column_int64(stmt, inx));
+            case SQLITE_FLOAT:
+                return std::to_string(sqlite3_column_double(stmt, inx));
+            case SQLITE_BLOB: // It's important to support both text and blob data as txdb has some text and trie has some blob
+                return std::string(reinterpret_cast<char const *>(sqlite3_column_blob(stmt, inx)), sqlite3_column_bytes(stmt, inx));
+            case SQLITE3_TEXT:
+                return std::string(reinterpret_cast<char const *>(sqlite3_column_text(stmt, inx)), sqlite3_column_bytes(stmt, inx));
+        }
+        return std::string(); // NULL
+    }
+    inline std::string get_val_from_db(sqlite3_value *value, result_type<std::string >) {
+        auto type = sqlite3_value_type(value);
+        switch (type) {
+            case SQLITE_INTEGER:
+                return std::to_string(sqlite3_value_int64(value));
+            case SQLITE_FLOAT:
+                return std::to_string(sqlite3_value_double(value));
+            case SQLITE_BLOB:
+                return std::string(reinterpret_cast<char const *>(sqlite3_value_blob(value)), sqlite3_value_bytes(value));
+            case SQLITE3_TEXT:
+                return std::string(reinterpret_cast<char const *>(sqlite3_value_text(value)), sqlite3_value_bytes(value));
+        }
+        return std::string(); // NULL
+    }
+
+    inline void store_result_in_db(sqlite3_context* db, str_ref val) {
+        sqlite3_result_blob(db, val.data(), val.length(), SQLITE_STATIC);
+    }
+
+    inline void store_result_in_db(sqlite3_context* db, str_uni_ref val) {
+        sqlite3_result_blob(db, val.data(), val.length(), SQLITE_TRANSIENT);
+    }
+    // u16str_ref
+    template<>
+    struct has_sqlite_type<std::u16string, SQLITE3_TEXT, void> : std::true_type {};
+    inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, u16str_ref val) {
+        return sqlite3_bind_text16(stmt, inx, val.data(), sizeof(char16_t) * val.length(), SQLITE_STATIC);
+    }
+
+    inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, u16str_uni_ref val) {
+        return sqlite3_bind_text16(stmt, inx, val.data(), sizeof(char16_t) * val.length(), SQLITE_TRANSIENT);
+    }
+
+    // Convert char* to string_view to trigger op<<(..., const str_ref )
+    template<std::size_t N> inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, const char16_t(&STR)[N]) {
+        return sqlite3_bind_text16(stmt, inx, &STR[0], sizeof(char16_t) * (N-1), SQLITE_STATIC);
+    }
+
+    inline std::u16string get_col_from_db(sqlite3_stmt* stmt, int inx, result_type<std::u16string>) {
+        return sqlite3_column_type(stmt, inx) == SQLITE_NULL ? std::u16string() :
+               std::u16string(reinterpret_cast<char16_t const *>(sqlite3_column_text16(stmt, inx)), sqlite3_column_bytes16(stmt, inx));
+    }
+    inline std::u16string  get_val_from_db(sqlite3_value *value, result_type<std::u16string>) {
+        return sqlite3_value_type(value) == SQLITE_NULL ? std::u16string() :
+               std::u16string(reinterpret_cast<char16_t const *>(sqlite3_value_text16(value)), sqlite3_value_bytes16(value));
+    }
+
+    inline void store_result_in_db(sqlite3_context* db, u16str_ref val) {
+        sqlite3_result_text16(db, val.data(), sizeof(char16_t) * val.length(), SQLITE_STATIC);
+    }
+
+    inline void store_result_in_db(sqlite3_context* db, u16str_uni_ref val) {
+        sqlite3_result_text16(db, val.data(), sizeof(char16_t) * val.length(), SQLITE_TRANSIENT);
+    }
+
+    // Other integer types
+    template<class Integral>
+    struct has_sqlite_type<Integral, SQLITE_INTEGER, typename std::enable_if<std::is_integral<Integral>::value>::type> : std::true_type {};
+
+    template<class Integral, class = typename std::enable_if<std::is_integral<Integral>::value>::type>
+    inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, const Integral& val) {
+        return bind_col_in_db(stmt, inx, static_cast<sqlite3_int64>(val));
+    }
+    template<class Integral, class = std::enable_if<std::is_integral<Integral>::type>>
+    inline void store_result_in_db(sqlite3_context* db, const Integral& val) {
+        store_result_in_db(db, static_cast<sqlite3_int64>(val));
+    }
+    template<class Integral, class = typename std::enable_if<std::is_integral<Integral>::value>::type>
+    inline Integral get_col_from_db(sqlite3_stmt* stmt, int inx, result_type<Integral>) {
+        return get_col_from_db(stmt, inx, result_type<sqlite3_int64>());
+    }
+    template<class Integral, class = typename std::enable_if<std::is_integral<Integral>::value>::type>
+    inline Integral get_val_from_db(sqlite3_value *value, result_type<Integral>) {
+        return get_val_from_db(value, result_type<sqlite3_int64>());
+    }
+
+    // vector<T, A>
+    template<typename T, typename A>
+    struct has_sqlite_type<std::vector<T, A>, SQLITE_BLOB, void> : std::true_type {};
+
+    template<typename T, typename A> inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, const std::vector<T, A>& vec) {
+        void const* buf = reinterpret_cast<void const *>(vec.data());
+        int bytes = vec.size() * sizeof(T);
+        return sqlite3_bind_blob(stmt, inx, buf, bytes, SQLITE_STATIC);
+    }
+    template<typename T, typename A> inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, std::vector<T, A>&& vec) {
+        void const* buf = reinterpret_cast<void const *>(vec.data());
+        int bytes = vec.size() * sizeof(T);
+        return sqlite3_bind_blob(stmt, inx, buf, bytes, SQLITE_TRANSIENT);
+    }
+    template<typename T, typename A> inline void store_result_in_db(sqlite3_context* db, const std::vector<T, A>& vec) {
+        void const* buf = reinterpret_cast<void const *>(vec.data());
+        int bytes = vec.size() * sizeof(T);
+        sqlite3_result_blob(db, buf, bytes, SQLITE_STATIC);
+    }
+    template<typename T, typename A> inline void store_result_in_db(sqlite3_context* db, std::vector<T, A>&& vec) {
+        void const* buf = reinterpret_cast<void const *>(vec.data());
+        int bytes = vec.size() * sizeof(T);
+        sqlite3_result_blob(db, buf, bytes, SQLITE_TRANSIENT);
+    }
+    template<typename T, typename A> inline std::vector<T, A> get_col_from_db(sqlite3_stmt* stmt, int inx, result_type<std::vector<T, A>>) {
+        if(sqlite3_column_type(stmt, inx) == SQLITE_NULL) {
+            return {};
+        }
+        int bytes = sqlite3_column_bytes(stmt, inx);
+        T const* buf = reinterpret_cast<T const *>(sqlite3_column_blob(stmt, inx));
+        return std::vector<T, A>(buf, buf + bytes/sizeof(T));
+    }
+    template<typename T, typename A> inline std::vector<T, A> get_val_from_db(sqlite3_value *value, result_type<std::vector<T, A>>) {
+        if(sqlite3_value_type(value) == SQLITE_NULL) {
+            return {};
+        }
+        int bytes = sqlite3_value_bytes(value);
+        T const* buf = reinterpret_cast<T const *>(sqlite3_value_blob(value));
+        return std::vector<T, A>(buf, buf + bytes/sizeof(T));
+    }
+
+    /* for unique_ptr<T> support */
+    template<typename T, int Type>
+    struct has_sqlite_type<std::unique_ptr<T>, Type, void> : has_sqlite_type<T, Type> {};
+    template<typename T>
+    struct has_sqlite_type<std::unique_ptr<T>, SQLITE_NULL, void> : std::true_type {};
+
+    template<typename T> inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, const std::unique_ptr<T>& val) {
+        return val ? bind_col_in_db(stmt, inx, *val) : bind_col_in_db(stmt, inx, nullptr);
+    }
+    template<typename T> inline std::unique_ptr<T> get_col_from_db(sqlite3_stmt* stmt, int inx, result_type<std::unique_ptr<T>>) {
+        if(sqlite3_column_type(stmt, inx) == SQLITE_NULL) {
+            return nullptr;
+        }
+        return std::make_unique<T>(get_col_from_db(stmt, inx, result_type<T>()));
+    }
+    template<typename T> inline std::unique_ptr<T> get_val_from_db(sqlite3_value *value, result_type<std::unique_ptr<T>>) {
+        if(sqlite3_value_type(value) == SQLITE_NULL) {
+            return nullptr;
+        }
+        return std::make_unique<T>(get_val_from_db(value, result_type<T>()));
+    }
+
+    // std::optional support for NULL values
+#ifdef MODERN_SQLITE_STD_OPTIONAL_SUPPORT
+    template<class T>
+	using optional = std::optional<T>;
+
+    template<typename T, int Type>
+    struct has_sqlite_type<optional<T>, Type, void> : has_sqlite_type<T, Type> {};
+    template<typename T>
+    struct has_sqlite_type<optional<T>, SQLITE_NULL, void> : std::true_type {};
+
+    template <typename OptionalT> inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, const optional<OptionalT>& val) {
+        return val ? bind_col_in_db(stmt, inx, *val) : bind_col_in_db(stmt, inx, nullptr);
+    }
+    template <typename OptionalT> inline void store_result_in_db(sqlite3_context* db, const optional<OptionalT>& val) {
+        if(val)
+            store_result_in_db(db, *val);
+        else
+            sqlite3_result_null(db);
+    }
+
+    template <typename OptionalT> inline optional<OptionalT> get_col_from_db(sqlite3_stmt* stmt, int inx, result_type<optional<OptionalT>>) {
+        if(sqlite3_column_type(stmt, inx) == SQLITE_NULL) {
+			return std::nullopt;
+		}
+		return std::make_optional(get_col_from_db(stmt, inx, result_type<OptionalT>()));
+    }
+    template <typename OptionalT> inline optional<OptionalT> get_val_from_db(sqlite3_value *value, result_type<optional<OptionalT>>) {
+        if(sqlite3_value_type(value) == SQLITE_NULL) {
+			return std::nullopt;
+		}
+		return std::make_optional(get_val_from_db(value, result_type<OptionalT>()));
+    }
+#endif
+
+#ifdef MODERN_SQLITE_STD_VARIANT_SUPPORT
+    namespace detail {
+		template<class T, class U>
+		struct tag_trait : U { using tag = T; };
+	}
+
+	template<int Type, class ...Options>
+	struct has_sqlite_type<std::variant<Options...>, Type, void> : std::disjunction<detail::tag_trait<Options, has_sqlite_type<Options, Type>>...> {};
+
+	namespace detail {
+		template<int Type, typename ...Options, typename Callback, typename first_compatible = has_sqlite_type<std::variant<Options...>, Type>>
+		inline std::variant<Options...> variant_select_type(Callback &&callback) {
+			if constexpr(first_compatible::value)
+				return callback(result_type<typename first_compatible::tag>());
+			else
+				throw errors::mismatch("The value is unsupported by this variant.", "", SQLITE_MISMATCH);
+		}
+		template<typename ...Options, typename Callback> inline decltype(auto) variant_select(int type, Callback &&callback) {
+			switch(type) {
+				case SQLITE_NULL:
+					return variant_select_type<SQLITE_NULL, Options...>(std::forward<Callback>(callback));
+				case SQLITE_INTEGER:
+					return variant_select_type<SQLITE_INTEGER, Options...>(std::forward<Callback>(callback));
+				case SQLITE_FLOAT:
+					return variant_select_type<SQLITE_FLOAT, Options...>(std::forward<Callback>(callback));
+				case SQLITE_TEXT:
+					return variant_select_type<SQLITE_TEXT, Options...>(std::forward<Callback>(callback));
+				case SQLITE_BLOB:
+					return variant_select_type<SQLITE_BLOB, Options...>(std::forward<Callback>(callback));
+			}
+#ifdef _MSC_VER
+			__assume(false);
+#else
+			__builtin_unreachable();
+#endif
+		}
+	}
+	template <typename ...Args> inline int bind_col_in_db(sqlite3_stmt* stmt, int inx, const std::variant<Args...>& val) {
+		return std::visit([&](auto &&opt) {return bind_col_in_db(stmt, inx, std::forward<decltype(opt)>(opt));}, val);
+	}
+	template <typename ...Args> inline void store_result_in_db(sqlite3_context* db, const std::variant<Args...>& val) {
+		std::visit([&](auto &&opt) {store_result_in_db(db, std::forward<decltype(opt)>(opt));}, val);
+	}
+	template <typename ...Args> inline std::variant<Args...> get_col_from_db(sqlite3_stmt* stmt, int inx, result_type<std::variant<Args...>>) {
+		return detail::variant_select<Args...>(sqlite3_column_type(stmt, inx), [&](auto v) {
+			return std::variant<Args...>(std::in_place_type<typename decltype(v)::type>, get_col_from_db(stmt, inx, v));
+		});
+	}
+	template <typename ...Args> inline std::variant<Args...> get_val_from_db(sqlite3_value *value, result_type<std::variant<Args...>>) {
+		return detail::variant_select<Args...>(sqlite3_value_type(value), [&](auto v) {
+			return std::variant<Args...>(std::in_place_type<typename decltype(v)::type>, get_val_from_db(value, v));
+		});
+	}
+#endif
+}

src/claimtrie/sqlite/hdr/sqlite_modern_cpp/utility/function_traits.h

@@ -0,0 +1,56 @@
+#pragma once
+
+#include <tuple>
+#include <type_traits>
+
+namespace sqlite {
+	namespace utility {
+
+		template<typename> struct function_traits;
+
+		template <typename Function>
+		struct function_traits : public function_traits<
+			decltype(&std::remove_reference<Function>::type::operator())
+		> { };
+
+		template <
+			typename    ClassType,
+			typename    ReturnType,
+			typename... Arguments
+		>
+		struct function_traits<
+			ReturnType(ClassType::*)(Arguments...) const
+		> : function_traits<ReturnType(*)(Arguments...)> { };
+
+    /* support the non-const operator ()
+     * this will work with user defined functors */
+		template <
+			typename    ClassType,
+			typename    ReturnType,
+			typename... Arguments
+		>
+		struct function_traits<
+			ReturnType(ClassType::*)(Arguments...)
+		> : function_traits<ReturnType(*)(Arguments...)> { };
+
+		template <
+			typename    ReturnType,
+			typename... Arguments
+		>
+		struct function_traits<
+			ReturnType(*)(Arguments...)
+		> {
+			typedef ReturnType result_type;
+
+			using argument_tuple = std::tuple<Arguments...>;
+			template <std::size_t Index>
+			using argument = typename std::tuple_element<
+				Index,
+				argument_tuple
+			>::type;
+
+			static const std::size_t arity = sizeof...(Arguments);
+		};
+
+	}
+}

src/claimtrie/sqlite/hdr/sqlite_modern_cpp/utility/uncaught_exceptions.h

@@ -0,0 +1,41 @@
+#pragma once
+
+#include <cassert>
+#include <exception>
+#include <iostream>
+
+// Consider that std::uncaught_exceptions is available if explicitly indicated
+// by the standard library, if compiler advertises full C++17 support or, as a
+// special case, for MSVS 2015+ (which doesn't define __cplusplus correctly by
+// default as of 2017.7 version and couldn't do it at all until it).
+#ifndef MODERN_SQLITE_UNCAUGHT_EXCEPTIONS_SUPPORT
+#ifdef __cpp_lib_uncaught_exceptions
+#define MODERN_SQLITE_UNCAUGHT_EXCEPTIONS_SUPPORT
+#elif __cplusplus >= 201703L
+#define MODERN_SQLITE_UNCAUGHT_EXCEPTIONS_SUPPORT
+#elif defined(_MSC_VER) && _MSC_VER >= 1900
+#define MODERN_SQLITE_UNCAUGHT_EXCEPTIONS_SUPPORT
+#endif
+#endif
+
+namespace sqlite {
+	namespace utility {
+#ifdef MODERN_SQLITE_UNCAUGHT_EXCEPTIONS_SUPPORT
+		class UncaughtExceptionDetector {
+		public:
+			operator bool() {
+				return count != std::uncaught_exceptions();
+			}
+		private:
+			int count = std::uncaught_exceptions();
+		};
+#else
+		class UncaughtExceptionDetector {
+		public:
+			operator bool() {
+				return std::uncaught_exception();
+			}
+		};
+#endif
+	}
+}

src/claimtrie/sqlite/hdr/sqlite_modern_cpp/utility/utf16_utf8.h

@@ -0,0 +1,42 @@
+#pragma once
+
+#include <locale>
+#include <string>
+#include <algorithm>
+
+#include "../errors.h"
+
+namespace sqlite {
+	namespace utility {
+		inline std::string utf16_to_utf8(u16str_ref input) {
+			struct : std::codecvt<char16_t, char, std::mbstate_t> {
+			} codecvt;
+			std::mbstate_t state{};
+			std::string result((std::max)(input.size() * 3 / 2, std::size_t(4)), '\0');
+			const char16_t *remaining_input = input.data();
+			std::size_t produced_output = 0;
+			while(true) {
+				char *used_output;
+				switch(codecvt.out(state, remaining_input, &input[input.size()],
+				                   remaining_input, &result[produced_output],
+				                   &result[result.size() - 1] + 1, used_output)) {
+				case std::codecvt_base::ok:
+					result.resize(used_output - result.data());
+					return result;
+				case std::codecvt_base::noconv:
+					// This should be unreachable
+				case std::codecvt_base::error:
+					throw errors::invalid_utf16("Invalid UTF-16 input", "");
+				case std::codecvt_base::partial:
+					if(used_output == result.data() + produced_output)
+						throw errors::invalid_utf16("Unexpected end of input", "");
+					produced_output = used_output - result.data();
+					result.resize(
+							result.size()
+							+ (std::max)((&input[input.size()] - remaining_input) * 3 / 2,
+							           std::ptrdiff_t(4)));
+				}
+			}
+		}
+	} // namespace utility
+} // namespace sqlite

src/claimtrie/sqlite/sqlite3ext.h

@@ -0,0 +1,638 @@
+/*
+** 2006 June 7
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This header file defines the SQLite interface for use by
+** shared libraries that want to be imported as extensions into
+** an SQLite instance.  Shared libraries that intend to be loaded
+** as extensions by SQLite should #include this file instead of 
+** sqlite3.h.
+*/
+#ifndef SQLITE3EXT_H
+#define SQLITE3EXT_H
+#include "sqlite3.h"
+
+/*
+** The following structure holds pointers to all of the SQLite API
+** routines.
+**
+** WARNING:  In order to maintain backwards compatibility, add new
+** interfaces to the end of this structure only.  If you insert new
+** interfaces in the middle of this structure, then older different
+** versions of SQLite will not be able to load each other's shared
+** libraries!
+*/
+struct sqlite3_api_routines {
+  void * (*aggregate_context)(sqlite3_context*,int nBytes);
+  int  (*aggregate_count)(sqlite3_context*);
+  int  (*bind_blob)(sqlite3_stmt*,int,const void*,int n,void(*)(void*));
+  int  (*bind_double)(sqlite3_stmt*,int,double);
+  int  (*bind_int)(sqlite3_stmt*,int,int);
+  int  (*bind_int64)(sqlite3_stmt*,int,sqlite_int64);
+  int  (*bind_null)(sqlite3_stmt*,int);
+  int  (*bind_parameter_count)(sqlite3_stmt*);
+  int  (*bind_parameter_index)(sqlite3_stmt*,const char*zName);
+  const char * (*bind_parameter_name)(sqlite3_stmt*,int);
+  int  (*bind_text)(sqlite3_stmt*,int,const char*,int n,void(*)(void*));
+  int  (*bind_text16)(sqlite3_stmt*,int,const void*,int,void(*)(void*));
+  int  (*bind_value)(sqlite3_stmt*,int,const sqlite3_value*);
+  int  (*busy_handler)(sqlite3*,int(*)(void*,int),void*);
+  int  (*busy_timeout)(sqlite3*,int ms);
+  int  (*changes)(sqlite3*);
+  int  (*close)(sqlite3*);
+  int  (*collation_needed)(sqlite3*,void*,void(*)(void*,sqlite3*,
+                           int eTextRep,const char*));
+  int  (*collation_needed16)(sqlite3*,void*,void(*)(void*,sqlite3*,
+                             int eTextRep,const void*));
+  const void * (*column_blob)(sqlite3_stmt*,int iCol);
+  int  (*column_bytes)(sqlite3_stmt*,int iCol);
+  int  (*column_bytes16)(sqlite3_stmt*,int iCol);
+  int  (*column_count)(sqlite3_stmt*pStmt);
+  const char * (*column_database_name)(sqlite3_stmt*,int);
+  const void * (*column_database_name16)(sqlite3_stmt*,int);
+  const char * (*column_decltype)(sqlite3_stmt*,int i);
+  const void * (*column_decltype16)(sqlite3_stmt*,int);
+  double  (*column_double)(sqlite3_stmt*,int iCol);
+  int  (*column_int)(sqlite3_stmt*,int iCol);
+  sqlite_int64  (*column_int64)(sqlite3_stmt*,int iCol);
+  const char * (*column_name)(sqlite3_stmt*,int);
+  const void * (*column_name16)(sqlite3_stmt*,int);
+  const char * (*column_origin_name)(sqlite3_stmt*,int);
+  const void * (*column_origin_name16)(sqlite3_stmt*,int);
+  const char * (*column_table_name)(sqlite3_stmt*,int);
+  const void * (*column_table_name16)(sqlite3_stmt*,int);
+  const unsigned char * (*column_text)(sqlite3_stmt*,int iCol);
+  const void * (*column_text16)(sqlite3_stmt*,int iCol);
+  int  (*column_type)(sqlite3_stmt*,int iCol);
+  sqlite3_value* (*column_value)(sqlite3_stmt*,int iCol);
+  void * (*commit_hook)(sqlite3*,int(*)(void*),void*);
+  int  (*complete)(const char*sql);
+  int  (*complete16)(const void*sql);
+  int  (*create_collation)(sqlite3*,const char*,int,void*,
+                           int(*)(void*,int,const void*,int,const void*));
+  int  (*create_collation16)(sqlite3*,const void*,int,void*,
+                             int(*)(void*,int,const void*,int,const void*));
+  int  (*create_function)(sqlite3*,const char*,int,int,void*,
+                          void (*xFunc)(sqlite3_context*,int,sqlite3_value**),
+                          void (*xStep)(sqlite3_context*,int,sqlite3_value**),
+                          void (*xFinal)(sqlite3_context*));
+  int  (*create_function16)(sqlite3*,const void*,int,int,void*,
+                            void (*xFunc)(sqlite3_context*,int,sqlite3_value**),
+                            void (*xStep)(sqlite3_context*,int,sqlite3_value**),
+                            void (*xFinal)(sqlite3_context*));
+  int (*create_module)(sqlite3*,const char*,const sqlite3_module*,void*);
+  int  (*data_count)(sqlite3_stmt*pStmt);
+  sqlite3 * (*db_handle)(sqlite3_stmt*);
+  int (*declare_vtab)(sqlite3*,const char*);
+  int  (*enable_shared_cache)(int);
+  int  (*errcode)(sqlite3*db);
+  const char * (*errmsg)(sqlite3*);
+  const void * (*errmsg16)(sqlite3*);
+  int  (*exec)(sqlite3*,const char*,sqlite3_callback,void*,char**);
+  int  (*expired)(sqlite3_stmt*);
+  int  (*finalize)(sqlite3_stmt*pStmt);
+  void  (*free)(void*);
+  void  (*free_table)(char**result);
+  int  (*get_autocommit)(sqlite3*);
+  void * (*get_auxdata)(sqlite3_context*,int);
+  int  (*get_table)(sqlite3*,const char*,char***,int*,int*,char**);
+  int  (*global_recover)(void);
+  void  (*interruptx)(sqlite3*);
+  sqlite_int64  (*last_insert_rowid)(sqlite3*);
+  const char * (*libversion)(void);
+  int  (*libversion_number)(void);
+  void *(*malloc)(int);
+  char * (*mprintf)(const char*,...);
+  int  (*open)(const char*,sqlite3**);
+  int  (*open16)(const void*,sqlite3**);
+  int  (*prepare)(sqlite3*,const char*,int,sqlite3_stmt**,const char**);
+  int  (*prepare16)(sqlite3*,const void*,int,sqlite3_stmt**,const void**);
+  void * (*profile)(sqlite3*,void(*)(void*,const char*,sqlite_uint64),void*);
+  void  (*progress_handler)(sqlite3*,int,int(*)(void*),void*);
+  void *(*realloc)(void*,int);
+  int  (*reset)(sqlite3_stmt*pStmt);
+  void  (*result_blob)(sqlite3_context*,const void*,int,void(*)(void*));
+  void  (*result_double)(sqlite3_context*,double);
+  void  (*result_error)(sqlite3_context*,const char*,int);
+  void  (*result_error16)(sqlite3_context*,const void*,int);
+  void  (*result_int)(sqlite3_context*,int);
+  void  (*result_int64)(sqlite3_context*,sqlite_int64);
+  void  (*result_null)(sqlite3_context*);
+  void  (*result_text)(sqlite3_context*,const char*,int,void(*)(void*));
+  void  (*result_text16)(sqlite3_context*,const void*,int,void(*)(void*));
+  void  (*result_text16be)(sqlite3_context*,const void*,int,void(*)(void*));
+  void  (*result_text16le)(sqlite3_context*,const void*,int,void(*)(void*));
+  void  (*result_value)(sqlite3_context*,sqlite3_value*);
+  void * (*rollback_hook)(sqlite3*,void(*)(void*),void*);
+  int  (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*,
+                         const char*,const char*),void*);
+  void  (*set_auxdata)(sqlite3_context*,int,void*,void (*)(void*));
+  char * (*xsnprintf)(int,char*,const char*,...);
+  int  (*step)(sqlite3_stmt*);
+  int  (*table_column_metadata)(sqlite3*,const char*,const char*,const char*,
+                                char const**,char const**,int*,int*,int*);
+  void  (*thread_cleanup)(void);
+  int  (*total_changes)(sqlite3*);
+  void * (*trace)(sqlite3*,void(*xTrace)(void*,const char*),void*);
+  int  (*transfer_bindings)(sqlite3_stmt*,sqlite3_stmt*);
+  void * (*update_hook)(sqlite3*,void(*)(void*,int ,char const*,char const*,
+                                         sqlite_int64),void*);
+  void * (*user_data)(sqlite3_context*);
+  const void * (*value_blob)(sqlite3_value*);
+  int  (*value_bytes)(sqlite3_value*);
+  int  (*value_bytes16)(sqlite3_value*);
+  double  (*value_double)(sqlite3_value*);
+  int  (*value_int)(sqlite3_value*);
+  sqlite_int64  (*value_int64)(sqlite3_value*);
+  int  (*value_numeric_type)(sqlite3_value*);
+  const unsigned char * (*value_text)(sqlite3_value*);
+  const void * (*value_text16)(sqlite3_value*);
+  const void * (*value_text16be)(sqlite3_value*);
+  const void * (*value_text16le)(sqlite3_value*);
+  int  (*value_type)(sqlite3_value*);
+  char *(*vmprintf)(const char*,va_list);
+  /* Added ??? */
+  int (*overload_function)(sqlite3*, const char *zFuncName, int nArg);
+  /* Added by 3.3.13 */
+  int (*prepare_v2)(sqlite3*,const char*,int,sqlite3_stmt**,const char**);
+  int (*prepare16_v2)(sqlite3*,const void*,int,sqlite3_stmt**,const void**);
+  int (*clear_bindings)(sqlite3_stmt*);
+  /* Added by 3.4.1 */
+  int (*create_module_v2)(sqlite3*,const char*,const sqlite3_module*,void*,
+                          void (*xDestroy)(void *));
+  /* Added by 3.5.0 */
+  int (*bind_zeroblob)(sqlite3_stmt*,int,int);
+  int (*blob_bytes)(sqlite3_blob*);
+  int (*blob_close)(sqlite3_blob*);
+  int (*blob_open)(sqlite3*,const char*,const char*,const char*,sqlite3_int64,
+                   int,sqlite3_blob**);
+  int (*blob_read)(sqlite3_blob*,void*,int,int);
+  int (*blob_write)(sqlite3_blob*,const void*,int,int);
+  int (*create_collation_v2)(sqlite3*,const char*,int,void*,
+                             int(*)(void*,int,const void*,int,const void*),
+                             void(*)(void*));
+  int (*file_control)(sqlite3*,const char*,int,void*);
+  sqlite3_int64 (*memory_highwater)(int);
+  sqlite3_int64 (*memory_used)(void);
+  sqlite3_mutex *(*mutex_alloc)(int);
+  void (*mutex_enter)(sqlite3_mutex*);
+  void (*mutex_free)(sqlite3_mutex*);
+  void (*mutex_leave)(sqlite3_mutex*);
+  int (*mutex_try)(sqlite3_mutex*);
+  int (*open_v2)(const char*,sqlite3**,int,const char*);
+  int (*release_memory)(int);
+  void (*result_error_nomem)(sqlite3_context*);
+  void (*result_error_toobig)(sqlite3_context*);
+  int (*sleep)(int);
+  void (*soft_heap_limit)(int);
+  sqlite3_vfs *(*vfs_find)(const char*);
+  int (*vfs_register)(sqlite3_vfs*,int);
+  int (*vfs_unregister)(sqlite3_vfs*);
+  int (*xthreadsafe)(void);
+  void (*result_zeroblob)(sqlite3_context*,int);
+  void (*result_error_code)(sqlite3_context*,int);
+  int (*test_control)(int, ...);
+  void (*randomness)(int,void*);
+  sqlite3 *(*context_db_handle)(sqlite3_context*);
+  int (*extended_result_codes)(sqlite3*,int);
+  int (*limit)(sqlite3*,int,int);
+  sqlite3_stmt *(*next_stmt)(sqlite3*,sqlite3_stmt*);
+  const char *(*sql)(sqlite3_stmt*);
+  int (*status)(int,int*,int*,int);
+  int (*backup_finish)(sqlite3_backup*);
+  sqlite3_backup *(*backup_init)(sqlite3*,const char*,sqlite3*,const char*);
+  int (*backup_pagecount)(sqlite3_backup*);
+  int (*backup_remaining)(sqlite3_backup*);
+  int (*backup_step)(sqlite3_backup*,int);
+  const char *(*compileoption_get)(int);
+  int (*compileoption_used)(const char*);
+  int (*create_function_v2)(sqlite3*,const char*,int,int,void*,
+                            void (*xFunc)(sqlite3_context*,int,sqlite3_value**),
+                            void (*xStep)(sqlite3_context*,int,sqlite3_value**),
+                            void (*xFinal)(sqlite3_context*),
+                            void(*xDestroy)(void*));
+  int (*db_config)(sqlite3*,int,...);
+  sqlite3_mutex *(*db_mutex)(sqlite3*);
+  int (*db_status)(sqlite3*,int,int*,int*,int);
+  int (*extended_errcode)(sqlite3*);
+  void (*log)(int,const char*,...);
+  sqlite3_int64 (*soft_heap_limit64)(sqlite3_int64);
+  const char *(*sourceid)(void);
+  int (*stmt_status)(sqlite3_stmt*,int,int);
+  int (*strnicmp)(const char*,const char*,int);
+  int (*unlock_notify)(sqlite3*,void(*)(void**,int),void*);
+  int (*wal_autocheckpoint)(sqlite3*,int);
+  int (*wal_checkpoint)(sqlite3*,const char*);
+  void *(*wal_hook)(sqlite3*,int(*)(void*,sqlite3*,const char*,int),void*);
+  int (*blob_reopen)(sqlite3_blob*,sqlite3_int64);
+  int (*vtab_config)(sqlite3*,int op,...);
+  int (*vtab_on_conflict)(sqlite3*);
+  /* Version 3.7.16 and later */
+  int (*close_v2)(sqlite3*);
+  const char *(*db_filename)(sqlite3*,const char*);
+  int (*db_readonly)(sqlite3*,const char*);
+  int (*db_release_memory)(sqlite3*);
+  const char *(*errstr)(int);
+  int (*stmt_busy)(sqlite3_stmt*);
+  int (*stmt_readonly)(sqlite3_stmt*);
+  int (*stricmp)(const char*,const char*);
+  int (*uri_boolean)(const char*,const char*,int);
+  sqlite3_int64 (*uri_int64)(const char*,const char*,sqlite3_int64);
+  const char *(*uri_parameter)(const char*,const char*);
+  char *(*xvsnprintf)(int,char*,const char*,va_list);
+  int (*wal_checkpoint_v2)(sqlite3*,const char*,int,int*,int*);
+  /* Version 3.8.7 and later */
+  int (*auto_extension)(void(*)(void));
+  int (*bind_blob64)(sqlite3_stmt*,int,const void*,sqlite3_uint64,
+                     void(*)(void*));
+  int (*bind_text64)(sqlite3_stmt*,int,const char*,sqlite3_uint64,
+                      void(*)(void*),unsigned char);
+  int (*cancel_auto_extension)(void(*)(void));
+  int (*load_extension)(sqlite3*,const char*,const char*,char**);
+  void *(*malloc64)(sqlite3_uint64);
+  sqlite3_uint64 (*msize)(void*);
+  void *(*realloc64)(void*,sqlite3_uint64);
+  void (*reset_auto_extension)(void);
+  void (*result_blob64)(sqlite3_context*,const void*,sqlite3_uint64,
+                        void(*)(void*));
+  void (*result_text64)(sqlite3_context*,const char*,sqlite3_uint64,
+                         void(*)(void*), unsigned char);
+  int (*strglob)(const char*,const char*);
+  /* Version 3.8.11 and later */
+  sqlite3_value *(*value_dup)(const sqlite3_value*);
+  void (*value_free)(sqlite3_value*);
+  int (*result_zeroblob64)(sqlite3_context*,sqlite3_uint64);
+  int (*bind_zeroblob64)(sqlite3_stmt*, int, sqlite3_uint64);
+  /* Version 3.9.0 and later */
+  unsigned int (*value_subtype)(sqlite3_value*);
+  void (*result_subtype)(sqlite3_context*,unsigned int);
+  /* Version 3.10.0 and later */
+  int (*status64)(int,sqlite3_int64*,sqlite3_int64*,int);
+  int (*strlike)(const char*,const char*,unsigned int);
+  int (*db_cacheflush)(sqlite3*);
+  /* Version 3.12.0 and later */
+  int (*system_errno)(sqlite3*);
+  /* Version 3.14.0 and later */
+  int (*trace_v2)(sqlite3*,unsigned,int(*)(unsigned,void*,void*,void*),void*);
+  char *(*expanded_sql)(sqlite3_stmt*);
+  /* Version 3.18.0 and later */
+  void (*set_last_insert_rowid)(sqlite3*,sqlite3_int64);
+  /* Version 3.20.0 and later */
+  int (*prepare_v3)(sqlite3*,const char*,int,unsigned int,
+                    sqlite3_stmt**,const char**);
+  int (*prepare16_v3)(sqlite3*,const void*,int,unsigned int,
+                      sqlite3_stmt**,const void**);
+  int (*bind_pointer)(sqlite3_stmt*,int,void*,const char*,void(*)(void*));
+  void (*result_pointer)(sqlite3_context*,void*,const char*,void(*)(void*));
+  void *(*value_pointer)(sqlite3_value*,const char*);
+  int (*vtab_nochange)(sqlite3_context*);
+  int (*value_nochange)(sqlite3_value*);
+  const char *(*vtab_collation)(sqlite3_index_info*,int);
+  /* Version 3.24.0 and later */
+  int (*keyword_count)(void);
+  int (*keyword_name)(int,const char**,int*);
+  int (*keyword_check)(const char*,int);
+  sqlite3_str *(*str_new)(sqlite3*);
+  char *(*str_finish)(sqlite3_str*);
+  void (*str_appendf)(sqlite3_str*, const char *zFormat, ...);
+  void (*str_vappendf)(sqlite3_str*, const char *zFormat, va_list);
+  void (*str_append)(sqlite3_str*, const char *zIn, int N);
+  void (*str_appendall)(sqlite3_str*, const char *zIn);
+  void (*str_appendchar)(sqlite3_str*, int N, char C);
+  void (*str_reset)(sqlite3_str*);
+  int (*str_errcode)(sqlite3_str*);
+  int (*str_length)(sqlite3_str*);
+  char *(*str_value)(sqlite3_str*);
+  /* Version 3.25.0 and later */
+  int (*create_window_function)(sqlite3*,const char*,int,int,void*,
+                            void (*xStep)(sqlite3_context*,int,sqlite3_value**),
+                            void (*xFinal)(sqlite3_context*),
+                            void (*xValue)(sqlite3_context*),
+                            void (*xInv)(sqlite3_context*,int,sqlite3_value**),
+                            void(*xDestroy)(void*));
+  /* Version 3.26.0 and later */
+  const char *(*normalized_sql)(sqlite3_stmt*);
+  /* Version 3.28.0 and later */
+  int (*stmt_isexplain)(sqlite3_stmt*);
+  int (*value_frombind)(sqlite3_value*);
+  /* Version 3.30.0 and later */
+  int (*drop_modules)(sqlite3*,const char**);
+};
+
+/*
+** This is the function signature used for all extension entry points.  It
+** is also defined in the file "loadext.c".
+*/
+typedef int (*sqlite3_loadext_entry)(
+  sqlite3 *db,                       /* Handle to the database. */
+  char **pzErrMsg,                   /* Used to set error string on failure. */
+  const sqlite3_api_routines *pThunk /* Extension API function pointers. */
+);
+
+/*
+** The following macros redefine the API routines so that they are
+** redirected through the global sqlite3_api structure.
+**
+** This header file is also used by the loadext.c source file
+** (part of the main SQLite library - not an extension) so that
+** it can get access to the sqlite3_api_routines structure
+** definition.  But the main library does not want to redefine
+** the API.  So the redefinition macros are only valid if the
+** SQLITE_CORE macros is undefined.
+*/
+#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)
+#define sqlite3_aggregate_context      sqlite3_api->aggregate_context
+#ifndef SQLITE_OMIT_DEPRECATED
+#define sqlite3_aggregate_count        sqlite3_api->aggregate_count
+#endif
+#define sqlite3_bind_blob              sqlite3_api->bind_blob
+#define sqlite3_bind_double            sqlite3_api->bind_double
+#define sqlite3_bind_int               sqlite3_api->bind_int
+#define sqlite3_bind_int64             sqlite3_api->bind_int64
+#define sqlite3_bind_null              sqlite3_api->bind_null
+#define sqlite3_bind_parameter_count   sqlite3_api->bind_parameter_count
+#define sqlite3_bind_parameter_index   sqlite3_api->bind_parameter_index
+#define sqlite3_bind_parameter_name    sqlite3_api->bind_parameter_name
+#define sqlite3_bind_text              sqlite3_api->bind_text
+#define sqlite3_bind_text16            sqlite3_api->bind_text16
+#define sqlite3_bind_value             sqlite3_api->bind_value
+#define sqlite3_busy_handler           sqlite3_api->busy_handler
+#define sqlite3_busy_timeout           sqlite3_api->busy_timeout
+#define sqlite3_changes                sqlite3_api->changes
+#define sqlite3_close                  sqlite3_api->close
+#define sqlite3_collation_needed       sqlite3_api->collation_needed
+#define sqlite3_collation_needed16     sqlite3_api->collation_needed16
+#define sqlite3_column_blob            sqlite3_api->column_blob
+#define sqlite3_column_bytes           sqlite3_api->column_bytes
+#define sqlite3_column_bytes16         sqlite3_api->column_bytes16
+#define sqlite3_column_count           sqlite3_api->column_count
+#define sqlite3_column_database_name   sqlite3_api->column_database_name
+#define sqlite3_column_database_name16 sqlite3_api->column_database_name16
+#define sqlite3_column_decltype        sqlite3_api->column_decltype
+#define sqlite3_column_decltype16      sqlite3_api->column_decltype16
+#define sqlite3_column_double          sqlite3_api->column_double
+#define sqlite3_column_int             sqlite3_api->column_int
+#define sqlite3_column_int64           sqlite3_api->column_int64
+#define sqlite3_column_name            sqlite3_api->column_name
+#define sqlite3_column_name16          sqlite3_api->column_name16
+#define sqlite3_column_origin_name     sqlite3_api->column_origin_name
+#define sqlite3_column_origin_name16   sqlite3_api->column_origin_name16
+#define sqlite3_column_table_name      sqlite3_api->column_table_name
+#define sqlite3_column_table_name16    sqlite3_api->column_table_name16
+#define sqlite3_column_text            sqlite3_api->column_text
+#define sqlite3_column_text16          sqlite3_api->column_text16
+#define sqlite3_column_type            sqlite3_api->column_type
+#define sqlite3_column_value           sqlite3_api->column_value
+#define sqlite3_commit_hook            sqlite3_api->commit_hook
+#define sqlite3_complete               sqlite3_api->complete
+#define sqlite3_complete16             sqlite3_api->complete16
+#define sqlite3_create_collation       sqlite3_api->create_collation
+#define sqlite3_create_collation16     sqlite3_api->create_collation16
+#define sqlite3_create_function        sqlite3_api->create_function
+#define sqlite3_create_function16      sqlite3_api->create_function16
+#define sqlite3_create_module          sqlite3_api->create_module
+#define sqlite3_create_module_v2       sqlite3_api->create_module_v2
+#define sqlite3_data_count             sqlite3_api->data_count
+#define sqlite3_db_handle              sqlite3_api->db_handle
+#define sqlite3_declare_vtab           sqlite3_api->declare_vtab
+#define sqlite3_enable_shared_cache    sqlite3_api->enable_shared_cache
+#define sqlite3_errcode                sqlite3_api->errcode
+#define sqlite3_errmsg                 sqlite3_api->errmsg
+#define sqlite3_errmsg16               sqlite3_api->errmsg16
+#define sqlite3_exec                   sqlite3_api->exec
+#ifndef SQLITE_OMIT_DEPRECATED
+#define sqlite3_expired                sqlite3_api->expired
+#endif
+#define sqlite3_finalize               sqlite3_api->finalize
+#define sqlite3_free                   sqlite3_api->free
+#define sqlite3_free_table             sqlite3_api->free_table
+#define sqlite3_get_autocommit         sqlite3_api->get_autocommit
+#define sqlite3_get_auxdata            sqlite3_api->get_auxdata
+#define sqlite3_get_table              sqlite3_api->get_table
+#ifndef SQLITE_OMIT_DEPRECATED
+#define sqlite3_global_recover         sqlite3_api->global_recover
+#endif
+#define sqlite3_interrupt              sqlite3_api->interruptx
+#define sqlite3_last_insert_rowid      sqlite3_api->last_insert_rowid
+#define sqlite3_libversion             sqlite3_api->libversion
+#define sqlite3_libversion_number      sqlite3_api->libversion_number
+#define sqlite3_malloc                 sqlite3_api->malloc
+#define sqlite3_mprintf                sqlite3_api->mprintf
+#define sqlite3_open                   sqlite3_api->open
+#define sqlite3_open16                 sqlite3_api->open16
+#define sqlite3_prepare                sqlite3_api->prepare
+#define sqlite3_prepare16              sqlite3_api->prepare16
+#define sqlite3_prepare_v2             sqlite3_api->prepare_v2
+#define sqlite3_prepare16_v2           sqlite3_api->prepare16_v2
+#define sqlite3_profile                sqlite3_api->profile
+#define sqlite3_progress_handler       sqlite3_api->progress_handler
+#define sqlite3_realloc                sqlite3_api->realloc
+#define sqlite3_reset                  sqlite3_api->reset
+#define sqlite3_result_blob            sqlite3_api->result_blob
+#define sqlite3_result_double          sqlite3_api->result_double
+#define sqlite3_result_error           sqlite3_api->result_error
+#define sqlite3_result_error16         sqlite3_api->result_error16
+#define sqlite3_result_int             sqlite3_api->result_int
+#define sqlite3_result_int64           sqlite3_api->result_int64
+#define sqlite3_result_null            sqlite3_api->result_null
+#define sqlite3_result_text            sqlite3_api->result_text
+#define sqlite3_result_text16          sqlite3_api->result_text16
+#define sqlite3_result_text16be        sqlite3_api->result_text16be
+#define sqlite3_result_text16le        sqlite3_api->result_text16le
+#define sqlite3_result_value           sqlite3_api->result_value
+#define sqlite3_rollback_hook          sqlite3_api->rollback_hook
+#define sqlite3_set_authorizer         sqlite3_api->set_authorizer
+#define sqlite3_set_auxdata            sqlite3_api->set_auxdata
+#define sqlite3_snprintf               sqlite3_api->xsnprintf
+#define sqlite3_step                   sqlite3_api->step
+#define sqlite3_table_column_metadata  sqlite3_api->table_column_metadata
+#define sqlite3_thread_cleanup         sqlite3_api->thread_cleanup
+#define sqlite3_total_changes          sqlite3_api->total_changes
+#define sqlite3_trace                  sqlite3_api->trace
+#ifndef SQLITE_OMIT_DEPRECATED
+#define sqlite3_transfer_bindings      sqlite3_api->transfer_bindings
+#endif
+#define sqlite3_update_hook            sqlite3_api->update_hook
+#define sqlite3_user_data              sqlite3_api->user_data
+#define sqlite3_value_blob             sqlite3_api->value_blob
+#define sqlite3_value_bytes            sqlite3_api->value_bytes
+#define sqlite3_value_bytes16          sqlite3_api->value_bytes16
+#define sqlite3_value_double           sqlite3_api->value_double
+#define sqlite3_value_int              sqlite3_api->value_int
+#define sqlite3_value_int64            sqlite3_api->value_int64
+#define sqlite3_value_numeric_type     sqlite3_api->value_numeric_type
+#define sqlite3_value_text             sqlite3_api->value_text
+#define sqlite3_value_text16           sqlite3_api->value_text16
+#define sqlite3_value_text16be         sqlite3_api->value_text16be
+#define sqlite3_value_text16le         sqlite3_api->value_text16le
+#define sqlite3_value_type             sqlite3_api->value_type
+#define sqlite3_vmprintf               sqlite3_api->vmprintf
+#define sqlite3_vsnprintf              sqlite3_api->xvsnprintf
+#define sqlite3_overload_function      sqlite3_api->overload_function
+#define sqlite3_prepare_v2             sqlite3_api->prepare_v2
+#define sqlite3_prepare16_v2           sqlite3_api->prepare16_v2
+#define sqlite3_clear_bindings         sqlite3_api->clear_bindings
+#define sqlite3_bind_zeroblob          sqlite3_api->bind_zeroblob
+#define sqlite3_blob_bytes             sqlite3_api->blob_bytes
+#define sqlite3_blob_close             sqlite3_api->blob_close
+#define sqlite3_blob_open              sqlite3_api->blob_open
+#define sqlite3_blob_read              sqlite3_api->blob_read
+#define sqlite3_blob_write             sqlite3_api->blob_write
+#define sqlite3_create_collation_v2    sqlite3_api->create_collation_v2
+#define sqlite3_file_control           sqlite3_api->file_control
+#define sqlite3_memory_highwater       sqlite3_api->memory_highwater
+#define sqlite3_memory_used            sqlite3_api->memory_used
+#define sqlite3_mutex_alloc            sqlite3_api->mutex_alloc
+#define sqlite3_mutex_enter            sqlite3_api->mutex_enter
+#define sqlite3_mutex_free             sqlite3_api->mutex_free
+#define sqlite3_mutex_leave            sqlite3_api->mutex_leave
+#define sqlite3_mutex_try              sqlite3_api->mutex_try
+#define sqlite3_open_v2                sqlite3_api->open_v2
+#define sqlite3_release_memory         sqlite3_api->release_memory
+#define sqlite3_result_error_nomem     sqlite3_api->result_error_nomem
+#define sqlite3_result_error_toobig    sqlite3_api->result_error_toobig
+#define sqlite3_sleep                  sqlite3_api->sleep
+#define sqlite3_soft_heap_limit        sqlite3_api->soft_heap_limit
+#define sqlite3_vfs_find               sqlite3_api->vfs_find
+#define sqlite3_vfs_register           sqlite3_api->vfs_register
+#define sqlite3_vfs_unregister         sqlite3_api->vfs_unregister
+#define sqlite3_threadsafe             sqlite3_api->xthreadsafe
+#define sqlite3_result_zeroblob        sqlite3_api->result_zeroblob
+#define sqlite3_result_error_code      sqlite3_api->result_error_code
+#define sqlite3_test_control           sqlite3_api->test_control
+#define sqlite3_randomness             sqlite3_api->randomness
+#define sqlite3_context_db_handle      sqlite3_api->context_db_handle
+#define sqlite3_extended_result_codes  sqlite3_api->extended_result_codes
+#define sqlite3_limit                  sqlite3_api->limit
+#define sqlite3_next_stmt              sqlite3_api->next_stmt
+#define sqlite3_sql                    sqlite3_api->sql
+#define sqlite3_status                 sqlite3_api->status
+#define sqlite3_backup_finish          sqlite3_api->backup_finish
+#define sqlite3_backup_init            sqlite3_api->backup_init
+#define sqlite3_backup_pagecount       sqlite3_api->backup_pagecount
+#define sqlite3_backup_remaining       sqlite3_api->backup_remaining
+#define sqlite3_backup_step            sqlite3_api->backup_step
+#define sqlite3_compileoption_get      sqlite3_api->compileoption_get
+#define sqlite3_compileoption_used     sqlite3_api->compileoption_used
+#define sqlite3_create_function_v2     sqlite3_api->create_function_v2
+#define sqlite3_db_config              sqlite3_api->db_config
+#define sqlite3_db_mutex               sqlite3_api->db_mutex
+#define sqlite3_db_status              sqlite3_api->db_status
+#define sqlite3_extended_errcode       sqlite3_api->extended_errcode
+#define sqlite3_log                    sqlite3_api->log
+#define sqlite3_soft_heap_limit64      sqlite3_api->soft_heap_limit64
+#define sqlite3_sourceid               sqlite3_api->sourceid
+#define sqlite3_stmt_status            sqlite3_api->stmt_status
+#define sqlite3_strnicmp               sqlite3_api->strnicmp
+#define sqlite3_unlock_notify          sqlite3_api->unlock_notify
+#define sqlite3_wal_autocheckpoint     sqlite3_api->wal_autocheckpoint
+#define sqlite3_wal_checkpoint         sqlite3_api->wal_checkpoint
+#define sqlite3_wal_hook               sqlite3_api->wal_hook
+#define sqlite3_blob_reopen            sqlite3_api->blob_reopen
+#define sqlite3_vtab_config            sqlite3_api->vtab_config
+#define sqlite3_vtab_on_conflict       sqlite3_api->vtab_on_conflict
+/* Version 3.7.16 and later */
+#define sqlite3_close_v2               sqlite3_api->close_v2
+#define sqlite3_db_filename            sqlite3_api->db_filename
+#define sqlite3_db_readonly            sqlite3_api->db_readonly
+#define sqlite3_db_release_memory      sqlite3_api->db_release_memory
+#define sqlite3_errstr                 sqlite3_api->errstr
+#define sqlite3_stmt_busy              sqlite3_api->stmt_busy
+#define sqlite3_stmt_readonly          sqlite3_api->stmt_readonly
+#define sqlite3_stricmp                sqlite3_api->stricmp
+#define sqlite3_uri_boolean            sqlite3_api->uri_boolean
+#define sqlite3_uri_int64              sqlite3_api->uri_int64
+#define sqlite3_uri_parameter          sqlite3_api->uri_parameter
+#define sqlite3_uri_vsnprintf          sqlite3_api->xvsnprintf
+#define sqlite3_wal_checkpoint_v2      sqlite3_api->wal_checkpoint_v2
+/* Version 3.8.7 and later */
+#define sqlite3_auto_extension         sqlite3_api->auto_extension
+#define sqlite3_bind_blob64            sqlite3_api->bind_blob64
+#define sqlite3_bind_text64            sqlite3_api->bind_text64
+#define sqlite3_cancel_auto_extension  sqlite3_api->cancel_auto_extension
+#define sqlite3_load_extension         sqlite3_api->load_extension
+#define sqlite3_malloc64               sqlite3_api->malloc64
+#define sqlite3_msize                  sqlite3_api->msize
+#define sqlite3_realloc64              sqlite3_api->realloc64
+#define sqlite3_reset_auto_extension   sqlite3_api->reset_auto_extension
+#define sqlite3_result_blob64          sqlite3_api->result_blob64
+#define sqlite3_result_text64          sqlite3_api->result_text64
+#define sqlite3_strglob                sqlite3_api->strglob
+/* Version 3.8.11 and later */
+#define sqlite3_value_dup              sqlite3_api->value_dup
+#define sqlite3_value_free             sqlite3_api->value_free
+#define sqlite3_result_zeroblob64      sqlite3_api->result_zeroblob64
+#define sqlite3_bind_zeroblob64        sqlite3_api->bind_zeroblob64
+/* Version 3.9.0 and later */
+#define sqlite3_value_subtype          sqlite3_api->value_subtype
+#define sqlite3_result_subtype         sqlite3_api->result_subtype
+/* Version 3.10.0 and later */
+#define sqlite3_status64               sqlite3_api->status64
+#define sqlite3_strlike                sqlite3_api->strlike
+#define sqlite3_db_cacheflush          sqlite3_api->db_cacheflush
+/* Version 3.12.0 and later */
+#define sqlite3_system_errno           sqlite3_api->system_errno
+/* Version 3.14.0 and later */
+#define sqlite3_trace_v2               sqlite3_api->trace_v2
+#define sqlite3_expanded_sql           sqlite3_api->expanded_sql
+/* Version 3.18.0 and later */
+#define sqlite3_set_last_insert_rowid  sqlite3_api->set_last_insert_rowid
+/* Version 3.20.0 and later */
+#define sqlite3_prepare_v3             sqlite3_api->prepare_v3
+#define sqlite3_prepare16_v3           sqlite3_api->prepare16_v3
+#define sqlite3_bind_pointer           sqlite3_api->bind_pointer
+#define sqlite3_result_pointer         sqlite3_api->result_pointer
+#define sqlite3_value_pointer          sqlite3_api->value_pointer
+/* Version 3.22.0 and later */
+#define sqlite3_vtab_nochange          sqlite3_api->vtab_nochange
+#define sqlite3_value_nochange         sqlite3_api->value_nochange
+#define sqlite3_vtab_collation         sqlite3_api->vtab_collation
+/* Version 3.24.0 and later */
+#define sqlite3_keyword_count          sqlite3_api->keyword_count
+#define sqlite3_keyword_name           sqlite3_api->keyword_name
+#define sqlite3_keyword_check          sqlite3_api->keyword_check
+#define sqlite3_str_new                sqlite3_api->str_new
+#define sqlite3_str_finish             sqlite3_api->str_finish
+#define sqlite3_str_appendf            sqlite3_api->str_appendf
+#define sqlite3_str_vappendf           sqlite3_api->str_vappendf
+#define sqlite3_str_append             sqlite3_api->str_append
+#define sqlite3_str_appendall          sqlite3_api->str_appendall
+#define sqlite3_str_appendchar         sqlite3_api->str_appendchar
+#define sqlite3_str_reset              sqlite3_api->str_reset
+#define sqlite3_str_errcode            sqlite3_api->str_errcode
+#define sqlite3_str_length             sqlite3_api->str_length
+#define sqlite3_str_value              sqlite3_api->str_value
+/* Version 3.25.0 and later */
+#define sqlite3_create_window_function sqlite3_api->create_window_function
+/* Version 3.26.0 and later */
+#define sqlite3_normalized_sql         sqlite3_api->normalized_sql
+/* Version 3.28.0 and later */
+#define sqlite3_stmt_isexplain         sqlite3_api->isexplain
+#define sqlite3_value_frombind         sqlite3_api->frombind
+/* Version 3.30.0 and later */
+#define sqlite3_drop_modules           sqlite3_api->drop_modules
+#endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */
+
+#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)
+  /* This case when the file really is being compiled as a loadable 
+  ** extension */
+# define SQLITE_EXTENSION_INIT1     const sqlite3_api_routines *sqlite3_api=0;
+# define SQLITE_EXTENSION_INIT2(v)  sqlite3_api=v;
+# define SQLITE_EXTENSION_INIT3     \
+    extern const sqlite3_api_routines *sqlite3_api;
+#else
+  /* This case when the file is being statically linked into the 
+  ** application */
+# define SQLITE_EXTENSION_INIT1     /*no-op*/
+# define SQLITE_EXTENSION_INIT2(v)  (void)v; /* unused parameter */
+# define SQLITE_EXTENSION_INIT3     /*no-op*/
+#endif
+
+#endif /* SQLITE3EXT_H */

src/claimtrie/takeoverworkarounds.h

@@ -0,0 +1,455 @@
+
+#ifndef TAKEOVERWORKAROUNDS_H
+#define TAKEOVERWORKAROUNDS_H
+
+#include <string>
+
+#include <boost/container/flat_map.hpp>
+
+const boost::container::flat_map<std::pair<int, std::string>, int> takeoverWorkarounds = {
+        {{496856, "HunterxHunterAMV"}, 496835},
+        {{542978, "namethattune1"}, 542429},
+        {{543508, "namethattune-5"}, 543306},
+        {{546780, "forecasts"}, 546624},
+        {{548730, "forecasts"}, 546780},
+        {{551540, "forecasts"}, 548730},
+        {{552380, "chicthinkingofyou"}, 550804},
+        {{560363, "takephotowithlbryteam"}, 559962},
+        {{563710, "test-img"}, 563700},
+        {{566750, "itila"}, 543261},
+        {{567082, "malabarismo-com-bolas-de-futebol-vs-chap"}, 563592},
+        {{596860, "180mphpullsthrougheurope"}, 596757},
+        {{617743, "vaccines"}, 572756},
+        {{619609, "copface-slamshandcuffedteengirlintoconcrete"}, 539940},
+        {{620392, "banker-exposes-satanic-elite"}, 597788},
+        {{624997, "direttiva-sulle-armi-ue-in-svizzera-di"}, 567908},
+        {{624997, "best-of-apex"}, 585580},
+        {{629970, "cannot-ignore-my-veins"}, 629914},
+        {{633058, "bio-waste-we-programmed-your-brain"}, 617185},
+        {{633601, "macrolauncher-overview-first-look"}, 633058},
+        {{640186, "its-up-to-you-and-i-2019"}, 639116},
+        {{640241, "tor-eas-3-20"}, 592645},
+        {{640522, "seadoxdark"}, 619531},
+        {{640617, "lbry-przewodnik-1-instalacja"}, 451186},
+        {{640623, "avxchange-2019-the-next-netflix-spotify"}, 606790},
+        {{640684, "algebra-introduction"}, 624152},
+        {{640684, "a-high-school-math-teacher-does-a"}, 600885},
+        {{640684, "another-random-life-update"}, 600884},
+        {{640684, "who-is-the-taylor-series-for"}, 600882},
+        {{640684, "tedx-talk-released"}, 612303},
+        {{640730, "e-mental"}, 615375},
+        {{641143, "amiga-1200-bespoke-virgin-cinema"}, 623542},
+        {{641161, "dreamscape-432-omega"}, 618894},
+        {{641162, "2019-topstone-carbon-force-etap-axs-bike"}, 639107},
+        {{641186, "arin-sings-big-floppy-penis-live-jazz-2"}, 638904},
+        {{641421, "edward-snowden-on-bitcoin-and-privacy"}, 522729},
+        {{641421, "what-is-libra-facebook-s-new"}, 598236},
+        {{641421, "what-are-stablecoins-counter-party-risk"}, 583508},
+        {{641421, "anthony-pomp-pompliano-discusses-crypto"}, 564416},
+        {{641421, "tim-draper-crypto-invest-summit-2019"}, 550329},
+        {{641421, "mass-adoption-and-what-will-it-take-to"}, 549781},
+        {{641421, "dragonwolftech-youtube-channel-trailer"}, 567128},
+        {{641421, "naomi-brockwell-s-weekly-crypto-recap"}, 540006},
+        {{641421, "blockchain-based-youtube-twitter"}, 580809},
+        {{641421, "andreas-antonopoulos-on-privacy-privacy"}, 533522},
+        {{641817, "mexico-submits-and-big-tech-worsens"}, 582977},
+        {{641817, "why-we-need-travel-bans"}, 581354},
+        {{641880, "censored-by-patreon-bitchute-shares"}, 482460},
+        {{641880, "crypto-wonderland"}, 485218},
+        {{642168, "1-diabolo-julio-cezar-16-cbmcp-freestyle"}, 374999},
+        {{642314, "tough-students"}, 615780},
+        {{642697, "gamercauldronep2"}, 642153},
+        {{643406, "the-most-fun-i-ve-had-in-a-long-time"}, 616506},
+        {{643893, "spitshine69-and-uk-freedom-audits"}, 616876},
+        {{644480, "my-mum-getting-attacked-a-duck"}, 567624},
+        {{644486, "the-cryptocurrency-experiment"}, 569189},
+        {{644486, "tag-you-re-it"}, 558316},
+        {{644486, "orange-county-mineral-society-rock-and"}, 397138},
+        {{644486, "sampling-with-the-gold-rush-nugget"}, 527960},
+        {{644562, "september-15-21-a-new-way-of-doing"}, 634792},
+        {{644562, "july-week-3-collective-frequency-general"}, 607942},
+        {{644562, "september-8-14-growing-up-general"}, 630977},
+        {{644562, "august-4-10-collective-frequency-general"}, 612307},
+        {{644562, "august-11-17-collective-frequency"}, 617279},
+        {{644562, "september-1-7-gentle-wake-up-call"}, 627104},
+        {{644607, "no-more-lol"}, 643497},
+        {{644607, "minion-masters-who-knew"}, 641313},
+        {{645236, "danganronpa-3-the-end-of-hope-s-peak"}, 644153},
+        {{645348, "captchabot-a-discord-bot-to-protect-your"}, 592810},
+        {{645701, "the-xero-hour-saint-greta-of-thunberg"}, 644081},
+        {{645701, "batman-v-superman-theological-notions"}, 590189},
+        {{645918, "emacs-is-great-ep-0-init-el-from-org"}, 575666},
+        {{645918, "emacs-is-great-ep-1-packages"}, 575666},
+        {{645918, "emacs-is-great-ep-40-pt-2-hebrew"}, 575668},
+        {{645923, "nasal-snuff-review-osp-batch-2"}, 575658},
+        {{645923, "why-bit-coin"}, 575658},
+        {{645929, "begin-quest"}, 598822},
+        {{645929, "filthy-foe"}, 588386},
+        {{645929, "unsanitary-snow"}, 588386},
+        {{645929, "famispam-1-music-box"}, 588386},
+        {{645929, "running-away"}, 598822},
+        {{645931, "my-beloved-chris-madsen"}, 589114},
+        {{645931, "space-is-consciousness-chris-madsen"}, 589116},
+        {{645947, "gasifier-rocket-stove-secondary-burn"}, 590595},
+        {{645949, "mouse-razer-abyssus-v2-e-mousepad"}, 591139},
+        {{645949, "pr-temporada-2018-league-of-legends"}, 591138},
+        {{645949, "windows-10-build-9901-pt-br"}, 591137},
+        {{645949, "abrindo-pacotes-do-festival-lunar-2018"}, 591139},
+        {{645949, "unboxing-camisetas-personalizadas-play-e"}, 591138},
+        {{645949, "abrindo-envelopes-do-festival-lunar-2017"}, 591138},
+        {{645951, "grub-my-grub-played-guruku-tersayang"}, 618033},
+        {{645951, "ismeeltimepiece"}, 618038},
+        {{645951, "thoughts-on-doom"}, 596485},
+        {{645951, "thoughts-on-god-of-war-about-as-deep-as"}, 596485},
+        {{645956, "linux-lite-3-6-see-what-s-new"}, 645195},
+        {{646191, "kahlil-gibran-the-prophet-part-1"}, 597637},
+        {{646551, "crypto-market-crash-should-you-sell-your"}, 442613},
+        {{646551, "live-crypto-trading-and-market-analysis"}, 442615},
+        {{646551, "5-reasons-trading-is-always-better-than"}, 500850},
+        {{646551, "digitex-futures-dump-panic-selling-or"}, 568065},
+        {{646552, "how-to-install-polarr-on-kali-linux-bynp"}, 466235},
+        {{646586, "electoral-college-kids-civics-lesson"}, 430818},
+        {{646602, "grapes-full-90-minute-watercolour"}, 537108},
+        {{646602, "meizu-mx4-the-second-ubuntu-phone"}, 537109},
+        {{646609, "how-to-set-up-the-ledger-nano-x"}, 569992},
+        {{646609, "how-to-buy-ethereum"}, 482354},
+        {{646609, "how-to-install-setup-the-exodus-multi"}, 482356},
+        {{646609, "how-to-manage-your-passwords-using"}, 531987},
+        {{646609, "cryptodad-s-live-q-a-friday-may-3rd-2019"}, 562303},
+        {{646638, "resident-evil-ada-chapter-5-final"}, 605612},
+        {{646639, "taurus-june-2019-career-love-tarot"}, 586910},
+        {{646652, "digital-bullpen-ep-5-building-a-digital"}, 589274},
+        {{646661, "sunlight"}, 591076},
+        {{646661, "grasp-lab-nasa-open-mct-series"}, 589414},
+        {{646663, "bunnula-s-creepers-tim-pool-s-beanie-a"}, 599669},
+        {{646663, "bunnula-music-hey-ya-by-outkast"}, 605685},
+        {{646663, "bunnula-tv-s-music-television-eunoia"}, 644437},
+        {{646663, "the-pussy-centipede-40-sneakers-and"}, 587265},
+        {{646663, "bunnula-reacts-ashton-titty-whitty"}, 596988},
+        {{646677, "filip-reviews-jeromes-dream-cataracts-so"}, 589751},
+        {{646691, "fascism-and-its-mobilizing-passions"}, 464342},
+        {{646692, "hsb-color-layers-action-for-adobe"}, 586533},
+        {{646692, "master-colorist-action-pack-extracting"}, 631830},
+        {{646693, "how-to-protect-your-garden-from-animals"}, 588476},
+        {{646693, "gardening-for-the-apocalypse-epic"}, 588472},
+        {{646693, "my-first-bee-hive-foundationless-natural"}, 588469},
+        {{646693, "dragon-fruit-and-passion-fruit-planting"}, 588470},
+        {{646693, "installing-my-first-foundationless"}, 588469},
+        {{646705, "first-naza-fpv"}, 590411},
+        {{646717, "first-burning-man-2019-detour-034"}, 630247},
+        {{646717, "why-bob-marley-was-an-idiot-test-driving"}, 477558},
+        {{646717, "we-are-addicted-to-gambling-ufc-207-w"}, 481398},
+        {{646717, "ghetto-swap-meet-selling-storage-lockers"}, 498291},
+        {{646738, "1-kings-chapter-7-summary-and-what-god"}, 586599},
+        {{646814, "brand-spanking-new-junior-high-school"}, 592378},
+        {{646814, "lupe-fiasco-freestyle-at-end-of-the-weak"}, 639535},
+        {{646824, "how-to-one-stroke-painting-doodles-mixed"}, 592404},
+        {{646824, "acrylic-pouring-landscape-with-a-tree"}, 592404},
+        {{646824, "how-to-make-a-diy-concrete-paste-planter"}, 595976},
+        {{646824, "how-to-make-a-rustic-sand-planter-sand"}, 592404},
+        {{646833, "3-day-festival-at-the-galilee-lake-and"}, 592842},
+        {{646833, "rainbow-circle-around-the-noon-sun-above"}, 592842},
+        {{646833, "energetic-self-control-demonstration"}, 623811},
+        {{646833, "bees-congregating"}, 592842},
+        {{646856, "formula-offroad-honefoss-sunday-track2"}, 592872},
+        {{646862, "h3video1-dc-vs-mb-1"}, 593237},
+        {{646862, "h3video1-iwasgoingto-load-up-gmod-but"}, 593237},
+        {{646883, "watch-this-game-developer-make-a-video"}, 592593},
+        {{646883, "how-to-write-secure-javascript"}, 592593},
+        {{646883, "blockchain-technology-explained-2-hour"}, 592593},
+        {{646888, "fl-studio-bits"}, 608155},
+        {{646914, "andy-s-shed-live-s03e02-the-longest"}, 592200},
+        {{646914, "gpo-telephone-776-phone-restoration"}, 592201},
+        {{646916, "toxic-studios-co-stream-pubg"}, 597126},
+        {{646916, "hyperlapse-of-prague-praha-from-inside"}, 597109},
+        {{646933, "videobits-1"}, 597378},
+        {{646933, "clouds-developing-daytime-8"}, 597378},
+        {{646933, "slechtvalk-in-watertoren-bodegraven"}, 597378},
+        {{646933, "timelapse-maansverduistering-16-juli"}, 605880},
+        {{646933, "startrails-27"}, 597378},
+        {{646933, "passing-clouds-daytime-3"}, 597378},
+        {{646940, "nerdgasm-unboxing-massive-playing-cards"}, 597421},
+        {{646946, "debunking-cops-volume-3-the-murder-of"}, 630570},
+        {{646961, "kingsong-ks16x-electric-unicycle-250km"}, 636725},
+        {{646968, "wild-mountain-goats-amazing-rock"}, 621940},
+        {{646968, "no-shelter-backcountry-camping-in"}, 621940},
+        {{646968, "can-i-live-in-this-through-winter-lets"}, 645750},
+        {{646968, "why-i-wear-a-chest-rig-backcountry-or"}, 621940},
+        {{646989, "marc-ivan-o-gorman-promo-producer-editor"}, 645656},
+        {{647045, "@moraltis"}, 646367},
+        {{647045, "moraltis-twitch-highlights-first-edit"}, 646368},
+        {{647075, "the-3-massive-tinder-convo-mistakes"}, 629464},
+        {{647075, "how-to-get-friend-zoned-via-text"}, 592298},
+        {{647075, "don-t-do-this-on-tinder"}, 624591},
+        {{647322, "world-of-tanks-7-kills"}, 609905},
+        {{647322, "the-tier-6-auto-loading-swedish-meatball"}, 591338},
+        {{647416, "hypnotic-soundscapes-garden-of-the"}, 596923},
+        {{647416, "hypnotic-soundscapes-the-cauldron-sacred"}, 596928},
+        {{647416, "schumann-resonance-to-theta-sweep"}, 596920},
+        {{647416, "conversational-indirect-hypnosis-why"}, 596913},
+        {{647493, "mimirs-brunnr"}, 590498},
+        {{648143, "live-ita-completiamo-the-evil-within-2"}, 646568},
+        {{648203, "why-we-love-people-that-hurt-us"}, 591128},
+        {{648203, "i-didn-t-like-my-baby-and-considered"}, 591128},
+        {{648220, "trade-talk-001-i-m-a-vlogger-now-fielder"}, 597303},
+        {{648220, "vise-restoration-record-no-6-vise"}, 597303},
+        {{648540, "amv-reign"}, 571863},
+        {{648540, "amv-virus"}, 571863},
+        {{648588, "audial-drift-(a-journey-into-sound)"}, 630217},
+        {{648616, "quick-zbrush-tip-transpose-master-scale"}, 463205},
+        {{648616, "how-to-create-3d-horns-maya-to-zbrush-2"}, 463205},
+        {{648815, "arduino-based-cartridge-game-handheld"}, 593252},
+        {{648815, "a-maze-update-3-new-game-modes-amazing"}, 593252},
+        {{649209, "denmark-trip"}, 591428},
+        {{649209, "stunning-4k-drone-footage"}, 591428},
+        {{649215, "how-to-create-a-channel-and-publish-a"}, 414908},
+        {{649215, "lbryclass-11-how-to-get-your-deposit"}, 632420},
+        {{649543, "spring-break-madness-at-universal"}, 599698},
+        {{649921, "navegador-brave-navegador-da-web-seguro"}, 649261},
+        {{650191, "stream-intro"}, 591301},
+        {{650946, "platelet-chan-fan-art"}, 584601},
+        {{650946, "aqua-fanart"}, 584601},
+        {{650946, "virginmedia-stores-password-in-plain"}, 619537},
+        {{650946, "running-linux-on-android-teaser"}, 604441},
+        {{650946, "hatsune-miku-ievan-polka"}, 600126},
+        {{650946, "digital-security-and-privacy-2-and-a-new"}, 600135},
+        {{650993, "my-editorial-comment-on-recent-youtube"}, 590305},
+        {{650993, "drive-7-18-2018"}, 590305},
+        {{651011, "old-world-put-on-realm-realms-gg"}, 591899},
+        {{651011, "make-your-own-soundboard-with-autohotkey"}, 591899},
+        {{651011, "ark-survival-https-discord-gg-ad26xa"}, 637680},
+        {{651011, "minecraft-featuring-seus-8-just-came-4"}, 596488},
+        {{651057, "found-footage-bikinis-at-the-beach-with"}, 593586},
+        {{651057, "found-footage-sexy-mom-a-mink-stole"}, 593586},
+        {{651067, "who-are-the-gentiles-gomer"}, 597094},
+        {{651067, "take-back-the-kingdom-ep-2-450-million"}, 597094},
+        {{651067, "mmxtac-implemented-footstep-sounds-and"}, 597094},
+        {{651067, "dynasoul-s-blender-to-unreal-animated"}, 597094},
+        {{651103, "calling-a-scammer-syntax-error"}, 612532},
+        {{651103, "quick-highlight-of-my-day"}, 647651},
+        {{651103, "calling-scammers-and-singing-christmas"}, 612531},
+        {{651109, "@livingtzm"}, 637322},
+        {{651109, "living-tzm-juuso-from-finland-september"}, 643412},
+        {{651373, "se-voc-rir-ou-sorrir-reinicie-o-v-deo"}, 649302},
+        {{651476, "what-is-pagan-online-polished-new-arpg"}, 592157},
+        {{651476, "must-have-elder-scrolls-online-addons"}, 592156},
+        {{651476, "who-should-play-albion-online"}, 592156},
+        {{651730, "person-detection-with-keras-tensorflow"}, 621276},
+        {{651730, "youtube-censorship-take-two"}, 587249},
+        {{651730, "new-red-tail-shark-and-two-silver-sharks"}, 587251},
+        {{651730, "around-auckland"}, 587250},
+        {{651730, "humanism-in-islam"}, 587250},
+        {{651730, "tigers-at-auckland-zoo"}, 587250},
+        {{651730, "gravity-demonstration"}, 587250},
+        {{651730, "copyright-question"}, 587249},
+        {{651730, "uberg33k-the-ultimate-software-developer"}, 599522},
+        {{651730, "chl-e-swarbrick-auckland-mayoral"}, 587250},
+        {{651730, "code-reviews"}, 587249},
+        {{651730, "raising-robots"}, 587251},
+        {{651730, "teaching-python"}, 587250},
+        {{651730, "kelly-tarlton-2016"}, 587250},
+        {{652172, "where-is-everything"}, 589491},
+        {{652172, "some-guy-and-his-camera"}, 617062},
+        {{652172, "practical-information-pt-1"}, 589491},
+        {{652172, "latent-vibrations"}, 589491},
+        {{652172, "maldek-compilation"}, 589491},
+        {{652444, "thank-you-etika-thank-you-desmond"}, 652121},
+        {{652611, "plants-vs-zombies-gw2-20190827183609"}, 624339},
+        {{652611, "wolfenstein-the-new-order-playthrough-6"}, 650299},
+        {{652887, "a-codeigniter-cms-open-source-download"}, 652737},
+        {{652966, "@pokesadventures"}, 632391},
+        {{653009, "flat-earth-uk-convention-is-a-bust"}, 585786},
+        {{653009, "flat-earth-reset-flat-earth-money-tree"}, 585786},
+        {{653011, "veil-of-thorns-dispirit-brutal-leech-3"}, 652475},
+        {{653069, "being-born-after-9-11"}, 632218},
+        {{653069, "8-years-on-youtube-what-it-has-done-for"}, 637130},
+        {{653069, "answering-questions-how-original"}, 521447},
+        {{653069, "talking-about-my-first-comedy-stand-up"}, 583450},
+        {{653069, "doing-push-ups-in-public"}, 650920},
+        {{653069, "vlog-extra"}, 465997},
+        {{653069, "crying-myself"}, 465997},
+        {{653069, "xbox-rejection"}, 465992},
+        {{653354, "msps-how-to-find-a-linux-job-where-no"}, 642537},
+        {{653354, "windows-is-better-than-linux-vlog-it-and"}, 646306},
+        {{653354, "luke-smith-is-wrong-about-everything"}, 507717},
+        {{653354, "advice-for-those-starting-out-in-tech"}, 612452},
+        {{653354, "treating-yourself-to-make-studying-more"}, 623561},
+        {{653354, "lpi-linux-essential-dns-tools-vlog-what"}, 559464},
+        {{653354, "is-learning-linux-worth-it-in-2019-vlog"}, 570886},
+        {{653354, "huawei-linux-and-cellphones-in-2019-vlog"}, 578501},
+        {{653354, "how-to-use-webmin-to-manage-linux"}, 511507},
+        {{653354, "latency-concurrency-and-the-best-value"}, 596857},
+        {{653354, "how-to-use-the-pomodoro-method-in-it"}, 506632},
+        {{653354, "negotiating-compensation-vlog-it-and"}, 542317},
+        {{653354, "procedural-goals-vs-outcome-goals-vlog"}, 626785},
+        {{653354, "intro-to-raid-understanding-how-raid"}, 529341},
+        {{653354, "smokeping"}, 574693},
+        {{653354, "richard-stallman-should-not-be-fired"}, 634928},
+        {{653354, "unusual-or-specialty-certifications-vlog"}, 620146},
+        {{653354, "gratitude-and-small-projects-vlog-it"}, 564900},
+        {{653354, "why-linux-on-the-smartphone-is-important"}, 649543},
+        {{653354, "opportunity-costs-vlog-it-devops-career"}, 549708},
+        {{653354, "double-giveaway-lpi-class-dates-and"}, 608129},
+        {{653354, "linux-on-the-smartphone-in-2019-librem"}, 530426},
+        {{653524, "celtic-folk-music-full-live-concert-mps"}, 589762},
+        {{653745, "aftermath-of-the-mac"}, 592768},
+        {{653745, "b-c-a-glock-17-threaded-barrel"}, 592770},
+        {{653800, "middle-earth-shadow-of-mordor-by"}, 590229},
+        {{654079, "tomand-jeremy-chirs45"}, 614296},
+        {{654096, "achamos-carteira-com-grana-olha-o-que"}, 466262},
+        {{654096, "viagem-bizarra-e-cansativa-ao-nordeste"}, 466263},
+        {{654096, "tedio-na-tailandia-limpeza-de-area"}, 466265},
+        {{654425, "schau-bung-2014-in-windischgarsten"}, 654410},
+        {{654425, "mitternachtseinlage-ball-rk"}, 654410},
+        {{654425, "zugabe-ball-rk-windischgarsten"}, 654412},
+        {{654722, "skytrain-in-korea"}, 463145},
+        {{654722, "luwak-coffee-the-shit-coffee"}, 463155},
+        {{654722, "puppet-show-in-bangkok-thailand"}, 462812},
+        {{654722, "kyaito-market-myanmar"}, 462813},
+        {{654724, "wipeout-zombies-bo3-custom-zombies-1st"}, 589569},
+        {{654724, "the-street-bo3-custom-zombies"}, 589544},
+        {{654880, "wwii-airsoft-pow"}, 586968},
+        {{654880, "dueling-geese-fight-to-the-death"}, 586968},
+        {{654880, "wwii-airsoft-torgau-raw-footage-part4"}, 586968},
+        {{655173, "april-2019-q-and-a"}, 554032},
+        {{655173, "the-meaning-and-reality-of-individual"}, 607892},
+        {{655173, "steven-pinker-progress-despite"}, 616984},
+        {{655173, "we-make-stories-out-of-totem-poles"}, 549090},
+        {{655173, "jamil-jivani-author-of-why-young-men"}, 542035},
+        {{655173, "commentaries-on-jb-peterson-rebel-wisdom"}, 528898},
+        {{655173, "auckland-clip-4-on-cain-and-abel"}, 629242},
+        {{655173, "peterson-vs-zizek-livestream-tickets"}, 545285},
+        {{655173, "auckland-clip-3-the-dawning-of-the-moral"}, 621154},
+        {{655173, "religious-belief-and-the-enlightenment"}, 606269},
+        {{655173, "auckland-lc-highlight-1-the-presumption"}, 565783},
+        {{655173, "q-a-sir-roger-scruton-dr-jordan-b"}, 544184},
+        {{655173, "cancellation-polish-national-foundation"}, 562529},
+        {{655173, "the-coddling-of-the-american-mind-haidt"}, 440185},
+        {{655173, "02-harris-weinstein-peterson-discussion"}, 430896},
+        {{655173, "jordan-peterson-threatens-everything-of"}, 519737},
+        {{655173, "on-claiming-belief-in-god-commentary"}, 581738},
+        {{655173, "how-to-make-the-world-better-really-with"}, 482317},
+        {{655173, "quillette-discussion-with-founder-editor"}, 413749},
+        {{655173, "jb-peterson-on-free-thought-and-speech"}, 462849},
+        {{655173, "marxism-zizek-peterson-official-video"}, 578453},
+        {{655173, "patreon-problem-solution-dave-rubin-dr"}, 490394},
+        {{655173, "next-week-st-louis-salt-lake-city"}, 445933},
+        {{655173, "conversations-with-john-anderson-jordan"}, 529981},
+        {{655173, "nz-australia-12-rules-tour-next-2-weeks"}, 518649},
+        {{655173, "a-call-to-rebellion-for-ontario-legal"}, 285451},
+        {{655173, "2016-personality-lecture-12"}, 578465},
+        {{655173, "on-the-vital-necessity-of-free-speech"}, 427404},
+        {{655173, "2017-01-23-social-justice-freedom-of"}, 578465},
+        {{655173, "discussion-sam-harris-the-idw-and-the"}, 423332},
+        {{655173, "march-2018-patreon-q-a"}, 413749},
+        {{655173, "take-aim-even-badly"}, 490395},
+        {{655173, "jp-f-wwbgo6a2w"}, 539940},
+        {{655173, "patreon-account-deletion"}, 503477},
+        {{655173, "canada-us-europe-tour-august-dec-2018"}, 413749},
+        {{655173, "leaders-myth-reality-general-stanley"}, 514333},
+        {{655173, "jp-ifi5kkxig3s"}, 539940},
+        {{655173, "documentary-a-glitch-in-the-matrix-david"}, 413749},
+        {{655173, "2017-08-14-patreon-q-and-a"}, 285451},
+        {{655173, "postmodernism-history-and-diagnosis"}, 285451},
+        {{655173, "23-minutes-from-maps-of-meaning-the"}, 413749},
+        {{655173, "milo-forbidden-conversation"}, 578493},
+        {{655173, "jp-wnjbasba-qw"}, 539940},
+        {{655173, "uk-12-rules-tour-october-and-november"}, 462849},
+        {{655173, "2015-maps-of-meaning-10-culture-anomaly"}, 578465},
+        {{655173, "ayaan-hirsi-ali-islam-mecca-vs-medina"}, 285452},
+        {{655173, "jp-f9393el2z1i"}, 539940},
+        {{655173, "campus-indoctrination-the-parasitization"}, 285453},
+        {{655173, "jp-owgc63khcl8"}, 539940},
+        {{655173, "the-death-and-resurrection-of-christ-a"}, 413749},
+        {{655173, "01-harris-weinstein-peterson-discussion"}, 430896},
+        {{655173, "enlightenment-now-steven-pinker-jb"}, 413749},
+        {{655173, "the-lindsay-shepherd-affair-update"}, 413749},
+        {{655173, "jp-g3fwumq5k8i"}, 539940},
+        {{655173, "jp-evvs3l-abv4"}, 539940},
+        {{655173, "former-australian-deputy-pm-john"}, 413750},
+        {{655173, "message-to-my-korean-readers-90-seconds"}, 477424},
+        {{655173, "jp--0xbomwjkgm"}, 539940},
+        {{655173, "ben-shapiro-jordan-peterson-and-a-12"}, 413749},
+        {{655173, "jp-91jwsb7zyhw"}, 539940},
+        {{655173, "deconstruction-the-lindsay-shepherd"}, 299272},
+        {{655173, "september-patreon-q-a"}, 285451},
+        {{655173, "jp-2c3m0tt5kce"}, 539940},
+        {{655173, "australia-s-john-anderson-dr-jordan-b"}, 413749},
+        {{655173, "jp-hdrlq7dpiws"}, 539940},
+        {{655173, "stephen-hicks-postmodernism-reprise"}, 578480},
+        {{655173, "october-patreon-q-a"}, 285451},
+        {{655173, "an-animated-intro-to-truth-order-and"}, 413749},
+        {{655173, "jp-bsh37-x5rny"}, 539940},
+        {{655173, "january-2019-q-a"}, 503477},
+        {{655173, "comedians-canaries-and-coalmines"}, 498586},
+        {{655173, "the-democrats-apology-and-promise"}, 465433},
+        {{655173, "jp-s4c-jodptn8"}, 539940},
+        {{655173, "2014-personality-lecture-16-extraversion"}, 578465},
+        {{655173, "dr-jordan-b-peterson-on-femsplainers"}, 490395},
+        {{655173, "higher-ed-our-cultural-inflection-point"}, 527291},
+        {{655173, "archetype-reality-friendship-and"}, 519736},
+        {{655173, "sir-roger-scruton-dr-jordan-b-peterson"}, 490395},
+        {{655173, "jp-cf2nqmqifxc"}, 539940},
+        {{655173, "penguin-uk-12-rules-for-life"}, 413749},
+        {{655173, "march-2019-q-and-a"}, 537138},
+        {{655173, "jp-ne5vbomsqjc"}, 539940},
+        {{655173, "dublin-london-harris-murray-new-usa-12"}, 413749},
+        {{655173, "12-rules-12-cities-tickets-now-available"}, 413749},
+        {{655173, "jp-j9j-bvdrgdi"}, 539940},
+        {{655173, "responsibility-conscience-and-meaning"}, 499123},
+        {{655173, "04-harris-murray-peterson-discussion"}, 436678},
+        {{655173, "jp-ayhaz9k008q"}, 539940},
+        {{655173, "with-jocko-willink-the-catastrophe-of"}, 490395},
+        {{655173, "interview-with-the-grievance-studies"}, 501296},
+        {{655173, "russell-brand-jordan-b-peterson-under"}, 413750},
+        {{655173, "goodbye-to-patreon"}, 496771},
+        {{655173, "revamped-podcast-announcement-with"}, 540943},
+        {{655173, "swedes-want-to-know"}, 285453},
+        {{655173, "auckland-clip-2-the-four-fundamental"}, 607892},
+        {{655173, "jp-dtirzqmgbdm"}, 539940},
+        {{655173, "political-correctness-a-force-for-good-a"}, 413750},
+        {{655173, "sean-plunket-full-interview-new-zealand"}, 597638},
+        {{655173, "q-a-the-meaning-and-reality-of"}, 616984},
+        {{655173, "lecture-and-q-a-with-jordan-peterson-the"}, 413749},
+        {{655173, "2017-personality-07-carl-jung-and-the"}, 578465},
+        {{655173, "nina-paley-animator-extraordinaire"}, 413750},
+        {{655173, "truth-as-the-antidote-to-suffering-with"}, 455127},
+        {{655173, "bishop-barron-word-on-fire"}, 599814},
+        {{655173, "zizek-vs-peterson-april-19"}, 527291},
+        {{655173, "revamped-podcast-with-westwood-one"}, 540943},
+        {{655173, "2016-11-19-university-of-toronto-free"}, 578465},
+        {{655173, "jp-1emrmtrj5jc"}, 539940},
+        {{655173, "who-is-joe-rogan-with-jordan-peterson"}, 585578},
+        {{655173, "who-dares-say-he-believes-in-god"}, 581738},
+        {{655252, "games-with-live2d"}, 589978},
+        {{655252, "kaenbyou-rin-live2d"}, 589978},
+        {{655374, "steam-groups-are-crazy"}, 607590},
+        {{655379, "asmr-captain-falcon-happily-beats-you-up"}, 644574},
+        {{655379, "pixel-art-series-5-link-holding-the"}, 442952},
+        {{655379, "who-can-cross-the-planck-length-the-hero"}, 610830},
+        {{655379, "ssbb-the-yoshi-grab-release-crash"}, 609747},
+        {{655379, "tas-captain-falcon-s-bizarre-adventure"}, 442958},
+        {{655379, "super-smash-bros-in-360-test"}, 442963},
+        {{655379, "what-if-luigi-was-b-u-f-f"}, 442971},
+        {{655803, "sun-time-lapse-test-7"}, 610634},
+        {{655952, "upper-build-complete"}, 591728},
+        {{656758, "cryptocurrency-awareness-adoption-the"}, 541770},
+        {{656829, "3d-printing-technologies-comparison"}, 462685},
+        {{656829, "3d-printing-for-everyone"}, 462685},
+        {{657052, "tni-punya-ilmu-kanuragan-gaya-baru"}, 657045},
+        {{657052, "papa-sunimah-nelpon-sri-utami-emon"}, 657045},
+        {{657274, "rapforlife-4-win"}, 656856},
+        {{657274, "bizzilion-proof-of-withdrawal"}, 656856},
+        {{657420, "quick-drawing-prince-tribute-colored"}, 605630},
+        {{657453, "white-boy-tom-mcdonald-facts"}, 597169},
+        {{657453, "is-it-ok-to-look-when-you-with-your-girl"}, 610508},
+        {{657584, "need-for-speed-ryzen-5-1600-gtx-1050-ti"}, 657161},
+        {{657584, "quantum-break-ryzen-5-1600-gtx-1050-ti-4"}, 657161},
+        {{657584, "nightcore-legends-never-die"}, 657161},
+        {{657706, "mtb-enduro-ferragosto-2019-sestri"}, 638904},
+        {{657706, "warface-free-for-all"}, 638908},
+        {{657782, "nick-warren-at-loveland-but-not-really"}, 444299},
+        {{658098, "le-temps-nous-glisse-entre-les-doigts"}, 600099},
+};
+
+#endif // TAKEOVERWORKAROUNDS_H

src/claimtrie/trie.h

@@ -0,0 +1,145 @@
+#ifndef CLAIMTRIE_TRIE_H
+#define CLAIMTRIE_TRIE_H
+
+#include <data.h>
+#include <sqlite.h>
+#include <txoutpoint.h>
+#include <uints.h>
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+#include <unordered_set>
+#include <utility>
+
+uint256 getValueHash(const COutPoint& outPoint, int nHeightOfLastTakeover);
+
+class CClaimTrie
+{
+    friend class CClaimTrieCacheBase;
+    friend class ClaimTrieChainFixture;
+    friend class CClaimTrieCacheHashFork;
+    friend class CClaimTrieCacheExpirationFork;
+    friend class CClaimTrieCacheNormalizationFork;
+
+public:
+    CClaimTrie() = delete;
+    CClaimTrie(CClaimTrie&&) = delete;
+    CClaimTrie(const CClaimTrie&) = delete;
+    CClaimTrie(std::size_t cacheBytes, bool fWipe, int height = 0,
+               const std::string& dataDir = ".",
+               int nNormalizedNameForkHeight = 1,
+               int nMinRemovalWorkaroundHeight = 1,
+               int nMaxRemovalWorkaroundHeight = -1,
+               int64_t nOriginalClaimExpirationTime = 1,
+               int64_t nExtendedClaimExpirationTime = 1,
+               int64_t nExtendedClaimExpirationForkHeight = 1,
+               int64_t nAllClaimsInMerkleForkHeight = 1,
+               int proportionalDelayFactor = 32);
+
+    CClaimTrie& operator=(CClaimTrie&&) = delete;
+    CClaimTrie& operator=(const CClaimTrie&) = delete;
+
+    bool empty();
+    bool SyncToDisk();
+
+protected:
+    int nNextHeight;
+    std::size_t dbCacheBytes;
+    const std::string dbFile;
+    sqlite::database db;
+    bool isNodeMigrationStart;
+    const int nProportionalDelayFactor;
+
+    const int nNormalizedNameForkHeight;
+    const int nMinRemovalWorkaroundHeight, nMaxRemovalWorkaroundHeight;
+    const int64_t nOriginalClaimExpirationTime;
+    const int64_t nExtendedClaimExpirationTime;
+    const int64_t nExtendedClaimExpirationForkHeight;
+    const int64_t nAllClaimsInMerkleForkHeight;
+
+private:
+    void doNodeTableMigration();
+};
+
+class CClaimTrieCacheBase
+{
+public:
+    explicit CClaimTrieCacheBase(CClaimTrie* base);
+    virtual ~CClaimTrieCacheBase();
+
+    bool flush();
+    bool checkConsistency();
+    uint256 getMerkleHash();
+    bool validateDb(int height, const uint256& rootHash);
+
+    std::size_t getTotalNamesInTrie() const;
+    std::size_t getTotalClaimsInTrie() const;
+    int64_t getTotalValueOfClaimsInTrie(bool fControllingOnly) const;
+
+    bool haveClaim(const std::string& name, const COutPoint& outPoint) const;
+    bool haveClaimInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const;
+
+    bool haveSupport(const std::string& name, const COutPoint& outPoint) const;
+    bool haveSupportInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const;
+
+    bool addClaim(const std::string& name, const COutPoint& outPoint, const uint160& claimId, int64_t nAmount,
+                  int nHeight, int nValidHeight = -1, int originalHeight = -1);
+
+    bool addSupport(const std::string& name, const COutPoint& outPoint, const uint160& supportedClaimId, int64_t nAmount,
+                    int nHeight, int nValidHeight = -1);
+
+    bool removeClaim(const uint160& claimId, const COutPoint& outPoint, std::string& nodeName,
+            int& validHeight, int& originalHeight);
+    bool removeSupport(const COutPoint& outPoint, std::string& nodeName, int& validHeight);
+
+    virtual bool incrementBlock();
+    virtual bool decrementBlock();
+    virtual bool finalizeDecrement();
+
+    virtual int expirationTime() const;
+
+    virtual bool getProofForName(const std::string& name, const uint160& claim, CClaimTrieProof& proof);
+    virtual bool getInfoForName(const std::string& name, CClaimValue& claim, int heightOffset = 0) const;
+
+    virtual CClaimSupportToName getClaimsForName(const std::string& name) const;
+    virtual std::string adjustNameForValidHeight(const std::string& name, int validHeight) const;
+
+    void getNamesInTrie(std::function<void(const std::string&)> callback) const;
+    bool getLastTakeoverForName(const std::string& name, uint160& claimId, int& takeoverHeight) const;
+    bool findNameForClaim(std::vector<unsigned char> claim, CClaimValue& value, std::string& name) const;
+
+    std::vector<uint160> getActivatedClaims(int height);
+    std::vector<uint160> getClaimsWithActivatedSupports(int height);
+    std::vector<uint160> getExpiredClaims(int height);
+    std::vector<uint160> getClaimsWithExpiredSupports(int height);
+
+protected:
+    int nNextHeight; // Height of the block that is being worked on, which is
+    CClaimTrie* base;
+    mutable sqlite::database db;
+    mutable std::unordered_set<std::string> removalWorkaround;
+
+    mutable sqlite::database_binder claimHashQuery, childHashQuery, claimHashQueryLimit;
+
+    virtual uint256 computeNodeHash(const std::string& name, uint256& claimsHash, int takeoverHeight);
+    supportEntryType getSupportsForName(const std::string& name) const;
+
+    virtual int getDelayForName(const std::string& name, const uint160& claimId) const;
+
+    bool deleteNodeIfPossible(const std::string& name, std::string& parent, int64_t& claims);
+    void ensureTreeStructureIsUpToDate();
+    void ensureTransacting();
+    void insertTakeovers(bool allowReplace=false);
+
+private:
+    bool transacting;
+    // for unit test
+    friend struct ClaimTrieChainFixture;
+    friend class CClaimTrieCacheTest;
+
+    bool activateAllFor(const std::string& name);
+};
+
+#endif // CLAIMTRIE_TRIE_H

src/claimtrie/txoutpoint.cpp

@@ -0,0 +1,42 @@
+
+#include <txoutpoint.h>
+
+#include <sstream>
+
+COutPoint::COutPoint(uint256 hashIn, uint32_t nIn) : hash(std::move(hashIn)), n(nIn)
+{
+}
+
+void COutPoint::SetNull()
+{
+    hash.SetNull();
+    n = uint32_t(-1);
+}
+
+bool COutPoint::IsNull() const
+{
+    return hash.IsNull() && n == uint32_t(-1);
+}
+
+bool COutPoint::operator<(const COutPoint& b) const
+{
+    int cmp = hash.Compare(b.hash);
+    return cmp < 0 || (cmp == 0 && n < b.n);
+}
+
+bool COutPoint::operator==(const COutPoint& b) const
+{
+    return hash == b.hash && n == b.n;
+}
+
+bool COutPoint::operator!=(const COutPoint& b) const
+{
+    return !(*this == b);
+}
+
+std::string COutPoint::ToString() const
+{
+    std::stringstream ss;
+    ss << "COutPoint(" << hash.ToString().substr(0, 10) << ", " << n << ')';
+    return ss.str();
+}

src/claimtrie/txoutpoint.h

@@ -0,0 +1,38 @@
+
+#ifndef CLAIMTRIE_TXOUTPUT_H
+#define CLAIMTRIE_TXOUTPUT_H
+
+#include <uints.h>
+
+#include <algorithm>
+#include <string>
+#include <type_traits>
+#include <vector>
+#include <utility>
+
+/** An outpoint - a combination of a transaction hash and an index n into its vout */
+class COutPoint
+{
+public:
+    uint256 hash;
+    uint32_t n = uint32_t(-1);
+
+    COutPoint() = default;
+    COutPoint(COutPoint&&) = default;
+    COutPoint(const COutPoint&) = default;
+    COutPoint(uint256 hashIn, uint32_t nIn);
+
+    COutPoint& operator=(COutPoint&&) = default;
+    COutPoint& operator=(const COutPoint&) = default;
+
+    void SetNull();
+    bool IsNull() const;
+
+    bool operator<(const COutPoint& b) const;
+    bool operator==(const COutPoint& b) const;
+    bool operator!=(const COutPoint& b) const;
+
+    std::string ToString() const;
+};
+
+#endif // CLAIMTRIE_TXOUTPUT_H

src/claimtrie/uints.cpp

@@ -0,0 +1,39 @@
+
+#include <uints.h>
+#include <cstring>
+
+uint160::uint160(const std::vector<uint8_t>& vec) : CBaseBlob<160>(vec)
+{
+}
+
+uint256::uint256(const std::vector<uint8_t>& vec) : CBaseBlob<256>(vec)
+{
+}
+
+uint256::uint256(int64_t value) : CBaseBlob<256>() {
+    std::memcpy(this->begin(), &value, sizeof(value)); // TODO: fix the endianness here
+}
+
+uint160 uint160S(const char* str)
+{
+    uint160 s;
+    s.SetHex(str);
+    return s;
+}
+
+uint160 uint160S(const std::string& s)
+{
+    return uint160S(s.c_str());
+}
+
+uint256 uint256S(const char* str)
+{
+    uint256 s;
+    s.SetHex(str);
+    return s;
+}
+
+uint256 uint256S(const std::string& s)
+{
+    return uint256S(s.c_str());
+}

src/claimtrie/uints.h

@@ -0,0 +1,67 @@
+
+#ifndef CLAIMTRIE_UINTS_H
+#define CLAIMTRIE_UINTS_H
+
+#include <blob.h>
+#include <string>
+
+#include <primitives/robin_hood.h>
+
+class uint160 : public CBaseBlob<160>
+{
+public:
+    uint160() = default;
+
+    explicit uint160(const std::vector<uint8_t>& vec);
+
+    uint160(uint160&&) = default;
+    uint160& operator=(uint160&&) = default;
+
+    uint160(const uint160&) = default;
+    uint160& operator=(const uint160&) = default;
+};
+
+class uint256 : public CBaseBlob<256>
+{
+public:
+    uint256() = default;
+
+    explicit uint256(const std::vector<uint8_t>& vec);
+    explicit uint256(int64_t value);
+
+    uint256(uint256&&) = default;
+    uint256& operator=(uint256&&) = default;
+
+    uint256(const uint256&) = default;
+    uint256& operator=(const uint256&) = default;
+};
+
+uint160 uint160S(const char* str);
+uint160 uint160S(const std::string& s);
+
+uint256 uint256S(const char* str);
+uint256 uint256S(const std::string& s);
+
+namespace std {
+
+    template <>
+    struct hash<uint160>
+    {
+        size_t operator()(const uint160& k) const
+        {
+            return robin_hood::hash_bytes(k.begin(), k.size());
+        }
+    };
+
+    template <>
+    struct hash<uint256>
+    {
+        size_t operator()(const uint256& k) const
+        {
+            return robin_hood::hash_bytes(k.begin(), k.size());
+        }
+    };
+}
+
+
+#endif // CLAIMTRIE_UINTS_H

src/claimtrie_serial.h

@@ -0,0 +1,103 @@
+
+#ifndef CLAIMTRIE_SERIAL_H
+#define CLAIMTRIE_SERIAL_H
+
+#include <claimtrie/data.h>
+#include <claimtrie/txoutpoint.h>
+#include <claimtrie/uints.h>
+
+template<typename Stream>
+void Serialize(Stream& s, const uint160& u)
+{
+    s.write((const char*)u.begin(), u.size());
+}
+
+template<typename Stream>
+void Serialize(Stream& s, const uint256& u)
+{
+    s.write((const char*)u.begin(), u.size());
+}
+
+template<typename Stream>
+void Unserialize(Stream& s, uint160& u)
+{
+    s.read((char*)u.begin(), u.size());
+}
+
+template<typename Stream>
+void Unserialize(Stream& s, uint256& u)
+{
+    s.read((char*)u.begin(), u.size());
+}
+
+template<typename Stream>
+void Serialize(Stream& s, const COutPoint& u)
+{
+    Serialize(s, u.hash);
+    Serialize(s, u.n);
+}
+
+template<typename Stream>
+void Unserialize(Stream& s, COutPoint& u)
+{
+    Unserialize(s, u.hash);
+    Unserialize(s, u.n);
+}
+
+template<typename Stream>
+void Serialize(Stream& s, const CClaimValue& u)
+{
+    Serialize(s, u.outPoint);
+    Serialize(s, u.claimId);
+    Serialize(s, u.nAmount);
+    Serialize(s, u.nHeight);
+    Serialize(s, u.nValidAtHeight);
+}
+
+template<typename Stream>
+void Unserialize(Stream& s, CClaimValue& u)
+{
+    Unserialize(s, u.outPoint);
+    Unserialize(s, u.claimId);
+    Unserialize(s, u.nAmount);
+    Unserialize(s, u.nHeight);
+    Unserialize(s, u.nValidAtHeight);
+}
+
+template<typename Stream>
+void Serialize(Stream& s, const CSupportValue& u)
+{
+    Serialize(s, u.outPoint);
+    Serialize(s, u.supportedClaimId);
+    Serialize(s, u.nAmount);
+    Serialize(s, u.nHeight);
+    Serialize(s, u.nValidAtHeight);
+}
+
+template<typename Stream>
+void Unserialize(Stream& s, CSupportValue& u)
+{
+    Unserialize(s, u.outPoint);
+    Unserialize(s, u.supportedClaimId);
+    Unserialize(s, u.nAmount);
+    Unserialize(s, u.nHeight);
+    Unserialize(s, u.nValidAtHeight);
+}
+
+template<typename Stream>
+void Serialize(Stream& s, const CNameOutPointHeightType& u)
+{
+    Serialize(s, u.name);
+    Serialize(s, u.outPoint);
+    Serialize(s, u.nValidHeight);
+}
+
+template<typename Stream>
+void Unserialize(Stream& s, CNameOutPointHeightType& u)
+{
+    Unserialize(s, u.name);
+    Unserialize(s, u.outPoint);
+    Unserialize(s, u.nValidHeight);
+}
+
+#endif // CLAIMTRIE_SERIAL_H

src/claimtrieforks.cpp

@@ -1,481 +0,0 @@
-
-#include <consensus/merkle.h>
-#include <chainparams.h>
-#include <claimtrie.h>
-#include <hash.h>
-
-#include <boost/locale.hpp>
-#include <boost/locale/conversion.hpp>
-#include <boost/locale/localization_backend.hpp>
-#include <boost/scope_exit.hpp>
-#include <boost/scoped_ptr.hpp>
-
-CClaimTrieCacheExpirationFork::CClaimTrieCacheExpirationFork(CClaimTrie* base)
-    : CClaimTrieCacheBase(base)
-{
-    setExpirationTime(Params().GetConsensus().GetExpirationTime(nNextHeight));
-}
-
-void CClaimTrieCacheExpirationFork::setExpirationTime(int time)
-{
-    nExpirationTime = time;
-}
-
-int CClaimTrieCacheExpirationFork::expirationTime() const
-{
-    return nExpirationTime;
-}
-
-bool CClaimTrieCacheExpirationFork::incrementBlock(insertUndoType& insertUndo, claimQueueRowType& expireUndo, insertUndoType& insertSupportUndo, supportQueueRowType& expireSupportUndo, std::vector<std::pair<std::string, int>>& takeoverHeightUndo)
-{
-    if (CClaimTrieCacheBase::incrementBlock(insertUndo, expireUndo, insertSupportUndo, expireSupportUndo, takeoverHeightUndo)) {
-        setExpirationTime(Params().GetConsensus().GetExpirationTime(nNextHeight));
-        return true;
-    }
-    return false;
-}
-
-bool CClaimTrieCacheExpirationFork::decrementBlock(insertUndoType& insertUndo, claimQueueRowType& expireUndo, insertUndoType& insertSupportUndo, supportQueueRowType& expireSupportUndo)
-{
-    if (CClaimTrieCacheBase::decrementBlock(insertUndo, expireUndo, insertSupportUndo, expireSupportUndo)) {
-        setExpirationTime(Params().GetConsensus().GetExpirationTime(nNextHeight));
-        return true;
-    }
-    return false;
-}
-
-void CClaimTrieCacheExpirationFork::initializeIncrement()
-{
-    // we could do this in the constructor, but that would not allow for multiple increments in a row (as done in unit tests)
-    if (nNextHeight != Params().GetConsensus().nExtendedClaimExpirationForkHeight)
-        return;
-
-    forkForExpirationChange(true);
-}
-
-bool CClaimTrieCacheExpirationFork::finalizeDecrement(std::vector<std::pair<std::string, int>>& takeoverHeightUndo)
-{
-    auto ret = CClaimTrieCacheBase::finalizeDecrement(takeoverHeightUndo);
-    if (ret && nNextHeight == Params().GetConsensus().nExtendedClaimExpirationForkHeight)
-       forkForExpirationChange(false);
-    return ret;
-}
-
-bool CClaimTrieCacheExpirationFork::forkForExpirationChange(bool increment)
-{
-    /*
-    If increment is True, we have forked to extend the expiration time, thus items in the expiration queue
-    will have their expiration extended by "new expiration time - original expiration time"
-
-    If increment is False, we are decremented a block to reverse the fork. Thus items in the expiration queue
-    will have their expiration extension removed.
-    */
-
-    //look through db for expiration queues, if we haven't already found it in dirty expiration queue
-    boost::scoped_ptr<CDBIterator> pcursor(base->db->NewIterator());
-    for (pcursor->SeekToFirst(); pcursor->Valid(); pcursor->Next()) {
-        std::pair<uint8_t, int> key;
-        if (!pcursor->GetKey(key))
-            continue;
-        int height = key.second;
-        if (key.first == CLAIM_EXP_QUEUE_ROW) {
-            expirationQueueRowType row;
-            if (pcursor->GetValue(row)) {
-                reactivateClaim(row, height, increment);
-            } else {
-                return error("%s(): error reading expiration queue rows from disk", __func__);
-            }
-        } else if (key.first == SUPPORT_EXP_QUEUE_ROW) {
-            expirationQueueRowType row;
-            if (pcursor->GetValue(row)) {
-                reactivateSupport(row, height, increment);
-            } else {
-                return error("%s(): error reading support expiration queue rows from disk", __func__);
-            }
-        }
-    }
-
-    return true;
-}
-
-bool CClaimTrieCacheNormalizationFork::shouldNormalize() const
-{
-    return nNextHeight > Params().GetConsensus().nNormalizedNameForkHeight;
-}
-
-std::string CClaimTrieCacheNormalizationFork::normalizeClaimName(const std::string& name, bool force) const
-{
-    if (!force && !shouldNormalize())
-        return name;
-
-    static std::locale utf8;
-    static bool initialized = false;
-    if (!initialized) {
-        static boost::locale::localization_backend_manager manager =
-            boost::locale::localization_backend_manager::global();
-        manager.select("icu");
-
-        static boost::locale::generator curLocale(manager);
-        utf8 = curLocale("en_US.UTF8");
-        initialized = true;
-    }
-
-    std::string normalized;
-    try {
-        // Check if it is a valid utf-8 string. If not, it will throw a
-        // boost::locale::conv::conversion_error exception which we catch later
-        normalized = boost::locale::conv::to_utf<char>(name, "UTF-8", boost::locale::conv::stop);
-        if (normalized.empty())
-            return name;
-
-        // these methods supposedly only use the "UTF8" portion of the locale object:
-        normalized = boost::locale::normalize(normalized, boost::locale::norm_nfd, utf8);
-        normalized = boost::locale::fold_case(normalized, utf8);
-    } catch (const boost::locale::conv::conversion_error& e) {
-        return name;
-    } catch (const std::bad_cast& e) {
-        LogPrintf("%s() is invalid or dependencies are missing: %s\n", __func__, e.what());
-        throw;
-    } catch (const std::exception& e) { // TODO: change to use ... with current_exception() in c++11
-        LogPrintf("%s() had an unexpected exception: %s\n", __func__, e.what());
-        return name;
-    }
-
-    return normalized;
-}
-
-bool CClaimTrieCacheNormalizationFork::insertClaimIntoTrie(const std::string& name, const CClaimValue& claim, bool fCheckTakeover)
-{
-    return CClaimTrieCacheExpirationFork::insertClaimIntoTrie(normalizeClaimName(name, overrideInsertNormalization), claim, fCheckTakeover);
-}
-
-bool CClaimTrieCacheNormalizationFork::removeClaimFromTrie(const std::string& name, const COutPoint& outPoint, CClaimValue& claim, bool fCheckTakeover)
-{
-    return CClaimTrieCacheExpirationFork::removeClaimFromTrie(normalizeClaimName(name, overrideRemoveNormalization), outPoint, claim, fCheckTakeover);
-}
-
-bool CClaimTrieCacheNormalizationFork::insertSupportIntoMap(const std::string& name, const CSupportValue& support, bool fCheckTakeover)
-{
-    return CClaimTrieCacheExpirationFork::insertSupportIntoMap(normalizeClaimName(name, overrideInsertNormalization), support, fCheckTakeover);
-}
-
-bool CClaimTrieCacheNormalizationFork::removeSupportFromMap(const std::string& name, const COutPoint& outPoint, CSupportValue& support, bool fCheckTakeover)
-{
-    return CClaimTrieCacheExpirationFork::removeSupportFromMap(normalizeClaimName(name, overrideRemoveNormalization), outPoint, support, fCheckTakeover);
-}
-
-bool CClaimTrieCacheNormalizationFork::normalizeAllNamesInTrieIfNecessary(insertUndoType& insertUndo, claimQueueRowType& removeUndo, insertUndoType& insertSupportUndo, supportQueueRowType& expireSupportUndo, std::vector<std::pair<std::string, int>>& takeoverHeightUndo)
-{
-    if (nNextHeight != Params().GetConsensus().nNormalizedNameForkHeight)
-        return false;
-
-    // run the one-time upgrade of all names that need to change
-    // it modifies the (cache) trie as it goes, so we need to grab everything to be modified first
-
-    for (auto it = base->cbegin(); it != base->cend(); ++it) {
-        const std::string normalized = normalizeClaimName(it.key(), true);
-        if (normalized == it.key())
-            continue;
-
-        auto& name = it.key();
-        auto supports = getSupportsForName(name);
-        for (auto support : supports) {
-            // if it's already going to expire just skip it
-            if (support.nHeight + expirationTime() <= nNextHeight)
-                continue;
-
-            assert(removeSupportFromMap(name, support.outPoint, support, false));
-            expireSupportUndo.emplace_back(name, support);
-            assert(insertSupportIntoMap(normalized, support, false));
-            insertSupportUndo.emplace_back(name, support.outPoint, -1);
-        }
-
-        namesToCheckForTakeover.insert(normalized);
-
-        auto cached = cacheData(name, false);
-        if (!cached || cached->empty())
-            continue;
-
-        auto claimsCopy = cached->claims;
-        auto takeoverHeightCopy = cached->nHeightOfLastTakeover;
-        for (auto claim : claimsCopy) {
-            if (claim.nHeight + expirationTime() <= nNextHeight)
-                continue;
-
-            assert(removeClaimFromTrie(name, claim.outPoint, claim, false));
-            removeUndo.emplace_back(name, claim);
-            assert(insertClaimIntoTrie(normalized, claim, true));
-            insertUndo.emplace_back(name, claim.outPoint, -1);
-        }
-
-        takeoverHeightUndo.emplace_back(name, takeoverHeightCopy);
-    }
-    return true;
-}
-
-bool CClaimTrieCacheNormalizationFork::incrementBlock(insertUndoType& insertUndo, claimQueueRowType& expireUndo, insertUndoType& insertSupportUndo, supportQueueRowType& expireSupportUndo, std::vector<std::pair<std::string, int>>& takeoverHeightUndo)
-{
-    overrideInsertNormalization = normalizeAllNamesInTrieIfNecessary(insertUndo, expireUndo, insertSupportUndo, expireSupportUndo, takeoverHeightUndo);
-    BOOST_SCOPE_EXIT(&overrideInsertNormalization) { overrideInsertNormalization = false; }
-    BOOST_SCOPE_EXIT_END
-    return CClaimTrieCacheExpirationFork::incrementBlock(insertUndo, expireUndo, insertSupportUndo, expireSupportUndo, takeoverHeightUndo);
-}
-
-bool CClaimTrieCacheNormalizationFork::decrementBlock(insertUndoType& insertUndo, claimQueueRowType& expireUndo, insertUndoType& insertSupportUndo, supportQueueRowType& expireSupportUndo)
-{
-    overrideRemoveNormalization = shouldNormalize();
-    BOOST_SCOPE_EXIT(&overrideRemoveNormalization) { overrideRemoveNormalization = false; }
-    BOOST_SCOPE_EXIT_END
-    return CClaimTrieCacheExpirationFork::decrementBlock(insertUndo, expireUndo, insertSupportUndo, expireSupportUndo);
-}
-
-bool CClaimTrieCacheNormalizationFork::getProofForName(const std::string& name, CClaimTrieProof& proof)
-{
-    return CClaimTrieCacheExpirationFork::getProofForName(normalizeClaimName(name), proof);
-}
-
-bool CClaimTrieCacheNormalizationFork::getInfoForName(const std::string& name, CClaimValue& claim) const
-{
-    return CClaimTrieCacheExpirationFork::getInfoForName(normalizeClaimName(name), claim);
-}
-
-CClaimSupportToName CClaimTrieCacheNormalizationFork::getClaimsForName(const std::string& name) const
-{
-    return CClaimTrieCacheExpirationFork::getClaimsForName(normalizeClaimName(name));
-}
-
-int CClaimTrieCacheNormalizationFork::getDelayForName(const std::string& name, const uint160& claimId) const
-{
-    return CClaimTrieCacheExpirationFork::getDelayForName(normalizeClaimName(name), claimId);
-}
-
-std::string CClaimTrieCacheNormalizationFork::adjustNameForValidHeight(const std::string& name, int validHeight) const
-{
-    return normalizeClaimName(name, validHeight > Params().GetConsensus().nNormalizedNameForkHeight);
-}
-
-CClaimTrieCacheHashFork::CClaimTrieCacheHashFork(CClaimTrie* base) : CClaimTrieCacheNormalizationFork(base)
-{
-}
-
-static const uint256 leafHash = uint256S("0000000000000000000000000000000000000000000000000000000000000002");
-static const uint256 emptyHash = uint256S("0000000000000000000000000000000000000000000000000000000000000003");
-
-std::vector<uint256> getClaimHashes(const CClaimTrieData& data)
-{
-    std::vector<uint256> hashes;
-    for (auto& claim : data.claims)
-        hashes.push_back(getValueHash(claim.outPoint, data.nHeightOfLastTakeover));
-    return hashes;
-}
-
-template <typename T>
-using iCbType = std::function<uint256(T&)>;
-
-template <typename TIterator>
-uint256 recursiveBinaryTreeHash(TIterator& it, const iCbType<TIterator>& process)
-{
-    std::vector<uint256> childHashes;
-    for (auto& child : it.children())
-        childHashes.emplace_back(process(child));
-
-    std::vector<uint256> claimHashes;
-    if (!it->empty())
-        claimHashes = getClaimHashes(it.data());
-    else if (!it.hasChildren())
-        return {};
-
-    auto left = childHashes.empty() ? leafHash : ComputeMerkleRoot(childHashes);
-    auto right = claimHashes.empty() ? emptyHash : ComputeMerkleRoot(claimHashes);
-
-    return Hash(left.begin(), left.end(), right.begin(), right.end());
-}
-
-uint256 CClaimTrieCacheHashFork::recursiveComputeMerkleHash(CClaimTrie::iterator& it)
-{
-    if (nNextHeight < Params().GetConsensus().nAllClaimsInMerkleForkHeight)
-        return CClaimTrieCacheNormalizationFork::recursiveComputeMerkleHash(it);
-
-    using iterator = CClaimTrie::iterator;
-    iCbType<iterator> process = [&process](iterator& it) -> uint256 {
-        if (it->hash.IsNull())
-            it->hash = recursiveBinaryTreeHash(it, process);
-        assert(!it->hash.IsNull());
-        return it->hash;
-    };
-    return process(it);
-}
-
-bool CClaimTrieCacheHashFork::recursiveCheckConsistency(CClaimTrie::const_iterator& it, std::string& failed) const
-{
-    if (nNextHeight < Params().GetConsensus().nAllClaimsInMerkleForkHeight)
-        return CClaimTrieCacheNormalizationFork::recursiveCheckConsistency(it, failed);
-
-    struct CRecursiveBreak {};
-    using iterator = CClaimTrie::const_iterator;
-    iCbType<iterator> process = [&failed, &process](iterator& it) -> uint256 {
-        if (it->hash.IsNull() || it->hash != recursiveBinaryTreeHash(it, process)) {
-            failed = it.key();
-            throw CRecursiveBreak();
-        }
-        return it->hash;
-    };
-
-    try {
-        process(it);
-    } catch (const CRecursiveBreak&) {
-        return false;
-    }
-    return true;
-}
-
-std::vector<uint256> ComputeMerklePath(const std::vector<uint256>& hashes, uint32_t idx)
-{
-    uint32_t count = 0;
-    int matchlevel = -1;
-    bool matchh = false;
-    uint256 inner[32], h;
-    const uint32_t one = 1;
-    std::vector<uint256> res;
-
-    const auto iterateInner = [&](int& level) {
-        for (; !(count & (one << level)); level++) {
-            const auto& ihash = inner[level];
-            if (matchh) {
-                res.push_back(ihash);
-            } else if (matchlevel == level) {
-                res.push_back(h);
-                matchh = true;
-            }
-            h = Hash(ihash.begin(), ihash.end(), h.begin(), h.end());
-        }
-    };
-
-    while (count < hashes.size()) {
-        h = hashes[count];
-        matchh = count == idx;
-        count++;
-        int level = 0;
-        iterateInner(level);
-        // Store the resulting hash at inner position level.
-        inner[level] = h;
-        if (matchh)
-            matchlevel = level;
-    }
-
-    int level = 0;
-    while (!(count & (one << level)))
-        level++;
-
-    h = inner[level];
-    matchh = matchlevel == level;
-
-    while (count != (one << level)) {
-        // If we reach this point, h is an inner value that is not the top.
-        if (matchh)
-            res.push_back(h);
-
-        h = Hash(h.begin(), h.end(), h.begin(), h.end());
-        // Increment count to the value it would have if two entries at this
-        count += (one << level);
-        level++;
-        iterateInner(level);
-    }
-    return res;
-}
-
-bool CClaimTrieCacheHashFork::getProofForName(const std::string& name, CClaimTrieProof& proof)
-{
-    return getProofForName(name, proof, nullptr);
-}
-
-bool CClaimTrieCacheHashFork::getProofForName(const std::string& name, CClaimTrieProof& proof, const std::function<bool(const CClaimValue&)>& comp)
-{
-    if (nNextHeight < Params().GetConsensus().nAllClaimsInMerkleForkHeight)
-        return CClaimTrieCacheNormalizationFork::getProofForName(name, proof);
-
-    auto fillPairs = [&proof](const std::vector<uint256>& hashes, uint32_t idx) {
-        auto partials = ComputeMerklePath(hashes, idx);
-        for (int i = partials.size() - 1; i >= 0; --i)
-            proof.pairs.emplace_back((idx >> i) & 1, partials[i]);
-    };
-
-    // cache the parent nodes
-    cacheData(name, false);
-    getMerkleHash();
-    proof = CClaimTrieProof();
-    for (auto& it : static_cast<const CClaimTrie&>(nodesToAddOrUpdate).nodes(name)) {
-        std::vector<uint256> childHashes;
-        uint32_t nextCurrentIdx = 0;
-        for (auto& child : it.children()) {
-            if (name.find(child.key()) == 0)
-                nextCurrentIdx = uint32_t(childHashes.size());
-            childHashes.push_back(child->hash);
-        }
-
-        std::vector<uint256> claimHashes;
-        if (!it->empty())
-            claimHashes = getClaimHashes(it.data());
-
-        // I am on a node; I need a hash(children, claims)
-        // if I am the last node on the list, it will be hash(children, x)
-        // else it will be hash(x, claims)
-        if (it.key() == name) {
-            uint32_t nClaimIndex = 0;
-            auto& claims = it->claims;
-            auto itClaim = !comp ? claims.begin() : std::find_if(claims.begin(), claims.end(), comp);
-            if (itClaim != claims.end()) {
-                proof.hasValue = true;
-                proof.outPoint = itClaim->outPoint;
-                proof.nHeightOfLastTakeover = it->nHeightOfLastTakeover;
-                nClaimIndex = std::distance(claims.begin(), itClaim);
-            }
-            auto hash = childHashes.empty() ? leafHash : ComputeMerkleRoot(childHashes);
-            proof.pairs.emplace_back(true, hash);
-            if (!claimHashes.empty())
-                fillPairs(claimHashes, nClaimIndex);
-        } else {
-            auto hash = claimHashes.empty() ? emptyHash : ComputeMerkleRoot(claimHashes);
-            proof.pairs.emplace_back(false, hash);
-            if (!childHashes.empty())
-                fillPairs(childHashes, nextCurrentIdx);
-        }
-    }
-    std::reverse(proof.pairs.begin(), proof.pairs.end());
-    return true;
-}
-
-void CClaimTrieCacheHashFork::copyAllBaseToCache()
-{
-    for (auto it = base->cbegin(); it != base->cend(); ++it)
-        if (nodesAlreadyCached.insert(it.key()).second)
-            nodesToAddOrUpdate.insert(it.key(), it.data());
-
-    for (auto it = nodesToAddOrUpdate.begin(); it != nodesToAddOrUpdate.end(); ++it)
-        it->hash.SetNull();
-}
-
-void CClaimTrieCacheHashFork::initializeIncrement()
-{
-    CClaimTrieCacheNormalizationFork::initializeIncrement();
-    // we could do this in the constructor, but that would not allow for multiple increments in a row (as done in unit tests)
-    if (nNextHeight != Params().GetConsensus().nAllClaimsInMerkleForkHeight - 1)
-        return;
-
-    // if we are forking, we load the entire base trie into the cache trie
-    // we reset its hash computation so it can be recomputed completely
-    copyAllBaseToCache();
-}
-
-bool CClaimTrieCacheHashFork::finalizeDecrement(std::vector<std::pair<std::string, int>>& takeoverHeightUndo)
-{
-    auto ret = CClaimTrieCacheNormalizationFork::finalizeDecrement(takeoverHeightUndo);
-    if (ret && nNextHeight == Params().GetConsensus().nAllClaimsInMerkleForkHeight - 1)
-        copyAllBaseToCache();
-    return ret;
-}
-
-bool CClaimTrieCacheHashFork::allowSupportMetadata() const
-{
-    return nNextHeight >= Params().GetConsensus().nAllClaimsInMerkleForkHeight;
-}

src/coins.cpp

@@ -10,14 +10,9 @@
 bool CCoinsView::GetCoin(const COutPoint &outpoint, Coin &coin) const { return false; }
 uint256 CCoinsView::GetBestBlock() const { return uint256(); }
 std::vector<uint256> CCoinsView::GetHeadBlocks() const { return std::vector<uint256>(); }
-bool CCoinsView::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return false; }
+bool CCoinsView::BatchWrite(const CCoinsMap &mapCoins, const uint256 &hashBlock, bool sync) { return false; }
 CCoinsViewCursor *CCoinsView::Cursor() const { return nullptr; }
-
+bool CCoinsView::HaveCoin(const COutPoint &outpoint) const { return false; }
-bool CCoinsView::HaveCoin(const COutPoint &outpoint) const
-{
-    Coin coin;
-    return GetCoin(outpoint, coin);
-}
 
 CCoinsViewBacked::CCoinsViewBacked(CCoinsView *viewIn) : base(viewIn) { }
 bool CCoinsViewBacked::GetCoin(const COutPoint &outpoint, Coin &coin) const { return base->GetCoin(outpoint, coin); }
@@ -25,7 +20,7 @@ bool CCoinsViewBacked::HaveCoin(const COutPoint &outpoint) const { return base->
 uint256 CCoinsViewBacked::GetBestBlock() const { return base->GetBestBlock(); }
 std::vector<uint256> CCoinsViewBacked::GetHeadBlocks() const { return base->GetHeadBlocks(); }
 void CCoinsViewBacked::SetBackend(CCoinsView &viewIn) { base = &viewIn; }
-bool CCoinsViewBacked::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return base->BatchWrite(mapCoins, hashBlock); }
+bool CCoinsViewBacked::BatchWrite(const CCoinsMap &mapCoins, const uint256 &hashBlock, bool sync) { return base->BatchWrite(mapCoins, hashBlock, sync); }
 CCoinsViewCursor *CCoinsViewBacked::Cursor() const { return base->Cursor(); }
 size_t CCoinsViewBacked::EstimateSize() const { return base->EstimateSize(); }
 
@@ -127,11 +122,6 @@ bool CCoinsViewCache::HaveCoin(const COutPoint &outpoint) const {
     return (it != cacheCoins.end() && !it->second.coin.IsSpent());
 }
 
-bool CCoinsViewCache::HaveCoinInCache(const COutPoint &outpoint) const {
-    CCoinsMap::const_iterator it = cacheCoins.find(outpoint);
-    return (it != cacheCoins.end() && !it->second.coin.IsSpent());
-}
-
 uint256 CCoinsViewCache::GetBestBlock() const {
     if (hashBlock.IsNull())
         hashBlock = base->GetBestBlock();
@@ -142,8 +132,8 @@ void CCoinsViewCache::SetBestBlock(const uint256 &hashBlockIn) {
     hashBlock = hashBlockIn;
 }
 
-bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn) {
+bool CCoinsViewCache::BatchWrite(const CCoinsMap &mapCoins, const uint256 &hashBlockIn, bool sync) {
-    for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end(); it = mapCoins.erase(it)) {
+    for (auto it = mapCoins.begin(); it != mapCoins.end(); ++it) {
         // Ignore non-dirty entries (optimization).
         if (!(it->second.flags & CCoinsCacheEntry::DIRTY)) {
             continue;
@@ -200,8 +190,8 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn
     return true;
 }
 
-bool CCoinsViewCache::Flush() {
+bool CCoinsViewCache::Flush(bool sync) {
-    bool fOk = base->BatchWrite(cacheCoins, hashBlock);
+    bool fOk = base->BatchWrite(cacheCoins, hashBlock, sync);
     cacheCoins.clear();
     cachedCoinsUsage = 0;
     return fOk;

src/coins.h

@@ -124,21 +124,19 @@ typedef std::unordered_map<COutPoint, CCoinsCacheEntry, SaltedOutpointHasher> CC
 /** Cursor for iterating over CoinsView state */
 class CCoinsViewCursor
 {
+    uint256 hashBlock;
 public:
     CCoinsViewCursor(const uint256 &hashBlockIn): hashBlock(hashBlockIn) {}
-    virtual ~CCoinsViewCursor() {}
+    virtual ~CCoinsViewCursor() noexcept {}
 
     virtual bool GetKey(COutPoint &key) const = 0;
     virtual bool GetValue(Coin &coin) const = 0;
-    virtual unsigned int GetValueSize() const = 0;
 
     virtual bool Valid() const = 0;
     virtual void Next() = 0;
 
     //! Get best block at the time this cursor was created
     const uint256 &GetBestBlock() const { return hashBlock; }
-private:
-    uint256 hashBlock;
 };
 
 /** Abstract view on the open txout dataset. */
@@ -165,7 +163,7 @@ public:
 
     //! Do a bulk modification (multiple Coin changes + BestBlock change).
     //! The passed mapCoins can be modified.
-    virtual bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock);
+    virtual bool BatchWrite(const CCoinsMap &mapCoins, const uint256 &hashBlock, bool sync);
 
     //! Get a cursor to iterate over the whole state
     virtual CCoinsViewCursor *Cursor() const;
@@ -191,7 +189,7 @@ public:
     uint256 GetBestBlock() const override;
     std::vector<uint256> GetHeadBlocks() const override;
     void SetBackend(CCoinsView &viewIn);
-    bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override;
+    bool BatchWrite(const CCoinsMap &mapCoins, const uint256 &hashBlock, bool sync) override;
     CCoinsViewCursor *Cursor() const override;
     size_t EstimateSize() const override;
 };
@@ -224,18 +222,11 @@ public:
     bool HaveCoin(const COutPoint &outpoint) const override;
     uint256 GetBestBlock() const override;
     void SetBestBlock(const uint256 &hashBlock);
-    bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override;
+    bool BatchWrite(const CCoinsMap &mapCoins, const uint256 &hashBlock, bool sync) override;
     CCoinsViewCursor* Cursor() const override {
         throw std::logic_error("CCoinsViewCache cursor iteration not supported.");
     }
 
-    /**
-     * Check if we have the given utxo already loaded in this cache.
-     * The semantics are the same as HaveCoin(), but no calls to
-     * the backing CCoinsView are made.
-     */
-    bool HaveCoinInCache(const COutPoint &outpoint) const;
-
     /**
      * Return a reference to Coin in the cache, or a pruned one if not found. This is
      * more efficient than GetCoin.
@@ -266,7 +257,7 @@ public:
      * Failure to call this method before destruction will cause the changes to be forgotten.
      * If false is returned, the state of this cache (and its backing view) will be undefined.
      */
-    bool Flush();
+    bool Flush(bool sync = false);
 
     /**
      * Removes the UTXO with the given outpoint from the cache, if it is

src/compat.h

@@ -102,8 +102,15 @@ typedef void* sockopt_arg_type;
 typedef char* sockopt_arg_type;
 #endif
 
+// Note these both should work with the current usage of poll, but best to be safe
+// WIN32 poll is broken https://daniel.haxx.se/blog/2012/10/10/wsapoll-is-broken/
+// __APPLE__ poll is broke https://github.com/bitcoin/bitcoin/pull/14336#issuecomment-437384408
+#if defined(__linux__)
+#define USE_POLL
+#endif
+
 bool static inline IsSelectableSocket(const SOCKET& s) {
-#ifdef WIN32
+#if defined(USE_POLL) || defined(WIN32)
     return true;
 #else
     return (s < FD_SETSIZE);

src/compat/byteswap.h

@@ -30,26 +30,40 @@
 #endif // !defined(bswap_16)
 
 #else
+
+#include <boost/endian/detail/intrinsic.hpp>
+
 // Non-Mac OS X / non-Darwin
 
 #if HAVE_DECL_BSWAP_16 == 0
 inline uint16_t bswap_16(uint16_t x)
 {
+#ifdef BOOST_ENDIAN_INTRINSIC_BYTE_SWAP_2
+    return BOOST_ENDIAN_INTRINSIC_BYTE_SWAP_2(x);
+#else
     return (x >> 8) | (x << 8);
+#endif
 }
 #endif // HAVE_DECL_BSWAP16 == 0
 
 #if HAVE_DECL_BSWAP_32 == 0
 inline uint32_t bswap_32(uint32_t x)
 {
+#ifdef BOOST_ENDIAN_INTRINSIC_BYTE_SWAP_4
+    return BOOST_ENDIAN_INTRINSIC_BYTE_SWAP_4(x);
+#else
     return (((x & 0xff000000U) >> 24) | ((x & 0x00ff0000U) >>  8) |
             ((x & 0x0000ff00U) <<  8) | ((x & 0x000000ffU) << 24));
+#endif
 }
 #endif // HAVE_DECL_BSWAP32 == 0
 
 #if HAVE_DECL_BSWAP_64 == 0
 inline uint64_t bswap_64(uint64_t x)
 {
+#ifdef BOOST_ENDIAN_INTRINSIC_BYTE_SWAP_8
+    return BOOST_ENDIAN_INTRINSIC_BYTE_SWAP_8(x);
+#else
      return (((x & 0xff00000000000000ull) >> 56)
           | ((x & 0x00ff000000000000ull) >> 40)
           | ((x & 0x0000ff0000000000ull) >> 24)
@@ -58,6 +72,7 @@ inline uint64_t bswap_64(uint64_t x)
           | ((x & 0x0000000000ff0000ull) << 24)
           | ((x & 0x000000000000ff00ull) << 40)
           | ((x & 0x00000000000000ffull) << 56));
+#endif
 }
 #endif // HAVE_DECL_BSWAP64 == 0
 

src/consensus/merkle.cpp

@@ -6,6 +6,9 @@
 #include <hash.h>
 #include <utilstrencodings.h>
 
+#include <functional>
+#include <vector>
+
 /*     WARNING! If you're reading this because you're learning about crypto
        and/or designing a new system that will use merkle trees, keep in mind
        that the following merkle tree algorithm has a serious flaw related to
@@ -42,6 +45,7 @@
        root.
 */
 
+extern std::function<void(std::vector<uint256>&)> sha256n_way;
 
 uint256 ComputeMerkleRoot(std::vector<uint256> hashes, bool* mutated) {
     bool mutation = false;
@@ -54,7 +58,7 @@ uint256 ComputeMerkleRoot(std::vector<uint256> hashes, bool* mutated) {
         if (hashes.size() & 1) {
             hashes.push_back(hashes.back());
         }
-        SHA256D64(hashes[0].begin(), hashes[0].begin(), hashes.size() / 2);
+        sha256n_way(hashes);
         hashes.resize(hashes.size() / 2);
     }
     if (mutated) *mutated = mutation;

src/consensus/params.h

@@ -78,8 +78,8 @@ struct Params {
     int nAllowMinDiffMaxHeight;
     int nNormalizedNameForkHeight;
 
-    int nMinTakeoverWorkaroundHeight;
+    int nMinRemovalWorkaroundHeight;
-    int nMaxTakeoverWorkaroundHeight;
+    int nMaxRemovalWorkaroundHeight;
 
     int nWitnessForkHeight;
 

src/dbwrapper.cpp

@@ -1,267 +0,0 @@
-// Copyright (c) 2012-2018 The Bitcoin Core developers
-// Distributed under the MIT software license, see the accompanying
-// file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-#include <dbwrapper.h>
-
-#include <memory>
-#include <random.h>
-
-#include <leveldb/cache.h>
-#include <leveldb/env.h>
-#include <leveldb/filter_policy.h>
-#include <memenv.h>
-#include <stdint.h>
-#include <algorithm>
-
-class CBitcoinLevelDBLogger : public leveldb::Logger {
-public:
-    // This code is adapted from posix_logger.h, which is why it is using vsprintf.
-    // Please do not do this in normal code
-    void Logv(const char * format, va_list ap) override {
-            if (!LogAcceptCategory(BCLog::LEVELDB)) {
-                return;
-            }
-            char buffer[500];
-            for (int iter = 0; iter < 2; iter++) {
-                char* base;
-                int bufsize;
-                if (iter == 0) {
-                    bufsize = sizeof(buffer);
-                    base = buffer;
-                }
-                else {
-                    bufsize = 30000;
-                    base = new char[bufsize];
-                }
-                char* p = base;
-                char* limit = base + bufsize;
-
-                // Print the message
-                if (p < limit) {
-                    va_list backup_ap;
-                    va_copy(backup_ap, ap);
-                    // Do not use vsnprintf elsewhere in bitcoin source code, see above.
-                    p += vsnprintf(p, limit - p, format, backup_ap);
-                    va_end(backup_ap);
-                }
-
-                // Truncate to available space if necessary
-                if (p >= limit) {
-                    if (iter == 0) {
-                        continue;       // Try again with larger buffer
-                    }
-                    else {
-                        p = limit - 1;
-                    }
-                }
-
-                // Add newline if necessary
-                if (p == base || p[-1] != '\n') {
-                    *p++ = '\n';
-                }
-
-                assert(p <= limit);
-                base[std::min(bufsize - 1, (int)(p - base))] = '\0';
-                LogPrintf("leveldb: %s", base);  /* Continued */
-                if (base != buffer) {
-                    delete[] base;
-                }
-                break;
-            }
-    }
-};
-
-static void SetMaxOpenFiles(leveldb::Options *options) {
-    // On most platforms the default setting of max_open_files (which is 1000)
-    // is optimal. On Windows using a large file count is OK because the handles
-    // do not interfere with select() loops. On 64-bit Unix hosts this value is
-    // also OK, because up to that amount LevelDB will use an mmap
-    // implementation that does not use extra file descriptors (the fds are
-    // closed after being mmaped).
-    //
-    // Increasing the value beyond the nmap count is dangerous because LevelDB will
-    // fall back to a non-mmap implementation when the file count is too large (thus contending select()).
-    // On 32-bit Unix host we should decrease the value because the handles use
-    // up real fds, and we want to avoid fd exhaustion issues.
-    //
-    // See PR #12495 for further discussion.
-
-    int default_open_files = 400;
-#ifndef WIN32
-    if (sizeof(void*) < 8) {
-        options->max_open_files = 64;
-    }
-#endif
-    LogPrint(BCLog::LEVELDB, "LevelDB using max_open_files=%d (default=%d)\n",
-             options->max_open_files, default_open_files);
-}
-
-static leveldb::Options GetOptions(size_t nCacheSize)
-{
-    leveldb::Options options;
-    auto write_cache = std::min(nCacheSize / 4, size_t(32 * 1024 * 1024)); // cap write_cache
-    options.block_cache = leveldb::NewLRUCache(nCacheSize - write_cache * 2);
-    options.write_buffer_size = write_cache; // up to two write buffers may be held in memory simultaneously
-    options.filter_policy = leveldb::NewBloomFilterPolicy(12);
-    options.compression = leveldb::kNoCompression;
-    options.info_log = new CBitcoinLevelDBLogger();
-    if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) {
-        // LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error
-        // on corruption in later versions.
-        options.paranoid_checks = true;
-    }
-    SetMaxOpenFiles(&options);
-    return options;
-}
-
-CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate)
-        : m_name(fs::basename(path)), ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION)
-{
-    penv = nullptr;
-    readoptions.verify_checksums = true;
-    iteroptions.verify_checksums = true;
-    iteroptions.fill_cache = false;
-    syncoptions.sync = true;
-    options = GetOptions(nCacheSize);
-    options.create_if_missing = true;
-    if (fMemory) {
-        penv = leveldb::NewMemEnv(leveldb::Env::Default());
-        options.env = penv;
-    } else {
-        if (fWipe) {
-            LogPrintf("Wiping LevelDB in %s\n", path.string());
-            leveldb::Status result = leveldb::DestroyDB(path.string(), options);
-            dbwrapper_private::HandleError(result);
-        }
-        TryCreateDirectories(path);
-        LogPrintf("Opening LevelDB in %s\n", path.string());
-    }
-    leveldb::Status status = leveldb::DB::Open(options, path.string(), &pdb);
-    dbwrapper_private::HandleError(status);
-    LogPrintf("Opened LevelDB successfully\n");
-
-    if (gArgs.GetBoolArg("-forcecompactdb", false)) {
-        LogPrintf("Starting database compaction of %s\n", path.string());
-        pdb->CompactRange(nullptr, nullptr);
-        LogPrintf("Finished database compaction of %s\n", path.string());
-    }
-
-    // The base-case obfuscation key, which is a noop.
-    obfuscate_key = std::vector<unsigned char>(OBFUSCATE_KEY_NUM_BYTES, '\000');
-
-    bool key_exists = Read(OBFUSCATE_KEY_KEY, obfuscate_key);
-
-    if (!key_exists && obfuscate && IsEmpty()) {
-        // Initialize non-degenerate obfuscation if it won't upset
-        // existing, non-obfuscated data.
-        std::vector<unsigned char> new_key = CreateObfuscateKey();
-
-        // Write `new_key` so we don't obfuscate the key with itself
-        Write(OBFUSCATE_KEY_KEY, new_key);
-        obfuscate_key = new_key;
-
-        LogPrintf("Wrote new obfuscate key for %s: %s\n", path.string(), HexStr(obfuscate_key));
-    }
-
-    LogPrintf("Using obfuscation key for %s: %s\n", path.string(), HexStr(obfuscate_key));
-}
-
-CDBWrapper::~CDBWrapper()
-{
-    delete pdb;
-    pdb = nullptr;
-    delete options.filter_policy;
-    options.filter_policy = nullptr;
-    delete options.info_log;
-    options.info_log = nullptr;
-    delete options.block_cache;
-    options.block_cache = nullptr;
-    delete penv;
-    options.env = nullptr;
-}
-
-bool CDBWrapper::Sync() {
-    CDBBatch batch(*this);
-    return WriteBatch(batch, true);
-}
-
-bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync)
-{
-    if (!pdb)
-        return false;
-
-    const bool log_memory = LogAcceptCategory(BCLog::LEVELDB);
-    double mem_before = 0;
-    if (log_memory) {
-        mem_before = DynamicMemoryUsage() / 1024.0 / 1024;
-    }
-    leveldb::Status status = pdb->Write(fSync ? syncoptions : writeoptions, &batch.batch);
-    dbwrapper_private::HandleError(status);
-    if (log_memory) {
-        double mem_after = DynamicMemoryUsage() / 1024.0 / 1024;
-        LogPrint(BCLog::LEVELDB, "WriteBatch memory usage: db=%s, before=%.1fMiB, after=%.1fMiB\n",
-                 m_name, mem_before, mem_after);
-    }
-    return true;
-}
-
-size_t CDBWrapper::DynamicMemoryUsage() const {
-    std::string memory;
-    if (!pdb->GetProperty("leveldb.approximate-memory-usage", &memory)) {
-        LogPrint(BCLog::LEVELDB, "Failed to get approximate-memory-usage property\n");
-        return 0;
-    }
-    return stoul(memory);
-}
-
-// Prefixed with null character to avoid collisions with other keys
-//
-// We must use a string constructor which specifies length so that we copy
-// past the null-terminator.
-const std::string CDBWrapper::OBFUSCATE_KEY_KEY("\000obfuscate_key", 14);
-
-const unsigned int CDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8;
-
-/**
- * Returns a string (consisting of 8 random bytes) suitable for use as an
- * obfuscating XOR key.
- */
-std::vector<unsigned char> CDBWrapper::CreateObfuscateKey() const
-{
-    unsigned char buff[OBFUSCATE_KEY_NUM_BYTES];
-    GetRandBytes(buff, OBFUSCATE_KEY_NUM_BYTES);
-    return std::vector<unsigned char>(&buff[0], &buff[OBFUSCATE_KEY_NUM_BYTES]);
-
-}
-
-bool CDBWrapper::IsEmpty()
-{
-    std::unique_ptr<CDBIterator> it(NewIterator());
-    it->SeekToFirst();
-    return !(it->Valid());
-}
-
-CDBIterator::~CDBIterator() { delete piter; }
-bool CDBIterator::Valid() const { return piter->Valid(); }
-void CDBIterator::SeekToFirst() { piter->SeekToFirst(); }
-void CDBIterator::Next() { piter->Next(); }
-
-namespace dbwrapper_private {
-
-void HandleError(const leveldb::Status& status)
-{
-    if (status.ok())
-        return;
-    const std::string errmsg = "Fatal LevelDB error: " + status.ToString();
-    LogPrintf("%s\n", errmsg);
-    LogPrintf("You can use -debug=leveldb to get more complete diagnostic messages\n");
-    throw dbwrapper_error(errmsg);
-}
-
-const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w)
-{
-    return w.obfuscate_key;
-}
-
-} // namespace dbwrapper_private

src/dbwrapper.h

@@ -1,350 +0,0 @@
-// Copyright (c) 2012-2018 The Bitcoin Core developers
-// Distributed under the MIT software license, see the accompanying
-// file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-#ifndef BITCOIN_DBWRAPPER_H
-#define BITCOIN_DBWRAPPER_H
-
-#include <clientversion.h>
-#include <fs.h>
-#include <serialize.h>
-#include <streams.h>
-#include <util.h>
-#include <utilstrencodings.h>
-#include <version.h>
-
-#include <leveldb/db.h>
-#include <leveldb/write_batch.h>
-
-class dbwrapper_error : public std::runtime_error
-{
-public:
-    explicit dbwrapper_error(const std::string& msg) : std::runtime_error(msg) {}
-};
-
-class CDBWrapper;
-
-/** These should be considered an implementation detail of the specific database.
- */
-namespace dbwrapper_private {
-
-/** Handle database error by throwing dbwrapper_error exception.
- */
-    void HandleError(const leveldb::Status& status);
-
-/** Work around circular dependency, as well as for testing in dbwrapper_tests.
- * Database obfuscation should be considered an implementation detail of the
- * specific database.
- */
-    const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w);
-
-};
-
-class CDBIterator
-{
-private:
-    const CDBWrapper &parent;
-    leveldb::Iterator *piter;
-
-public:
-
-    /**
-     * @param[in] _parent          Parent CDBWrapper instance.
-     * @param[in] _piter           The original leveldb iterator.
-     */
-    CDBIterator(const CDBWrapper &_parent, leveldb::Iterator *_piter) :
-            parent(_parent), piter(_piter) { };
-    ~CDBIterator();
-
-    bool Valid() const;
-
-    void SeekToFirst();
-
-    template<typename K> void Seek(const K& key) {
-        CDataStream ssKey(SER_DISK, CLIENT_VERSION);
-        ssKey << key;
-        leveldb::Slice slKey(ssKey.data(), ssKey.size());
-        piter->Seek(slKey);
-    }
-
-    void Next();
-
-    template<typename K> bool GetKey(K& key) {
-        leveldb::Slice slKey = piter->key();
-        try {
-            CDataStream ssKey(slKey.data(), slKey.data() + slKey.size(), SER_DISK, CLIENT_VERSION);
-            ssKey >> key;
-        } catch (const std::exception&) {
-            return false;
-        }
-        return true;
-    }
-
-    template<typename V> bool GetValue(V& value) {
-        leveldb::Slice slValue = piter->value();
-        try {
-            CDataStream ssValue(slValue.data(), slValue.data() + slValue.size(), SER_DISK, CLIENT_VERSION);
-            ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent));
-            ssValue >> value;
-        } catch (const std::exception&) {
-            return false;
-        }
-        return true;
-    }
-
-    unsigned int GetValueSize() {
-        return piter->value().size();
-    }
-
-};
-
-class CDBBatch;
-
-class CDBWrapper
-{
-    friend const std::vector<unsigned char>& dbwrapper_private::GetObfuscateKey(const CDBWrapper &w);
-private:
-    //! custom environment this database is using (may be nullptr in case of default environment)
-    leveldb::Env* penv;
-
-    //! database options used
-    leveldb::Options options;
-
-    //! options used when reading from the database
-    leveldb::ReadOptions readoptions;
-
-    //! options used when iterating over values of the database
-    leveldb::ReadOptions iteroptions;
-
-    //! options used when writing to the database
-    leveldb::WriteOptions writeoptions;
-
-    //! options used when sync writing to the database
-    leveldb::WriteOptions syncoptions;
-
-    //! the database itself
-    leveldb::DB* pdb;
-
-    //! the name of this database
-    std::string m_name;
-
-    //! a key used for optional XOR-obfuscation of the database
-    std::vector<unsigned char> obfuscate_key;
-
-    //! the key under which the obfuscation key is stored
-    static const std::string OBFUSCATE_KEY_KEY;
-
-    //! the length of the obfuscate key in number of bytes
-    static const unsigned int OBFUSCATE_KEY_NUM_BYTES;
-
-    std::vector<unsigned char> CreateObfuscateKey() const;
-
-public:
-    mutable CDataStream ssKey, ssValue;
-
-    /**
-     * @param[in] path        Location in the filesystem where leveldb data will be stored.
-     * @param[in] nCacheSize  Configures various leveldb cache settings.
-     * @param[in] fMemory     If true, use leveldb's memory environment.
-     * @param[in] fWipe       If true, remove all existing data.
-     * @param[in] obfuscate   If true, store data obfuscated via simple XOR. If false, XOR
-     *                        with a zero'd byte array.
-     */
-    CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false);
-    virtual ~CDBWrapper();
-
-    CDBWrapper(const CDBWrapper&) = delete;
-    /* CDBWrapper& operator=(const CDBWrapper&) = delete; */
-
-    template <typename K, typename V>
-    bool Read(const K& key, V& value) const
-    {
-        assert(ssKey.empty());
-        ssKey << key;
-        leveldb::Slice slKey(ssKey.data(), ssKey.size());
-
-        std::string strValue;
-        leveldb::Status status = pdb->Get(readoptions, slKey, &strValue);
-        ssKey.clear();
-        if (!status.ok()) {
-            if (status.IsNotFound())
-                return false;
-            LogPrintf("LevelDB read failure: %s\n", status.ToString());
-            dbwrapper_private::HandleError(status);
-        }
-        try {
-            CDataStream ssValue(strValue.data(), strValue.data() + strValue.size(), SER_DISK, CLIENT_VERSION);
-            ssValue.Xor(obfuscate_key);
-            ssValue >> value;
-        } catch (const std::exception&) {
-            return false;
-        }
-        return true;
-    }
-
-    template <typename K, typename V>
-    bool Write(const K& key, const V& value, bool fSync = false);
-
-    template <typename K>
-    bool Exists(const K& key) const
-    {
-        ssKey << key;
-        leveldb::Slice slKey(ssKey.data(), ssKey.size());
-
-        std::string strValue;
-        leveldb::Status status = pdb->Get(readoptions, slKey, &strValue);
-        ssKey.clear();
-        if (!status.ok()) {
-            if (status.IsNotFound())
-                return false;
-            LogPrintf("LevelDB read failure: %s\n", status.ToString());
-            dbwrapper_private::HandleError(status);
-        }
-        return true;
-    }
-
-    template <typename K>
-    bool Erase(const K& key, bool fSync = false);
-
-    bool WriteBatch(CDBBatch& batch, bool fSync = false);
-
-    // Get an estimate of LevelDB memory usage (in bytes).
-    size_t DynamicMemoryUsage() const;
-
-    // not available for LevelDB; provide for compatibility with BDB
-    bool Flush()
-    {
-        return true;
-    }
-
-    bool Sync();
-
-    CDBIterator *NewIterator()
-    {
-        return new CDBIterator(*this, pdb->NewIterator(iteroptions));
-    }
-
-    /**
-     * Return true if the database managed by this class contains no entries.
-     */
-    bool IsEmpty();
-
-    template<typename K>
-    size_t EstimateSize(const K& key_begin, const K& key_end) const
-    {
-        CDataStream ssKey1(SER_DISK, CLIENT_VERSION), ssKey2(SER_DISK, CLIENT_VERSION);
-        ssKey1.reserve(ssKey.capacity());
-        ssKey2.reserve(ssKey.capacity());
-        ssKey1 << key_begin;
-        ssKey2 << key_end;
-        leveldb::Slice slKey1(ssKey1.data(), ssKey1.size());
-        leveldb::Slice slKey2(ssKey2.data(), ssKey2.size());
-        uint64_t size = 0;
-        leveldb::Range range(slKey1, slKey2);
-        pdb->GetApproximateSizes(&range, 1, &size);
-        return size;
-    }
-
-    /**
-     * Compact a certain range of keys in the database.
-     */
-    template<typename K>
-    void CompactRange(const K& key_begin, const K& key_end) const
-    {
-        CDataStream ssKey1(SER_DISK, CLIENT_VERSION), ssKey2(SER_DISK, CLIENT_VERSION);
-        ssKey1.reserve(ssKey.capacity());
-        ssKey2.reserve(ssKey.capacity());
-        ssKey1 << key_begin;
-        ssKey2 << key_end;
-        leveldb::Slice slKey1(ssKey1.data(), ssKey1.size());
-        leveldb::Slice slKey2(ssKey2.data(), ssKey2.size());
-        pdb->CompactRange(&slKey1, &slKey2);
-    }
-
-};
-
-/** Batch of changes queued to be written to a CDBWrapper */
-class CDBBatch
-{
-    friend class CDBWrapper;
-    const CDBWrapper &parent;
-    leveldb::WriteBatch batch;
-
-    size_t size_estimate;
-    CDataStream ssKey, ssValue;
-
-public:
-    /**
-     * @param[in] _parent   CDBWrapper that this batch is to be submitted to
-     */
-    explicit CDBBatch(const CDBWrapper &_parent) : parent(_parent), size_estimate(0),
-        ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION) {
-        ssKey.reserve(parent.ssKey.capacity());
-        ssValue.reserve(parent.ssValue.capacity());
-    };
-
-    void Clear()
-    {
-        batch.Clear();
-        size_estimate = 0;
-    }
-
-    template <typename K, typename V>
-    void Write(const K& key, const V& value)
-    {
-        assert(ssKey.empty());
-        ssKey << key;
-        leveldb::Slice slKey(ssKey.data(), ssKey.size());
-
-        assert(ssValue.empty());
-        ssValue << value;
-        ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent));
-        leveldb::Slice slValue(ssValue.data(), ssValue.size());
-
-        batch.Put(slKey, slValue);
-        // LevelDB serializes writes as:
-        // - byte: header
-        // - varint: key length (1 byte up to 127B, 2 bytes up to 16383B, ...)
-        // - byte[]: key
-        // - varint: value length
-        // - byte[]: value
-        // The formula below assumes the key and value are both less than 16k.
-        size_estimate += 3 + (slKey.size() > 127) + slKey.size() + (slValue.size() > 127) + slValue.size();
-        ssKey.clear();
-        ssValue.clear();
-    }
-
-    template <typename K>
-    void Erase(const K& key)
-    {
-        ssKey << key;
-        leveldb::Slice slKey(ssKey.data(), ssKey.size());
-
-        batch.Delete(slKey);
-        // LevelDB serializes erases as:
-        // - byte: header
-        // - varint: key length
-        // - byte[]: key
-        // The formula below assumes the key is less than 16kB.
-        size_estimate += 2 + (slKey.size() > 127) + slKey.size();
-        ssKey.clear();
-    }
-
-    size_t SizeEstimate() const { return size_estimate; }
-};
-
-template<typename K>
-bool CDBWrapper::Erase(const K &key, bool fSync) {
-    CDBBatch batch(*this);
-    batch.Erase(key);
-    return WriteBatch(batch, fSync);
-}
-
-template<typename K, typename V>
-bool CDBWrapper::Write(const K &key, const V &value, bool fSync) {
-    CDBBatch batch(*this);
-    batch.Write(key, value);
-    return WriteBatch(batch, fSync);
-}
-
-#endif // BITCOIN_DBWRAPPER_H

src/httpserver.cpp

@@ -13,6 +13,7 @@
 #include <sync.h>
 #include <ui_interface.h>
 
+#include <deque>
 #include <memory>
 #include <stdio.h>
 #include <stdlib.h>

src/index/base.cpp

@@ -1,278 +0,0 @@
-// Copyright (c) 2017-2018 The Bitcoin Core developers
-// Distributed under the MIT software license, see the accompanying
-// file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-#include <chainparams.h>
-#include <index/base.h>
-#include <shutdown.h>
-#include <tinyformat.h>
-#include <ui_interface.h>
-#include <util.h>
-#include <validation.h>
-#include <warnings.h>
-
-constexpr char DB_BEST_BLOCK = 'B';
-
-constexpr int64_t SYNC_LOG_INTERVAL = 30; // seconds
-constexpr int64_t SYNC_LOCATOR_WRITE_INTERVAL = 30; // seconds
-
-template<typename... Args>
-static void FatalError(const char* fmt, const Args&... args)
-{
-    std::string strMessage = tfm::format(fmt, args...);
-    SetMiscWarning(strMessage);
-    LogPrintf("*** %s\n", strMessage);
-    uiInterface.ThreadSafeMessageBox(
-        "Error: A fatal internal error occurred, see debug.log for details",
-        "", CClientUIInterface::MSG_ERROR);
-    StartShutdown();
-}
-
-BaseIndex::DB::DB(const fs::path& path, size_t n_cache_size, bool f_memory, bool f_wipe, bool f_obfuscate) :
-    CDBWrapper(path, n_cache_size, f_memory, f_wipe, f_obfuscate)
-{}
-
-bool BaseIndex::DB::ReadBestBlock(CBlockLocator& locator) const
-{
-    bool success = Read(DB_BEST_BLOCK, locator);
-    if (!success) {
-        locator.SetNull();
-    }
-    return success;
-}
-
-bool BaseIndex::DB::WriteBestBlock(const CBlockLocator& locator)
-{
-    return Write(DB_BEST_BLOCK, locator);
-}
-
-BaseIndex::~BaseIndex()
-{
-    Interrupt();
-    Stop();
-}
-
-bool BaseIndex::Init()
-{
-    CBlockLocator locator;
-    if (!GetDB().ReadBestBlock(locator)) {
-        locator.SetNull();
-    }
-
-    LOCK(cs_main);
-    m_best_block_index = FindForkInGlobalIndex(chainActive, locator);
-    m_synced = m_best_block_index.load() == chainActive.Tip();
-    return true;
-}
-
-static const CBlockIndex* NextSyncBlock(const CBlockIndex* pindex_prev)
-{
-    AssertLockHeld(cs_main);
-
-    if (!pindex_prev) {
-        return chainActive.Genesis();
-    }
-
-    const CBlockIndex* pindex = chainActive.Next(pindex_prev);
-    if (pindex) {
-        return pindex;
-    }
-
-    return chainActive.Next(chainActive.FindFork(pindex_prev));
-}
-
-void BaseIndex::ThreadSync()
-{
-    const CBlockIndex* pindex = m_best_block_index.load();
-    if (!m_synced) {
-        auto& consensus_params = Params().GetConsensus();
-
-        int64_t last_log_time = 0;
-        int64_t last_locator_write_time = 0;
-        while (true) {
-            if (m_interrupt) {
-                WriteBestBlock(pindex);
-                return;
-            }
-
-            {
-                LOCK(cs_main);
-                const CBlockIndex* pindex_next = NextSyncBlock(pindex);
-                if (!pindex_next) {
-                    WriteBestBlock(pindex);
-                    m_best_block_index = pindex;
-                    m_synced = true;
-                    break;
-                }
-                pindex = pindex_next;
-            }
-
-            int64_t current_time = GetTime();
-            if (last_log_time + SYNC_LOG_INTERVAL < current_time) {
-                LogPrintf("Syncing %s with block chain from height %d\n",
-                          GetName(), pindex->nHeight);
-                last_log_time = current_time;
-            }
-
-            if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) {
-                WriteBestBlock(pindex);
-                last_locator_write_time = current_time;
-            }
-
-            CBlock block;
-            if (!ReadBlockFromDisk(block, pindex, consensus_params)) {
-                FatalError("%s: Failed to read block %s from disk",
-                           __func__, pindex->GetBlockHash().ToString());
-                return;
-            }
-            if (!WriteBlock(block, pindex)) {
-                FatalError("%s: Failed to write block %s to index database",
-                           __func__, pindex->GetBlockHash().ToString());
-                return;
-            }
-        }
-    }
-
-    if (pindex) {
-        LogPrintf("%s is enabled at height %d\n", GetName(), pindex->nHeight);
-    } else {
-        LogPrintf("%s is enabled\n", GetName());
-    }
-}
-
-bool BaseIndex::WriteBestBlock(const CBlockIndex* block_index)
-{
-    LOCK(cs_main);
-    if (!GetDB().WriteBestBlock(chainActive.GetLocator(block_index))) {
-        return error("%s: Failed to write locator to disk", __func__);
-    }
-    return true;
-}
-
-void BaseIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex,
-                               const std::vector<CTransactionRef>& txn_conflicted)
-{
-    if (!m_synced) {
-        return;
-    }
-
-    const CBlockIndex* best_block_index = m_best_block_index.load();
-    if (!best_block_index) {
-        if (pindex->nHeight != 0) {
-            FatalError("%s: First block connected is not the genesis block (height=%d)",
-                       __func__, pindex->nHeight);
-            return;
-        }
-    } else {
-        // Ensure block connects to an ancestor of the current best block. This should be the case
-        // most of the time, but may not be immediately after the sync thread catches up and sets
-        // m_synced. Consider the case where there is a reorg and the blocks on the stale branch are
-        // in the ValidationInterface queue backlog even after the sync thread has caught up to the
-        // new chain tip. In this unlikely event, log a warning and let the queue clear.
-        if (best_block_index->GetAncestor(pindex->nHeight - 1) != pindex->pprev) {
-            LogPrintf("%s: WARNING: Block %s does not connect to an ancestor of " /* Continued */
-                      "known best chain (tip=%s); not updating index\n",
-                      __func__, pindex->GetBlockHash().ToString(),
-                      best_block_index->GetBlockHash().ToString());
-            return;
-        }
-    }
-
-    if (WriteBlock(*block, pindex)) {
-        m_best_block_index = pindex;
-    } else {
-        FatalError("%s: Failed to write block %s to index",
-                   __func__, pindex->GetBlockHash().ToString());
-        return;
-    }
-}
-
-void BaseIndex::ChainStateFlushed(const CBlockLocator& locator)
-{
-    if (!m_synced) {
-        return;
-    }
-
-    const uint256& locator_tip_hash = locator.vHave.front();
-    const CBlockIndex* locator_tip_index;
-    {
-        LOCK(cs_main);
-        locator_tip_index = LookupBlockIndex(locator_tip_hash);
-    }
-
-    if (!locator_tip_index) {
-        FatalError("%s: First block (hash=%s) in locator was not found",
-                   __func__, locator_tip_hash.ToString());
-        return;
-    }
-
-    // This checks that ChainStateFlushed callbacks are received after BlockConnected. The check may fail
-    // immediately after the sync thread catches up and sets m_synced. Consider the case where
-    // there is a reorg and the blocks on the stale branch are in the ValidationInterface queue
-    // backlog even after the sync thread has caught up to the new chain tip. In this unlikely
-    // event, log a warning and let the queue clear.
-    const CBlockIndex* best_block_index = m_best_block_index.load();
-    if (best_block_index->GetAncestor(locator_tip_index->nHeight) != locator_tip_index) {
-        LogPrintf("%s: WARNING: Locator contains block (hash=%s) not on known best " /* Continued */
-                  "chain (tip=%s); not writing index locator\n",
-                  __func__, locator_tip_hash.ToString(),
-                  best_block_index->GetBlockHash().ToString());
-        return;
-    }
-
-    if (!GetDB().WriteBestBlock(locator)) {
-        error("%s: Failed to write locator to disk", __func__);
-    }
-}
-
-bool BaseIndex::BlockUntilSyncedToCurrentChain()
-{
-    AssertLockNotHeld(cs_main);
-
-    if (!m_synced) {
-        return false;
-    }
-
-    {
-        // Skip the queue-draining stuff if we know we're caught up with
-        // chainActive.Tip().
-        LOCK(cs_main);
-        const CBlockIndex* chain_tip = chainActive.Tip();
-        const CBlockIndex* best_block_index = m_best_block_index.load();
-        if (best_block_index->GetAncestor(chain_tip->nHeight) == chain_tip) {
-            return true;
-        }
-    }
-
-    LogPrintf("%s: %s is catching up on block notifications\n", __func__, GetName());
-    SyncWithValidationInterfaceQueue();
-    return true;
-}
-
-void BaseIndex::Interrupt()
-{
-    m_interrupt();
-}
-
-void BaseIndex::Start()
-{
-    // Need to register this ValidationInterface before running Init(), so that
-    // callbacks are not missed if Init sets m_synced to true.
-    RegisterValidationInterface(this);
-    if (!Init()) {
-        FatalError("%s: %s failed to initialize", __func__, GetName());
-        return;
-    }
-
-    m_thread_sync = std::thread(&TraceThread<std::function<void()>>, GetName(),
-                                std::bind(&BaseIndex::ThreadSync, this));
-}
-
-void BaseIndex::Stop()
-{
-    UnregisterValidationInterface(this);
-
-    if (m_thread_sync.joinable()) {
-        m_thread_sync.join();
-    }
-}

src/index/base.h

@@ -1,100 +0,0 @@
-// Copyright (c) 2017-2018 The Bitcoin Core developers
-// Distributed under the MIT software license, see the accompanying
-// file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-#ifndef BITCOIN_INDEX_BASE_H
-#define BITCOIN_INDEX_BASE_H
-
-#include <dbwrapper.h>
-#include <primitives/block.h>
-#include <primitives/transaction.h>
-#include <threadinterrupt.h>
-#include <uint256.h>
-#include <validationinterface.h>
-
-class CBlockIndex;
-
-/**
- * Base class for indices of blockchain data. This implements
- * CValidationInterface and ensures blocks are indexed sequentially according
- * to their position in the active chain.
- */
-class BaseIndex : public CValidationInterface
-{
-protected:
-    class DB : public CDBWrapper
-    {
-    public:
-        DB(const fs::path& path, size_t n_cache_size,
-           bool f_memory = false, bool f_wipe = false, bool f_obfuscate = false);
-
-        ~DB() override {}
-
-        /// Read block locator of the chain that the txindex is in sync with.
-        bool ReadBestBlock(CBlockLocator& locator) const;
-
-        /// Write block locator of the chain that the txindex is in sync with.
-        bool WriteBestBlock(const CBlockLocator& locator);
-    };
-
-private:
-    /// Whether the index is in sync with the main chain. The flag is flipped
-    /// from false to true once, after which point this starts processing
-    /// ValidationInterface notifications to stay in sync.
-    std::atomic<bool> m_synced{false};
-
-    /// The last block in the chain that the index is in sync with.
-    std::atomic<const CBlockIndex*> m_best_block_index{nullptr};
-
-    std::thread m_thread_sync;
-    CThreadInterrupt m_interrupt;
-
-    /// Sync the index with the block index starting from the current best block.
-    /// Intended to be run in its own thread, m_thread_sync, and can be
-    /// interrupted with m_interrupt. Once the index gets in sync, the m_synced
-    /// flag is set and the BlockConnected ValidationInterface callback takes
-    /// over and the sync thread exits.
-    void ThreadSync();
-
-    /// Write the current chain block locator to the DB.
-    bool WriteBestBlock(const CBlockIndex* block_index);
-
-protected:
-    void BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex,
-                        const std::vector<CTransactionRef>& txn_conflicted) override;
-
-    void ChainStateFlushed(const CBlockLocator& locator) override;
-
-    /// Initialize internal state from the database and block index.
-    virtual bool Init();
-
-    /// Write update index entries for a newly connected block.
-    virtual bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) { return true; }
-
-    virtual DB& GetDB() const = 0;
-
-    /// Get the name of the index for display in logs.
-    virtual const char* GetName() const = 0;
-
-public:
-    /// Destructor interrupts sync thread if running and blocks until it exits.
-    virtual ~BaseIndex();
-
-    /// Blocks the current thread until the index is caught up to the current
-    /// state of the block chain. This only blocks if the index has gotten in
-    /// sync once and only needs to process blocks in the ValidationInterface
-    /// queue. If the index is catching up from far behind, this method does
-    /// not block and immediately returns false.
-    bool BlockUntilSyncedToCurrentChain();
-
-    void Interrupt();
-
-    /// Start initializes the sync state and registers the instance as a
-    /// ValidationInterface so that it stays in sync with blockchain updates.
-    void Start();
-
-    /// Stops the instance from staying in sync with blockchain updates.
-    void Stop();
-};
-
-#endif // BITCOIN_INDEX_BASE_H

src/index/txindex.cpp

@@ -1,262 +0,0 @@
-// Copyright (c) 2017-2018 The Bitcoin Core developers
-// Distributed under the MIT software license, see the accompanying
-// file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-#include <index/txindex.h>
-#include <shutdown.h>
-#include <ui_interface.h>
-#include <util.h>
-#include <validation.h>
-
-#include <boost/thread.hpp>
-
-constexpr char DB_BEST_BLOCK = 'B';
-constexpr char DB_TXINDEX = 't';
-constexpr char DB_TXINDEX_BLOCK = 'T';
-
-std::unique_ptr<TxIndex> g_txindex;
-
-/**
- * Access to the txindex database (indexes/txindex/)
- *
- * The database stores a block locator of the chain the database is synced to
- * so that the TxIndex can efficiently determine the point it last stopped at.
- * A locator is used instead of a simple hash of the chain tip because blocks
- * and block index entries may not be flushed to disk until after this database
- * is updated.
- */
-class TxIndex::DB : public BaseIndex::DB
-{
-public:
-    explicit DB(size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
-    ~DB() override {}
-
-    /// Read the disk location of the transaction data with the given hash. Returns false if the
-    /// transaction hash is not indexed.
-    bool ReadTxPos(const uint256& txid, CDiskTxPos& pos) const;
-
-    /// Write a batch of transaction positions to the DB.
-    bool WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos);
-
-    /// Migrate txindex data from the block tree DB, where it may be for older nodes that have not
-    /// been upgraded yet to the new database.
-    bool MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator);
-};
-
-TxIndex::DB::DB(size_t n_cache_size, bool f_memory, bool f_wipe) :
-    BaseIndex::DB(GetDataDir() / "indexes" / "txindex", n_cache_size, f_memory, f_wipe)
-{}
-
-bool TxIndex::DB::ReadTxPos(const uint256 &txid, CDiskTxPos& pos) const
-{
-    return Read(std::make_pair(DB_TXINDEX, txid), pos);
-}
-
-bool TxIndex::DB::WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos)
-{
-    CDBBatch batch(*this);
-    for (const auto& tuple : v_pos) {
-        batch.Write(std::make_pair(DB_TXINDEX, tuple.first), tuple.second);
-    }
-    return WriteBatch(batch);
-}
-
-/*
- * Safely persist a transfer of data from the old txindex database to the new one, and compact the
- * range of keys updated. This is used internally by MigrateData.
- */
-static void WriteTxIndexMigrationBatches(CDBWrapper& newdb, CDBWrapper& olddb,
-                                         CDBBatch& batch_newdb, CDBBatch& batch_olddb,
-                                         const std::pair<unsigned char, uint256>& begin_key,
-                                         const std::pair<unsigned char, uint256>& end_key)
-{
-    // Sync new DB changes to disk before deleting from old DB.
-    newdb.WriteBatch(batch_newdb, /*fSync=*/ true);
-    olddb.WriteBatch(batch_olddb);
-    olddb.CompactRange(begin_key, end_key);
-
-    batch_newdb.Clear();
-    batch_olddb.Clear();
-}
-
-bool TxIndex::DB::MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator)
-{
-    // The prior implementation of txindex was always in sync with block index
-    // and presence was indicated with a boolean DB flag. If the flag is set,
-    // this means the txindex from a previous version is valid and in sync with
-    // the chain tip. The first step of the migration is to unset the flag and
-    // write the chain hash to a separate key, DB_TXINDEX_BLOCK. After that, the
-    // index entries are copied over in batches to the new database. Finally,
-    // DB_TXINDEX_BLOCK is erased from the old database and the block hash is
-    // written to the new database.
-    //
-    // Unsetting the boolean flag ensures that if the node is downgraded to a
-    // previous version, it will not see a corrupted, partially migrated index
-    // -- it will see that the txindex is disabled. When the node is upgraded
-    // again, the migration will pick up where it left off and sync to the block
-    // with hash DB_TXINDEX_BLOCK.
-    bool f_legacy_flag = false;
-    block_tree_db.ReadFlag("txindex", f_legacy_flag);
-    if (f_legacy_flag) {
-        if (!block_tree_db.Write(DB_TXINDEX_BLOCK, best_locator)) {
-            return error("%s: cannot write block indicator", __func__);
-        }
-        if (!block_tree_db.WriteFlag("txindex", false)) {
-            return error("%s: cannot write block index db flag", __func__);
-        }
-    }
-
-    CBlockLocator locator;
-    if (!block_tree_db.Read(DB_TXINDEX_BLOCK, locator)) {
-        return true;
-    }
-
-    int64_t count = 0;
-    LogPrintf("Upgrading txindex database... [0%%]\n");
-    uiInterface.ShowProgress(_("Upgrading txindex database"), 0, true);
-    int report_done = 0;
-    const size_t batch_size = 1 << 24; // 16 MiB
-
-    CDBBatch batch_newdb(*this);
-    CDBBatch batch_olddb(block_tree_db);
-
-    std::pair<unsigned char, uint256> key;
-    std::pair<unsigned char, uint256> begin_key{DB_TXINDEX, uint256()};
-    std::pair<unsigned char, uint256> prev_key = begin_key;
-
-    bool interrupted = false;
-    std::unique_ptr<CDBIterator> cursor(block_tree_db.NewIterator());
-    for (cursor->Seek(begin_key); cursor->Valid(); cursor->Next()) {
-        boost::this_thread::interruption_point();
-        if (ShutdownRequested()) {
-            interrupted = true;
-            break;
-        }
-
-        if (!cursor->GetKey(key)) {
-            return error("%s: cannot get key from valid cursor", __func__);
-        }
-        if (key.first != DB_TXINDEX) {
-            break;
-        }
-
-        // Log progress every 10%.
-        if (++count % 256 == 0) {
-            // Since txids are uniformly random and traversed in increasing order, the high 16 bits
-            // of the hash can be used to estimate the current progress.
-            const uint256& txid = key.second;
-            uint32_t high_nibble =
-                (static_cast<uint32_t>(*(txid.begin() + 0)) << 8) +
-                (static_cast<uint32_t>(*(txid.begin() + 1)) << 0);
-            int percentage_done = (int)(high_nibble * 100.0 / 65536.0 + 0.5);
-
-            uiInterface.ShowProgress(_("Upgrading txindex database"), percentage_done, true);
-            if (report_done < percentage_done/10) {
-                LogPrintf("Upgrading txindex database... [%d%%]\n", percentage_done);
-                report_done = percentage_done/10;
-            }
-        }
-
-        CDiskTxPos value;
-        if (!cursor->GetValue(value)) {
-            return error("%s: cannot parse txindex record", __func__);
-        }
-        batch_newdb.Write(key, value);
-        batch_olddb.Erase(key);
-
-        if (batch_newdb.SizeEstimate() > batch_size || batch_olddb.SizeEstimate() > batch_size) {
-            // NOTE: it's OK to delete the key pointed at by the current DB cursor while iterating
-            // because LevelDB iterators are guaranteed to provide a consistent view of the
-            // underlying data, like a lightweight snapshot.
-            WriteTxIndexMigrationBatches(*this, block_tree_db,
-                                         batch_newdb, batch_olddb,
-                                         prev_key, key);
-            prev_key = key;
-        }
-    }
-
-    // If these final DB batches complete the migration, write the best block
-    // hash marker to the new database and delete from the old one. This signals
-    // that the former is fully caught up to that point in the blockchain and
-    // that all txindex entries have been removed from the latter.
-    if (!interrupted) {
-        batch_olddb.Erase(DB_TXINDEX_BLOCK);
-        batch_newdb.Write(DB_BEST_BLOCK, locator);
-    }
-
-    WriteTxIndexMigrationBatches(*this, block_tree_db,
-                                 batch_newdb, batch_olddb,
-                                 begin_key, key);
-
-    if (interrupted) {
-        LogPrintf("[CANCELLED].\n");
-        return false;
-    }
-
-    uiInterface.ShowProgress("", 100, false);
-
-    LogPrintf("[DONE].\n");
-    return true;
-}
-
-TxIndex::TxIndex(size_t n_cache_size, bool f_memory, bool f_wipe)
-    : m_db(MakeUnique<TxIndex::DB>(n_cache_size, f_memory, f_wipe))
-{}
-
-TxIndex::~TxIndex() {}
-
-bool TxIndex::Init()
-{
-    LOCK(cs_main);
-
-    // Attempt to migrate txindex from the old database to the new one. Even if
-    // chain_tip is null, the node could be reindexing and we still want to
-    // delete txindex records in the old database.
-    if (!m_db->MigrateData(*pblocktree, chainActive.GetLocator())) {
-        return false;
-    }
-
-    return BaseIndex::Init();
-}
-
-bool TxIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex)
-{
-    CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size()));
-    std::vector<std::pair<uint256, CDiskTxPos>> vPos;
-    vPos.reserve(block.vtx.size());
-    for (const auto& tx : block.vtx) {
-        vPos.emplace_back(tx->GetHash(), pos);
-        pos.nTxOffset += ::GetSerializeSize(*tx, SER_DISK, CLIENT_VERSION);
-    }
-    return m_db->WriteTxs(vPos);
-}
-
-BaseIndex::DB& TxIndex::GetDB() const { return *m_db; }
-
-bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const
-{
-    CDiskTxPos postx;
-    if (!m_db->ReadTxPos(tx_hash, postx)) {
-        return false;
-    }
-
-    CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION);
-    if (file.IsNull()) {
-        return error("%s: OpenBlockFile failed", __func__);
-    }
-    CBlockHeader header;
-    try {
-        file >> header;
-        if (fseek(file.Get(), postx.nTxOffset, SEEK_CUR)) {
-            return error("%s: fseek(...) failed", __func__);
-        }
-        file >> tx;
-    } catch (const std::exception& e) {
-        return error("%s: Deserialize or I/O error - %s", __func__, e.what());
-    }
-    if (tx->GetHash() != tx_hash) {
-        return error("%s: txid mismatch", __func__);
-    }
-    block_hash = header.GetHash();
-    return true;
-}

src/index/txindex.h

@@ -1,79 +0,0 @@
-// Copyright (c) 2017-2018 The Bitcoin Core developers
-// Distributed under the MIT software license, see the accompanying
-// file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-#ifndef BITCOIN_INDEX_TXINDEX_H
-#define BITCOIN_INDEX_TXINDEX_H
-
-#include <chain.h>
-#include <index/base.h>
-#include <txdb.h>
-
-struct CDiskTxPos : public CDiskBlockPos
-{
-    unsigned int nTxOffset; // after header
-
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action) {
-        READWRITEAS(CDiskBlockPos, *this);
-        READWRITE(VARINT(nTxOffset));
-    }
-
-    CDiskTxPos(const CDiskBlockPos &blockIn, unsigned int nTxOffsetIn) : CDiskBlockPos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) {
-    }
-
-    CDiskTxPos() {
-        SetNull();
-    }
-
-    void SetNull() {
-        CDiskBlockPos::SetNull();
-        nTxOffset = 0;
-    }
-};
-
-/**
- * TxIndex is used to look up transactions included in the blockchain by hash.
- * The index is written to a LevelDB database and records the filesystem
- * location of each transaction by transaction hash.
- */
-class TxIndex final : public BaseIndex
-{
-protected:
-    class DB;
-
-private:
-    const std::unique_ptr<DB> m_db;
-
-protected:
-    /// Override base class init to migrate from old database.
-    bool Init() override;
-
-    bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override;
-
-    BaseIndex::DB& GetDB() const override;
-
-    const char* GetName() const override { return "txindex"; }
-
-public:
-    /// Constructs the index, which becomes available to be queried.
-    explicit TxIndex(size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
-
-    // Destructor is declared because this class contains a unique_ptr to an incomplete type.
-    virtual ~TxIndex() override;
-
-    /// Look up a transaction by hash.
-    ///
-    /// @param[in]   tx_hash  The hash of the transaction to be returned.
-    /// @param[out]  block_hash  The hash of the block the transaction is found in.
-    /// @param[out]  tx  The transaction itself.
-    /// @return  true if transaction is found, false otherwise
-    bool FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const;
-};
-
-/// The global transaction index, used in GetTransaction. May be null.
-extern std::unique_ptr<TxIndex> g_txindex;
-
-#endif // BITCOIN_INDEX_TXINDEX_H

src/init.cpp

@@ -14,13 +14,14 @@
 #include <chain.h>
 #include <chainparams.h>
 #include <checkpoints.h>
-#include <claimtrie.h>
+#include <claimtrie/forks.h>
+#include <claimtrie/hashes.h>
+#include <clientversion.h>
 #include <compat/sanity.h>
 #include <consensus/validation.h>
 #include <fs.h>
 #include <httpserver.h>
 #include <httprpc.h>
-#include <index/txindex.h>
 #include <key.h>
 #include <lbry.h>
 #include <validation.h>
@@ -36,6 +37,7 @@
 #include <rpc/blockchain.h>
 #include <script/standard.h>
 #include <script/sigcache.h>
+#include <sqlite/hdr/sqlite_modern_cpp/log.h>
 #include <scheduler.h>
 #include <shutdown.h>
 #include <timedata.h>
@@ -46,11 +48,10 @@
 #include <uint256.h>
 #include <util.h>
 #include <utilmoneystr.h>
+#include <utilstrencodings.h>
 #include <validationinterface.h>
 #include <warnings.h>
 #include <walletinitinterface.h>
-#include <stdint.h>
-#include <stdio.h>
 
 #ifndef WIN32
 #include <signal.h>
@@ -181,9 +182,6 @@ void Interrupt()
     InterruptMapPort();
     if (g_connman)
         g_connman->Interrupt();
-    if (g_txindex) {
-        g_txindex->Interrupt();
-    }
 }
 
 void Shutdown()
@@ -212,7 +210,6 @@ void Shutdown()
     // using the other before destroying them.
     if (peerLogic) UnregisterValidationInterface(peerLogic.get());
     if (g_connman) g_connman->Stop();
-    if (g_txindex) g_txindex->Stop();
 
     StopTorControl();
 
@@ -225,7 +222,6 @@ void Shutdown()
     // destruct and reset all to nullptr.
     peerLogic.reset();
     g_connman.reset();
-    g_txindex.reset();
 
     if (g_is_mempool_loaded && gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) {
         DumpMempool();
@@ -245,7 +241,7 @@ void Shutdown()
 
     // FlushStateToDisk generates a ChainStateFlushed callback, which we should avoid missing
     if (pcoinsTip != nullptr) {
-        FlushStateToDisk();
+        FlushStateToDisk(true);
     }
 
     // After there are no more peers/RPC left to give us new data which may generate
@@ -261,7 +257,7 @@ void Shutdown()
     {
         LOCK(cs_main);
         if (pcoinsTip != nullptr) {
-            FlushStateToDisk();
+            FlushStateToDisk(true);
         }
         pcoinsTip.reset();
         pcoinscatcher.reset();
@@ -353,7 +349,7 @@ void SetupServerArgs()
 
     // Hidden Options
     std::vector<std::string> hidden_args = {"-rpcssl", "-benchmark", "-h", "-help", "-socks", "-tor", "-debugnet", "-whitelistalwaysrelay",
-        "-prematurewitness", "-walletprematurewitness", "-promiscuousmempoolflags", "-blockminsize", "-dbcrashratio", "-forcecompactdb", "-usehd",
+        "-prematurewitness", "-walletprematurewitness", "-promiscuousmempoolflags", "-blockminsize", "-forcecompactdb", "-usehd",
         // GUI args. These will be overwritten by SetupUIArgs for the GUI
         "-allowselfsignedrootcertificates", "-choosedatadir", "-lang=<lang>", "-min", "-resetguisettings", "-rootcertificates=<file>", "-splash", "-uiplatform"};
 
@@ -369,13 +365,12 @@ void SetupServerArgs()
     gArgs.AddArg("-blocksonly", strprintf("Whether to operate in a blocks only mode (default: %u)", DEFAULT_BLOCKSONLY), true, OptionsCategory::OPTIONS);
     gArgs.AddArg("-conf=<file>", strprintf("Specify configuration file. Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME), false, OptionsCategory::OPTIONS);
     gArgs.AddArg("-datadir=<dir>", "Specify data directory", false, OptionsCategory::OPTIONS);
-    gArgs.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize), true, OptionsCategory::OPTIONS);
     gArgs.AddArg("-dbcache=<n>", strprintf("Set database cache size in megabytes (%d to %d, default: %d)", nMinDbCache, nMaxDbCache, nDefaultDbCache), false, OptionsCategory::OPTIONS);
-    gArgs.AddArg("-claimtriecache=<n>", strprintf("Set claim trie cache size in megabytes (%d to %d, default: %d)", nMinDbCache, nMaxDbCache, nDefaultDbCache), false, OptionsCategory::OPTIONS);
     gArgs.AddArg("-debuglogfile=<file>", strprintf("Specify location of debug log file. Relative paths will be prefixed by a net-specific datadir location. (-nodebuglogfile to disable; default: %s)", DEFAULT_DEBUGLOGFILE), false, OptionsCategory::OPTIONS);
     gArgs.AddArg("-feefilter", strprintf("Tell other nodes to filter invs to us by our mempool min fee (default: %u)", DEFAULT_FEEFILTER), true, OptionsCategory::OPTIONS);
     gArgs.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", false, OptionsCategory::OPTIONS);
     gArgs.AddArg("-loadblock=<file>", "Imports blocks from external blk000??.dat file on startup", false, OptionsCategory::OPTIONS);
+    gArgs.AddArg("-maxblockfilesize=<n>", strprintf("Keep the block files below n megabytes each. (default: %d, minimum: 8)", DEFAULT_MAX_BLOCKFILE_SIZE >> 20U), false, OptionsCategory::OPTIONS);
     gArgs.AddArg("-maxmempool=<n>", strprintf("Keep the transaction memory pool below <n> megabytes (default: %u)", DEFAULT_MAX_MEMPOOL_SIZE), false, OptionsCategory::OPTIONS);
     gArgs.AddArg("-maxorphantx=<n>", strprintf("Keep at most <n> unconnectable transactions in memory (default: %u)", DEFAULT_MAX_ORPHAN_TRANSACTIONS), false, OptionsCategory::OPTIONS);
     gArgs.AddArg("-mempoolexpiry=<n>", strprintf("Do not keep transactions in the mempool longer than <n> hours (default: %u)", DEFAULT_MEMPOOL_EXPIRY), false, OptionsCategory::OPTIONS);
@@ -388,7 +383,7 @@ void SetupServerArgs()
 #else
     hidden_args.emplace_back("-pid");
 #endif
-    gArgs.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex and -rescan. "
+    gArgs.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -rescan. "
             "Warning: Reverting this setting requires re-downloading the entire blockchain. "
             "(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >=%u = automatically prune block files to stay under the specified target size in MiB)", MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024), false, OptionsCategory::OPTIONS);
     gArgs.AddArg("-reindex", "Rebuild chain state and block index from the blk*.dat files on disk", false, OptionsCategory::OPTIONS);
@@ -398,9 +393,7 @@ void SetupServerArgs()
 #else
     hidden_args.emplace_back("-sysperms");
 #endif
-    gArgs.AddArg("-txindex", strprintf("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)", DEFAULT_TXINDEX), false, OptionsCategory::OPTIONS);
+    gArgs.AddArg("-txindex", "Deprecated", false, OptionsCategory::HIDDEN);
-    gArgs.AddArg("-memfile=<GiB>", "Use a memory mapped file for the claimtrie allocations (default: use RAM instead)", false, OptionsCategory::OPTIONS);
-
     gArgs.AddArg("-addnode=<ip>", "Add a node to connect to and attempt to keep the connection open (see the `addnode` RPC command help for more info). This option can be specified multiple times to add multiple nodes.", false, OptionsCategory::CONNECTION);
     gArgs.AddArg("-banscore=<n>", strprintf("Threshold for disconnecting misbehaving peers (default: %u)", DEFAULT_BANSCORE_THRESHOLD), false, OptionsCategory::CONNECTION);
     gArgs.AddArg("-bantime=<n>", strprintf("Number of seconds to keep misbehaving peers from reconnecting (default: %u)", DEFAULT_MISBEHAVING_BANTIME), false, OptionsCategory::CONNECTION);
@@ -628,7 +621,7 @@ static void CleanupBlockRevFiles()
     // start removing block files.
     int nContigCounter = 0;
     for (const std::pair<const std::string, fs::path>& item : mapBlockFiles) {
-        if (atoi(item.first) == nContigCounter) {
+        if (std::atoi(item.first.c_str()) == nContigCounter) {
             nContigCounter++;
             continue;
         }
@@ -936,12 +929,6 @@ bool AppInitParameterInteraction()
         return InitError(strprintf(_("Specified blocks directory \"%s\" does not exist."), gArgs.GetArg("-blocksdir", "").c_str()));
     }
 
-    // if using block pruning, then disallow txindex
-    if (gArgs.GetArg("-prune", 0)) {
-        if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX))
-            return InitError(_("Prune mode is incompatible with -txindex."));
-    }
-
     // -bind and -whitebind can't be set when not listening
     size_t nUserBind = gArgs.GetArgs("-bind").size() + gArgs.GetArgs("-whitebind").size();
     if (nUserBind != 0 && !gArgs.GetBoolArg("-listen", DEFAULT_LISTEN)) {
@@ -955,8 +942,13 @@ bool AppInitParameterInteraction()
 
     // Trim requested connection counts, to fit into system limitations
     // <int> in std::min<int>(...) to work around FreeBSD compilation issue described in #2695
-    nMaxConnections = std::max(std::min<int>(nMaxConnections, FD_SETSIZE - nBind - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS), 0);
     nFD = RaiseFileDescriptorLimit(nMaxConnections + MIN_CORE_FILEDESCRIPTORS + MAX_ADDNODE_CONNECTIONS);
+#ifdef USE_POLL
+    int fd_max = nFD;
+#else
+    int fd_max = FD_SETSIZE;
+#endif
+    nMaxConnections = std::max(std::min<int>(nMaxConnections, fd_max - nBind - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS), 0);
     if (nFD < MIN_CORE_FILEDESCRIPTORS)
         return InitError(_("Not enough file descriptors available."));
     nMaxConnections = std::min(nFD - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS, nMaxConnections);
@@ -1227,6 +1219,8 @@ bool AppInitLockDataDirectory()
     return true;
 }
 
+extern CChainState g_chainstate;
+
 bool AppInitMain()
 {
     const CChainParams& chainparams = Params();
@@ -1257,6 +1251,15 @@ bool AppInitMain()
             gArgs.GetArg("-datadir", ""), fs::current_path().string());
     }
 
+    sqlite::error_log(
+            [](sqlite::sqlite_exception& e) {
+                LogPrintf("Error with Sqlite: %s\n", e.what());
+            },
+            [](sqlite::errors::misuse& e) {
+                // You can behave differently to specific errors
+            }
+    );
+
     InitSignatureCache();
     InitScriptExecutionCache();
 
@@ -1419,29 +1422,29 @@ bool AppInitMain()
     int64_t nTotalCache = (gArgs.GetArg("-dbcache", nDefaultDbCache) << 20);
     nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache
     nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); // total cache cannot be greater than nMaxDbcache
-    int64_t nBlockTreeDBCache = std::min(nTotalCache / 8, nMaxBlockDBCache << 20);
+
+    // we're going to chop the cache into three pieces:
+    // the coin cache, the block cache, the claimtrie cache
+    // however, we want the claimtrie cache to be larger than the others
+
+    int64_t nBlockTreeDBCache = std::min(nTotalCache / 4, nMaxBlockDBCache << 20);
+    int64_t nCoinDBCache = std::min(nTotalCache / 4, nMaxCoinsDBCache << 20);
+    int64_t nClaimtrieCache = nTotalCache / 4;
     nTotalCache -= nBlockTreeDBCache;
-    int64_t nTxIndexCache = std::min(nTotalCache / 8, gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxTxIndexCache << 20 : 0);
-    nTotalCache -= nTxIndexCache;
-    int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache
-    nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache
     nTotalCache -= nCoinDBCache;
+    nTotalCache -= nClaimtrieCache;
     nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache
     std::cout << "nTotalCache: " << nTotalCache << ", nCoinCacheUsage: " << nCoinCacheUsage << ", nCoinDBCache: " << nCoinDBCache << std::endl;
     int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000;
     LogPrintf("Cache configuration:\n");
-    LogPrintf("* Using %.1fMiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024));
+    LogPrintf("* Using %.1fMiB for block index database cache\n", nBlockTreeDBCache * (1.0 / 1024 / 1024));
-    if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) {
+    LogPrintf("* Using %.1fMiB for chain state database cache\n", nCoinDBCache * (1.0 / 1024 / 1024));
-        LogPrintf("* Using %.1fMiB for transaction index database\n", nTxIndexCache * (1.0 / 1024 / 1024));
+    LogPrintf("* Using %.1fMiB for claimtrie database cache\n", nClaimtrieCache * (1.0 / 1024 / 1024));
-    }
-    LogPrintf("* Using %.1fMiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024));
     LogPrintf("* Using %.1fMiB for in-memory UTXO set (plus up to %.1fMiB of unused mempool space)\n", nCoinCacheUsage * (1.0 / 1024 / 1024), nMempoolSizeMax * (1.0 / 1024 / 1024));
 
-    g_memfileSize = gArgs.GetArg("-memfile", 0u);
-
     bool fLoaded = false;
     while (!fLoaded && !ShutdownRequested()) {
-        bool fReset = fReindex;
+        bool fReset = fReindex; // reindex flag gets changed out from underneath us
         std::string strLoadError;
 
         uiInterface.InitMessage(_("Loading block index..."));
@@ -1459,11 +1462,13 @@ bool AppInitMain()
                 // fails if it's still open from the previous loop. Close it first:
                 pblocktree.reset();
                 pblocktree.reset(new CBlockTreeDB(nBlockTreeDBCache, false, fReset));
-                delete pclaimTrie;
+
-                int64_t trieCacheMB = gArgs.GetArg("-claimtriecache", nDefaultDbCache);
+                // use faster N way hash function
-                trieCacheMB = std::min(trieCacheMB, nMaxDbCache);
+                // NOTE: it assumes memory is continuous
-                trieCacheMB = std::max(trieCacheMB, nMinDbCache);
+                // that means uint256 itself should use std::array or raw pointer
-                pclaimTrie = new CClaimTrie(false, fReindex || fReindexChainState, 32, trieCacheMB);
+                sha256n_way = [](std::vector<uint256>& hashes) {
+                    SHA256D64(hashes[0].begin(), hashes[0].begin(), hashes.size() / 2);
+                };
 
                 if (fReset) {
                     pblocktree->WriteReindexing(true);
@@ -1479,13 +1484,13 @@ bool AppInitMain()
                 // Note that it also sets fReindex based on the disk flag!
                 // From here on out fReindex and fReset mean something different!
                 if (!LoadBlockIndex(chainparams)) {
-                    strLoadError = _("Error loading block database");
+                    strLoadError = _("Error loading block index database");
                     break;
                 }
 
                 // If the loaded chain has a wrong genesis, bail out immediately
                 // (we're likely using a testnet datadir, or the other way around).
-                if (!mapBlockIndex.empty() && !LookupBlockIndex(chainparams.GetConsensus().hashGenesisBlock)) {
+                if (!g_chainstate.mapBlockIndex.empty() && !LookupBlockIndex(chainparams.GetConsensus().hashGenesisBlock)) {
                     return InitError(_("Incorrect or no genesis block found. Wrong datadir for network?"));
                 }
 
@@ -1511,12 +1516,21 @@ bool AppInitMain()
                 pcoinsdbview.reset(new CCoinsViewDB(nCoinDBCache, false, fReset || fReindexChainState));
                 pcoinscatcher.reset(new CCoinsViewErrorCatcher(pcoinsdbview.get()));
 
-                // If necessary, upgrade from older database format.
+                if (g_logger->Enabled() && LogAcceptCategory(BCLog::CLAIMS))
-                // This is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate
+                    CLogPrint::global().setLogger(g_logger);
-                if (!pcoinsdbview->Upgrade()) {
+
-                    strLoadError = _("Error upgrading chainstate database");
+                delete pclaimTrie;
-                    break;
+                auto& consensus = chainparams.GetConsensus();
-                }
+                pclaimTrie = new CClaimTrie(nClaimtrieCache, fReset || fReindexChainState, 0,
+                                            GetDataDir().string(),
+                                            consensus.nNormalizedNameForkHeight,
+                                            consensus.nMinRemovalWorkaroundHeight,
+                                            consensus.nMaxRemovalWorkaroundHeight,
+                                            consensus.nOriginalClaimExpirationTime,
+                                            consensus.nExtendedClaimExpirationTime,
+                                            consensus.nExtendedClaimExpirationForkHeight,
+                                            consensus.nAllClaimsInMerkleForkHeight,
+                                            32);
 
                 // ReplayBlocks is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate
                 if (!ReplayBlocks(chainparams, pcoinsdbview.get())) {
@@ -1537,10 +1551,10 @@ bool AppInitMain()
                     assert(chainActive.Tip() != nullptr);
                 }
 
-                CClaimTrieCache trieCache(pclaimTrie);
+                auto tip = chainActive.Tip();
-                if (!trieCache.ReadFromDisk(chainActive.Tip()))
+                LogPrintf("Checking existing claim trie consistency...\n");
-                {
+                if (tip && !CClaimTrieCache(pclaimTrie).validateDb(tip->nHeight, tip->hashClaimTrie)) {
-                    strLoadError = _("Error loading the claim trie from disk");
+                    strLoadError = _("Error validating the stored claim trie");
                     break;
                 }
 
@@ -1592,7 +1606,7 @@ bool AppInitMain()
             if (!fReset) {
                 bool fRet = uiInterface.ThreadSafeQuestion(
                     strLoadError + ".\n\n" + _("Do you want to rebuild the block database now?"),
-                    strLoadError + ".\nPlease restart with -reindex or -reindex-chainstate to recover.",
+                    strLoadError + ".\nPlease restart with -reindex to recover.",
                     "", CClientUIInterface::MSG_ERROR | CClientUIInterface::BTN_ABORT);
                 if (fRet) {
                     fReindex = true;
@@ -1605,6 +1619,14 @@ bool AppInitMain()
                 return InitError(strLoadError);
             }
         }
+
+        if (fReindex) {
+            // remove old LevelDB indexes
+            boost::system::error_code ec;
+            fs::remove_all(GetBlocksDir() / "index", ec);
+            fs::remove_all(GetDataDir() / "chainstate", ec);
+            fs::remove_all(GetDataDir() / "claimtrie", ec);
+        }
     }
 
     // As LoadBlockIndex can take several minutes, it's possible the user
@@ -1624,8 +1646,7 @@ bool AppInitMain()
 
     // ********************************************************* Step 8: start indexers
     if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) {
-        g_txindex = MakeUnique<TxIndex>(nTxIndexCache, false, fReindex);
+        LogPrintf("The txindex parameter is no longer necessary. It is always on.\n");
-        g_txindex->Start();
     }
 
     // ********************************************************* Step 9: load wallet
@@ -1699,7 +1720,6 @@ bool AppInitMain()
     //// debug print
     {
         LOCK(cs_main);
-        LogPrintf("mapBlockIndex.size() = %u\n", mapBlockIndex.size());
         chain_active_height = chainActive.Height();
     }
     LogPrintf("nBestHeight = %d\n", chain_active_height);

src/interfaces/wallet.cpp

@@ -88,8 +88,7 @@ WalletTx MakeWalletTx(CWallet& wallet, const CWalletTx& wtx)
 WalletTxStatus MakeWalletTxStatus(const CWalletTx& wtx)
 {
     WalletTxStatus result;
-    auto mi = ::mapBlockIndex.find(wtx.hashBlock);
+    CBlockIndex* block = LookupBlockIndex(wtx.hashBlock);
-    CBlockIndex* block = mi != ::mapBlockIndex.end() ? mi->second : nullptr;
     result.block_height = (block ? block->nHeight : std::numeric_limits<int>::max());
     result.blocks_to_maturity = wtx.GetBlocksToMaturity();
     result.depth_in_main_chain = wtx.GetDepthInMainChain();

src/keystore.h

@@ -15,6 +15,9 @@
 
 #include <boost/signals2/signal.hpp>
 
+#include <primitives/robin_hood.h>
+#include <set>
+
 /** A virtual base class for key stores */
 class CKeyStore : public SigningProvider
 {
@@ -44,9 +47,9 @@ class CBasicKeyStore : public CKeyStore
 protected:
     mutable CCriticalSection cs_KeyStore;
 
-    using KeyMap = std::map<CKeyID, CKey>;
+    using KeyMap = robin_hood::unordered_map<CKeyID, CKey>;
-    using WatchKeyMap = std::map<CKeyID, CPubKey>;
+    using WatchKeyMap = robin_hood::unordered_map<CKeyID, CPubKey>;
-    using ScriptMap = std::map<CScriptID, CScript>;
+    using ScriptMap = robin_hood::unordered_map<CScriptID, CScript>;
     using WatchOnlySet = std::set<CScript>;
 
     KeyMap mapKeys GUARDED_BY(cs_KeyStore);