Added synctoolv2, claimTrie resolving instead of full chain!

Added synctoolv2, claimTrie resolving instead of full chain and some small patches!

claimTrieCache.json

@@ -0,0 +1 @@
+[]

package.json

@@ -23,7 +23,7 @@
   "homepage": "https://github.com/lbryio/lighthouse#readme",
   "main": "server/index.js",
   "scripts": {
-    "start": "nodemon server/ --exec babel-node",
+    "start": "nodemon server/ --exec babel-node --ignore 'claimTrieCache.json'",
     "build": "babel server -d dist",
     "lint": "eslint ./server",
     "test": "npm run lint && npm run mocha",
@@ -31,10 +31,12 @@
     "mocha": "./node_modules/.bin/mocha --compilers js:babel-register --require babel-polyfill"
   },
   "dependencies": {
+    "app-root-path": "^2.0.1",
     "babel-polyfill": "^6.5.0",
-    "bitcoin-promise": "filipnyquist/node-bitcoin-promise#c3eb4bea552a7a136a4a1405d831da3e92f2efea",
+    "bitcoin-promise": "filipnyquist/node-bitcoin-promise#1fbf1cb8913ca3542b66060d48ebea185661e0a7",
     "bluebird": "^3.5.0",
     "chalk": "^2.0.1",
+    "elastic-queue": "^0.3.0",
     "elasticsearch": "^13.2.0",
     "glob": "^7.1.1",
     "jsonfile": "^3.0.1",

server/controllers/lighthouse.js

@@ -34,7 +34,7 @@ class LighthouseControllers {
   // Start syncing blocks...
   startSync () {
     winston.log('info', '[Importer] Started importer, indexing claims.');
-    sync(200000);
+    sync();
   }
   /**
    * Search api here

server/utils/importer/getClaims.js

@@ -1,60 +0,0 @@
-'use strict';
-
-const Promise = require('bluebird');
-const rp = require('request-promise');
-
-let client;
-
-async function getClaims (height, gclient) {
-  return new Promise(async (resolve, reject) => {
-    try {
-      client = gclient;
-      let blockHash = await client.getBlockHash(height).then(blockHash => { return blockHash }).catch(err => reject(err));
-      let block = await client.getBlock(blockHash).then(block => { return block }).catch(err => reject(err));
-      let claims = await getClaimsForTxes(block.tx, height); // should return an array of claims, decoded if possible.
-      resolve(claims);
-    } catch (err) {
-      return reject(err);
-    }
-  });
-}
-async function getClaimsForTxes (txes, height) {
-  return new Promise(async (resolve, reject) => {
-    try {
-      let claimsArr = [];
-      for (let tx of txes) {
-        let claimsTx = await client.getClaimsForTx(tx).then(claims => { return claims }).catch(err => reject(err));
-        if (claimsTx != null) {
-          for (let claim of claimsTx) {
-            claim['height'] = height;
-            let dClaim = await getValue(tx, claim['nOut']);
-            if (dClaim !== 'error when decoding claims' && claim['value']) {
-              claim['value'] = JSON.parse(dClaim);
-              claimsArr.push(claim);
-            } else {
-              claim['value'] = { error: 'non_decodable' };
-              claimsArr.push(claim);
-            }
-          }
-        }
-      }
-      resolve(claimsArr);
-    } catch (err) {
-      return reject(err);
-    }
-  });
-}
-
-async function getValue (tx, i) {
-  return new Promise(async (resolve, reject) => {
-    rp(`http://localhost:5000/claim_decode/${tx}/${i}`)
-    .then(function (htmlString) {
-      resolve(htmlString);
-    })
-    .catch(function (err) {
-      reject(err);
-    });
-  });
-}
-
-module.exports = exports = getClaims;

server/utils/importer/index.js

@@ -1,13 +1,17 @@
 /*
  *  Importer code, handles all the syncing with the blockchain into elasticsearch.
  */
-const bitcoin = require('bitcoin-promise');
+import bitcoin from 'bitcoin-promise';
-const elasticsearch = require('elasticsearch');
+import elasticsearch from 'elasticsearch';
-const winston = require('winston');
+import ElasticQueue from 'elastic-queue';
-const winstonStream = require('winston-stream');
+import winston from 'winston';
+import winstonStream from 'winston-stream';
+import jsonfile from 'jsonfile';
+import path from 'path';
+import rp from 'request-promise';
+import appRoot from 'app-root-path';
 
 const loggerStream = winstonStream(winston, 'info');
-
 const eclient = new elasticsearch.Client({
   host: 'http://elastic:changeme@localhost:9200',
   log : {
@@ -16,7 +20,7 @@ const eclient = new elasticsearch.Client({
     stream: loggerStream,
   },
 });
-
+const queue = new ElasticQueue({elastic: eclient});
 const client = new bitcoin.Client({
   host   : 'localhost',
   port   : 9245,
@@ -24,26 +28,64 @@ const client = new bitcoin.Client({
   pass   : 'lbry',
   timeout: 30000,
 });
-let claimsSynced = 0;
 let status = {};
 
-export async function sync (currentHeight) {
+export async function sync () {
   try {
-    let maxHeight = await client.getBlockCount().then(blockHash => { return blockHash }).catch(err => { throw err });
+    status.info = 'Grabbing the claimTrie...';
-    if (currentHeight <= maxHeight) {
+    let claimTrie = await client.getClaimsInTrie().then(claimtrie => { return claimtrie }).catch(err => { throw err });
-      let claims = await require('./getClaims')(currentHeight, client);
+    let txList = [];
-      send(claims);
+    let latestClaimTrie = [];
-      claimsSynced += claims.length;
+    for (let i in claimTrie) {
-      // currentHeight / maxHeight / claimsSynced
+      for (let o in claimTrie[i].claims) {
-      status.message = `Running,${currentHeight} / ${maxHeight} done, ${claimsSynced} claims imported.`;
+        txList.push({
-      sync(currentHeight + 1);
+          txid   : claimTrie[i].claims[o].txid,
-    } else {
+          nOut   : claimTrie[i].claims[o].n,
-      await sleep(2000);
+          claimId: claimTrie[i].claims[o].claimId,
-      status.message = `All claims imported, waiting for new blocks at ${maxHeight}`;
+        });
-      sync(currentHeight); // eslint-disable-line no-unreachable
+        latestClaimTrie.push(claimTrie[i].claims[o].claimId);
+      }
     }
+    let oldClaimTrie = await getJSON(path.join(appRoot.path, 'claimTrieCache.json')); // get our old claimTrieCache....
+    let added = await getAddedClaims(oldClaimTrie, latestClaimTrie); // get all new that should be added
+    let removed = await getRemovedClaims(oldClaimTrie, latestClaimTrie); // get all old that should be removed
+    status.info = 'Adding/Removing Claims, please wait...';
+    for (let claimId of added) { // for all new get their tx info and add to database
+      let tx = txList.find(x => x.claimId === claimId);
+      if (typeof tx !== 'undefined') {
+        let value = await getValue(tx.txid, tx.nOut);
+        if ((value !== 'error when decoding claims')) {
+          value = JSON.parse(value);
+          if (value['is controlling'] && value['name'] !== '') {
+            if (value.name && value.value) {
+              value.suggest_name = {
+                input : value.name,
+                weight: 30,
+              };
+            }
+            pushElastic(value);
+          }
+        }
+      }
+    }
+    for (let claimId of removed) { // Call elastic and remove claim by id if it exists.
+      eclient.delete({
+        index: 'claims',
+        type : 'claim',
+        id   : claimId,
+      }, function (error, response) {
+        if (error) {
+          winston.log(error);
+        }
+      });
+    }
+    // Done adding, update our claimTrie cache to latest and wait a bit...
+    await saveJSON(path.join(appRoot.path, 'claimTrieCache.json'), latestClaimTrie);
+    status.info = 'Done updating the claimTrieCache, waiting 5 minutes before doing a recheck..';
+    await sleep(300000);
+    sync();
   } catch (err) {
-    console.log(err);
+    winston.log(err);
     status.err = err;
   }
 }
@@ -52,31 +94,64 @@ export function getStats () {
   return status;
 }
 
-function send (arr) { // Modular change output here :)
+function getAddedClaims (oldClaimTrie, newClaimTrie) {
-  arr.forEach(function (claim) {
+  return new Promise((resolve, reject) => {
-    claim['id'] = claim['claimId'];
+    let a = new Set(oldClaimTrie);
-    // Check if our value is a object, else make it a object...
+    let b = new Set(newClaimTrie);
-    claim['value'] = (typeof claim.value === 'object' ? claim.value : JSON.parse(claim.value));
+    resolve(new Set([...b].filter(x => !a.has(x))));
-    // claim['value'] = JSON.stringify(claim['value']);
+  });
-    if (claim.name && claim.value) {
+}
-      claim.suggest_name = {
+
-        input : claim.name,
+function getRemovedClaims (oldClaimTrie, newClaimTrie) {
-        weight: 30,
+  return new Promise((resolve, reject) => {
-      };
+    let a = new Set(oldClaimTrie);
-      if (claim.value.claimType === 'streamType' && claim.value.stream.metadata && claim.value.stream.metadata.description) {
+    let b = new Set(newClaimTrie);
-        claim.suggest_desc = {
+    resolve(new Set([...a].filter(x => !b.has(x))));
-          input : claim.value.stream.metadata.description.split(' '),
+  });
-          weight: 10,
+}
-        };
+
-      }
+function getValue (tx, i) {
-    }
+  return new Promise((resolve, reject) => {
-    eclient.create({
+    rp(`http://localhost:5000/claim_decode/${tx}/${i}`)
+      .then(function (htmlString) {
+        resolve(htmlString);
+      })
+      .catch(function (err) {
+        reject(err);
+      });
+  });
+}
+
+async function pushElastic (claim) {
+  return new Promise(async(resolve, reject) => {
+    queue.push({
       index: 'claims',
       type : 'claim',
       id   : claim.claimId,
       body : claim,
-    }, function (error, response) {
+    });
-      if (error) { status.err = error; console.log(error) }
+  });
+}
+
+function getJSON (path) {
+  return new Promise((resolve, reject) => {
+    jsonfile.readFile(path, function (err, jsoncontent) {
+      if (err) {
+        reject(err);
+      } else {
+        resolve(jsoncontent);
+      }
+    });
+  });
+}
+function saveJSON (path, obj) {
+  return new Promise((resolve, reject) => {
+    jsonfile.writeFile(path, obj, function (err, jsoncontent) {
+      if (err) {
+        reject(err);
+      } else {
+        resolve();
+      }
     });
   });
 }

yarn.lock

@@ -75,6 +75,10 @@ anymatch@^1.3.0:
     micromatch "^2.1.5"
     normalize-path "^2.0.0"
 
+app-root-path@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/app-root-path/-/app-root-path-2.0.1.tgz#cd62dcf8e4fd5a417efc664d2e5b10653c651b46"
+
 aproba@^1.0.3:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.1.2.tgz#45c6629094de4e96f693ef7eab74ae079c240fc1"
@@ -140,7 +144,7 @@ async-each@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/async-each/-/async-each-1.0.1.tgz#19d386a1d9edc6e7c1c85d388aedbcc56d33602d"
 
-async@^1.5.2:
+"async@>=0.9 <2.0", async@^1.5.2:
   version "1.5.2"
   resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a"
 
@@ -678,15 +682,15 @@ binary-extensions@^1.0.0:
   version "1.10.0"
   resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-1.10.0.tgz#9aeb9a6c5e88638aad171e167f5900abe24835d0"
 
-bitcoin-promise@filipnyquist/node-bitcoin-promise#c3eb4bea552a7a136a4a1405d831da3e92f2efea:
+bitcoin-promise@filipnyquist/node-bitcoin-promise#1fbf1cb8913ca3542b66060d48ebea185661e0a7:
   version "1.3.2"
-  resolved "https://codeload.github.com/filipnyquist/node-bitcoin-promise/tar.gz/c3eb4bea552a7a136a4a1405d831da3e92f2efea"
+  resolved "https://codeload.github.com/filipnyquist/node-bitcoin-promise/tar.gz/1fbf1cb8913ca3542b66060d48ebea185661e0a7"
   dependencies:
-    bitcoin filipnyquist/node-bitcoin
+    bitcoin filipnyquist/node-bitcoin#17552efad852a0ae929dc153988649259536a23d
 
-bitcoin@filipnyquist/node-bitcoin:
+bitcoin@filipnyquist/node-bitcoin#17552efad852a0ae929dc153988649259536a23d:
   version "3.0.2"
-  resolved "https://codeload.github.com/filipnyquist/node-bitcoin/tar.gz/6e39c26a6b462ad315832bfc8f9e6411e1d8e1aa"
+  resolved "https://codeload.github.com/filipnyquist/node-bitcoin/tar.gz/17552efad852a0ae929dc153988649259536a23d"
 
 block-stream@*:
   version "0.0.9"
@@ -1125,7 +1129,15 @@ ee-first@1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d"
 
-elasticsearch@^13.2.0:
+elastic-queue@^0.3.0:
+  version "0.3.0"
+  resolved "https://registry.yarnpkg.com/elastic-queue/-/elastic-queue-0.3.0.tgz#36fbd56543a5e1c10fff118e09f69293d41ac2fd"
+  dependencies:
+    async ">=0.9 <2.0"
+    elasticsearch ">4.0"
+    hoek "^2.11.1"
+
+elasticsearch@>4.0, elasticsearch@^13.2.0:
   version "13.3.1"
   resolved "https://registry.yarnpkg.com/elasticsearch/-/elasticsearch-13.3.1.tgz#c530aea9afb17ea91c3d0a56f1f111ba49bc9239"
   dependencies:
@@ -1729,7 +1741,7 @@ hide-powered-by@1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/hide-powered-by/-/hide-powered-by-1.0.0.tgz#4a85ad65881f62857fc70af7174a1184dccce32b"
 
-hoek@2.x.x:
+hoek@2.x.x, hoek@^2.11.1:
   version "2.16.3"
   resolved "https://registry.yarnpkg.com/hoek/-/hoek-2.16.3.tgz#20bb7403d3cea398e91dc4710a8ff1b8274a25ed"