replace API with db connector

chainquery-config.json.example

@@ -0,0 +1,6 @@
+{
+  "host": "chainquery.lbry.com",
+  "user": "lighthouse",
+  "password": "",
+  "db": "chainquery"
+}

decoder/decoder.py

@@ -1,67 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-import json, os
-from bitcoinrpc.authproxy import AuthServiceProxy
-from lbryschema.decode import smart_decode
-from flask import Flask, url_for
-app = Flask(__name__)
-
-def get_lbrycrdd_connection_details(wallet_conf):
-    settings = {"username": "lbry",
-                "password": "lbry",
-                "rpc_port": 9245}
-    if wallet_conf and os.path.exists(wallet_conf):
-        with open(wallet_conf, "r") as conf:
-            conf_lines = conf.readlines()
-        for l in conf_lines:
-            if l.startswith("rpcuser="):
-                settings["username"] = l[8:].rstrip('\n')
-            if l.startswith("rpcpassword="):
-                settings["password"] = l[12:].rstrip('\n')
-            if l.startswith("rpcport="):
-                settings["rpc_port"] = int(l[8:].rstrip('\n'))
-    rpc_user = settings["username"]
-    rpc_pass = settings["password"]
-    rpc_port = settings["rpc_port"]
-    rpc_url = "127.0.0.1"
-    return "http://%s:%s@%s:%i" % (rpc_user, rpc_pass, rpc_url, rpc_port)
-
-@app.errorhandler(500)
-def internal_error(error):
-
-    return 'error when decoding claims'
-
-
-@app.route('/claim_decode/<txid>/<nout>')
-def api_decode(txid, nout):
-    connection_string = get_lbrycrdd_connection_details(os.path.expanduser("~")+"/.lbrycrd/lbrycrd.conf")
-    rpc = AuthServiceProxy(connection_string)
-    result = rpc.getclaimsfortx(txid)
-    claim = None
-    for claim_out in result:
-        if claim_out['nOut'] == int(nout):
-            claim = claim_out
-            break
-    if claim:
-        converted = ''.join([chr(ord(i)) for i in claim['value']])
-        decoded = smart_decode(converted)
-        claim['value'] = decoded.claim_dict
-        return json.dumps(claim)
-
-
-@app.route('/claim_decodeinv/<claimid>')
-def api_decodebyclaim(claimid):
-    connection_string = get_lbrycrdd_connection_details(os.path.expanduser("~")+"/.lbrycrd/lbrycrd.conf")
-    rpc = AuthServiceProxy(connection_string)
-    claim = rpc.getvalueforname(claimid)
-    if claim:
-        converted = ''.join([chr(ord(i)) for i in claim['value']])
-        decoded = smart_decode(converted)
-        claim['value'] = decoded.claim_dict
-        return json.dumps(claim)
-
-if __name__ == '__main__':
-    app.run(host='127.0.0.1')
-
-
-			

decoder/requirements.txt

@@ -1,3 +0,0 @@
-git+https://github.com/lbryio/lbryschema.git#egg=lbryschema
-python-bitcoinrpc==0.1
-flask

package-lock.json

@@ -2814,6 +2814,11 @@
       "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz",
       "integrity": "sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ=="
     },
+    "bignumber.js": {
+      "version": "7.2.1",
+      "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz",
+      "integrity": "sha512-S4XzBk5sMB+Rcb/LNcpzXr57VRTxgAvaAEDAl1AwRx27j00hT84O6OkteE7u8UB3NuaaygCRrEpqox4uDOrbdQ=="
+    },
     "binary-extensions": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.0.0.tgz",
@@ -7122,6 +7127,24 @@
       "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.6.tgz",
       "integrity": "sha1-SJYrGeFp/R38JAs/HnMXYnu8R9s="
     },
+    "mysql": {
+      "version": "2.17.1",
+      "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.17.1.tgz",
+      "integrity": "sha512-7vMqHQ673SAk5C8fOzTG2LpPcf3bNt0oL3sFpxPEEFp1mdlDcrLK0On7z8ZYKaaHrHwNcQ/MTUz7/oobZ2OyyA==",
+      "requires": {
+        "bignumber.js": "7.2.1",
+        "readable-stream": "2.3.6",
+        "safe-buffer": "5.1.2",
+        "sqlstring": "2.3.1"
+      },
+      "dependencies": {
+        "safe-buffer": {
+          "version": "5.1.2",
+          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+          "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
+        }
+      }
+    },
     "nan": {
       "version": "2.14.0",
       "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz",
@@ -10984,6 +11007,11 @@
       "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
       "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw="
     },
+    "sqlstring": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz",
+      "integrity": "sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A="
+    },
     "sshpk": {
       "version": "1.16.1",
       "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz",

package.json

@@ -56,6 +56,7 @@
     "koa-logger": "^2.0.0",
     "koa-router": "^7.0.0",
     "limited-request-queue": "^3.0.4",
+    "mysql": "^2.17.1",
     "node-slack": "^0.0.7",
     "oas": "^0.8.15",
     "ora": "^1.3.0",

server/utils/chainquery/index.js

@@ -13,6 +13,7 @@ import fs from 'fs';
 import fileExists from 'file-exists';
 import * as util from './util';
 import {logErrorToSlack} from '../../index';
+import mysql from 'mysql';
 
 const elasticsearchloglevel = 'info';
 const MaxClaimsToProcessPerIteration = 100000;
@@ -31,7 +32,9 @@ const queue = new ElasticQueue({elastic: eclient});
 
 // Check that our syncState file exist.
 fileExists(path.join(appRoot.path, 'syncState.json'), (err, exists) => {
-  if (err) { throw err }
+  if (err) {
+    throw err;
+  }
   if (!exists) {
     fs.writeFileSync(path.join(appRoot.path, 'syncState.json'), '{}');
   }
@@ -153,6 +156,7 @@ function getJSON (path) {
     });
   });
 }
+
 function saveJSON (path, obj) {
   return new Promise((resolve, reject) => {
     jsonfile.writeFile(path, obj, function (err, jsoncontent) {
@@ -183,34 +187,30 @@ function getBlockedOutpoints () {
   });
 }
 
+let connection = null;
+const chainqueryConfig = require('../../../chainquery-config.json');
+
 function getClaimsSince (time, lastID, MaxClaimsInCall) {
+  if (connection === null) {
+    connection = mysql.createConnection({
+      host    : chainqueryConfig.host,
+      user    : chainqueryConfig.user,
+      password: chainqueryConfig.password,
+      database: chainqueryConfig.db,
+    });
+    connection.connect();
+  }
+
   return new Promise((resolve, reject) => {
-    let query =  `` +
+    let query = `SELECT c.id, c.name,p.name as channel, p.claim_id as channel_id, c.bid_state,c.effective_amount,COALESCE(p.effective_amount,1) as certificate_amount,c.claim_id as claimId,c.value_as_json as value FROM claim c LEFT JOIN claim p on p.claim_id = c.publisher_id WHERE c.id >${lastID} AND c.modified_at >='${time}' ORDER BY c.id LIMIT ${MaxClaimsInCall}`;
-      `SELECT ` +
-      `c.id, ` +
-      `c.name,` +
-      `p.name as channel,` +
-      `p.claim_id as channel_id,` +
-      `c.bid_state,` +
-      `c.effective_amount,` +
-      `COALESCE(p.effective_amount,1) as certificate_amount,` +
-      `c.claim_id as claimId,` +
-      `c.value_as_json as value ` +
-      `FROM claim c ` +
-      `LEFT JOIN claim p on p.claim_id = c.publisher_id ` +
-      `WHERE c.id >` + lastID + ` ` +
-      `AND c.modified_at >='` + time + `' ` +
-      `ORDER BY c.id ` +
-      `LIMIT ` + MaxClaimsInCall;
     // Outputs full query to console for copy/paste into chainquery (debugging)
     console.log(query);
-    rp(`https://chainquery.lbry.com/api/sql?query=` + query)
+    connection.query(query, function (err, results, fields) {
-      .then(function (htmlString) {
+      if (err) {
-        resolve(htmlString);
-      })
-      .catch(function (err) {
         logErrorToSlack('[Importer] Error getting updated claims. ' + err);
-        reject(err);
+        return reject(err);
-      });
+      }
+      resolve(results);
+    });
   });
 }