LBRYCommunity/lighthouse.js
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Fixes for gendb, decoder and claimTrieCache gen. #25

Merged
filipnyquist merged 9 commits from master into master 2017-09-26 20:35:13 +02:00
Conversation 0 Commits 9 Files changed 12 +131 -35
12 changed files with 131 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
apidoc.json
View file

@ -3,8 +3,8 @@
"version": "0.0.1",
"description": "Lighthouse - Next-gen search api for LBRY",
"title": "Lighthouse API DOCS",
"url" : "http://localhost/api/lighthouse",
"sampleUrl": "http://localhost/api/lighthouse",
"url" : "http://localhost",
"sampleUrl": "http://localhost",
"json_body": true,
"template": {
"withCompare": true,

1
decoder/config.json
View file

@ -1 +0,0 @@
{"rpc_user": "lbry", "rpc_password": "lbry", "rpc_port": 9245, "rpc_url": "127.0.0.1"}

34
decoder/decoder.py
View file

@ -6,17 +6,25 @@ from lbryschema.decode import smart_decode
from flask import Flask, url_for
app = Flask(__name__)
def get_lbrycrdd_connection_details():
with open(os.path.dirname(os.path.realpath(__file__))+'/config.json', 'r') as f:
config = json.load(f)
rpc_user = config['rpc_user']
rpc_pass = config['rpc_password']
rpc_port = config['rpc_port']
rpc_url = config['rpc_url']
return 'http://%s:%s@%s:%i' % (rpc_user, rpc_pass, rpc_url,
rpc_port)
def get_lbrycrdd_connection_details(wallet_conf):
settings = {"username": "lbry",
"password": "lbry",
"rpc_port": 9245}
if wallet_conf and os.path.exists(wallet_conf):
with open(wallet_conf, "r") as conf:
conf_lines = conf.readlines()
for l in conf_lines:
if l.startswith("rpcuser="):
settings["username"] = l[8:].rstrip('\n')
if l.startswith("rpcpassword="):
settings["password"] = l[12:].rstrip('\n')
if l.startswith("rpcport="):
settings["rpc_port"] = int(l[8:].rstrip('\n'))
rpc_user = settings["username"]
rpc_pass = settings["password"]
rpc_port = settings["rpc_port"]
rpc_url = "127.0.0.1"
return "http://%s:%s@%s:%i" % (rpc_user, rpc_pass, rpc_url, rpc_port)
@app.errorhandler(500)
def internal_error(error):
@ -26,7 +34,7 @@ def internal_error(error):
@app.route('/claim_decode/<txid>/<nout>')
def api_decode(txid, nout):
connection_string = get_lbrycrdd_connection_details()
connection_string = get_lbrycrdd_connection_details(os.path.expanduser("~")+"/.lbrycrd/lbrycrd.conf")
rpc = AuthServiceProxy(connection_string)
result = rpc.getclaimsfortx(txid)
claim = None
@ -43,7 +51,7 @@ def api_decode(txid, nout):
@app.route('/claim_decodeinv/<claimid>')
def api_decodebyclaim(claimid):
connection_string = get_lbrycrdd_connection_details()
connection_string = get_lbrycrdd_connection_details(os.path.expanduser("~")+"/.lbrycrd/lbrycrd.conf")
rpc = AuthServiceProxy(connection_string)
claim = rpc.getvalueforname(claimid)
if claim:

10
gendb.sh
View file

@ -1,2 +1,10 @@
#!/bin/bash
curl -X PUT http://localhost:9200/claims -d '{ "settings" : { "number_of_shards" : 1 }, "mappings" : { "claim" : { "properties" : { "value" : { "type" : "nested" }, "suggest_name": { "type": "completion" }, "suggest_desc": { "type": "completion" } } } } }'
if [ "$(curl -IHEAD -w '%{http_code}' 'localhost:9200/claims' -o /dev/null --connect-timeout 3 --max-time 5)" == "200" ] ;
then
echo "Index already exists." ;
exit 1;
else
echo "Index did not exist, creating..." ;
curl -X PUT http://localhost:9200/claims -d '{ "settings" : { "number_of_shards" : 1 }, "mappings" : { "claim" : { "properties" : { "value" : { "type" : "nested" }, "suggest_name": { "type": "completion" }, "suggest_desc": { "type": "completion" } } } } }';
exit 0;
fi

3
package.json
View file

@ -39,6 +39,7 @@
"chalk": "^2.0.1",
"elastic-queue": "^0.3.0",
"elasticsearch": "^13.2.0",
"file-exists": "^4.0.0",
"glob": "^7.1.1",
"jsonfile": "^3.0.1",
"jsonwebtoken": "^7.2.1",
@ -50,6 +51,8 @@
"koa-router": "^7.0.0",
"limited-request-queue": "^3.0.4",
"ora": "^1.3.0",
"prettysize": "^1.1.0",
"properties-reader": "^0.0.16",
"request": "^2.81.0",
"request-promise": "^4.2.1",
"winston": "^2.3.1",

2
server/config.js
View file

@ -1,2 +1,2 @@
export const port = process.env.PORT || 80;
export const port = process.env.PORT || 50005;
export const baseApi = 'api';

19
server/controllers/lighthouse.js
View file

@ -3,6 +3,8 @@ import winston from 'winston';
import winstonStream from 'winston-stream';
import { sync, getStats } from '../utils/importer';
import elasticsearch from 'elasticsearch';
import rp from 'request-promise';
import pretty from 'prettysize';
const loggerStream = winstonStream(winston, 'info');
@ -71,6 +73,19 @@ function getAutoComplete (input) {
});
}
function getStatus () {
return new Promise((resolve, reject) => {
rp(`http://localhost:9200/claims/_stats`)
.then(function (data) {
data = JSON.parse(data);
resolve({status: getStats().info, spaceUsed: pretty(data._all.total.store.size_in_bytes, true), claimsInIndex: data._all.total.indexing.index_total, totSearches: data._all.total.search.query_total});
})
.catch(function (err) {
reject(err);
});
});
}
class LighthouseControllers {
/* eslint-disable no-param-reassign */
// Start syncing blocks...
@ -124,7 +139,7 @@ class LighthouseControllers {
* @param {ctx} Koa Context
*/
async info (ctx) {
ctx.body = 'Lighthouse';
ctx.body = 'Lighthouse search API';
}
/**
@ -132,7 +147,7 @@ class LighthouseControllers {
* @param {ctx} Koa Context
*/
async status (ctx) {
ctx.body = eclient.getStats();
ctx.body = await getStatus();
}
/* eslint-enable no-param-reassign */

5
server/package.json Normal file
View file

@ -0,0 +1,5 @@
{
"dependencies": {
"prettysize": "^1.1.0"
}
}

5
server/routes/lighthouse.js
View file

@ -1,16 +1,11 @@
import 'babel-polyfill';
import Router from 'koa-router';
import { baseApi } from '../config';
import LighthouseControllers from '../controllers/lighthouse';
LighthouseControllers.startSync();
const api = 'lighthouse';
const router = new Router();
router.prefix(`/${baseApi}/${api}`);
// GET /api/lighthouse
router.get('/', LighthouseControllers.info);

60
server/utils/importer/index.js
View file

@ -10,6 +10,10 @@ import jsonfile from 'jsonfile';
import path from 'path';
import rp from 'request-promise';
import appRoot from 'app-root-path';
import fs from 'fs';
import fileExists from 'file-exists';
import PropertiesReader from 'properties-reader';
import os from 'os';
const loggerStream = winstonStream(winston, 'info');
const eclient = new elasticsearch.Client({
@ -22,18 +26,55 @@ const eclient = new elasticsearch.Client({
},
});
const queue = new ElasticQueue({elastic: eclient});
const client = new bitcoin.Client({
host : 'localhost',
port : 9245,
user : 'lbry',
pass : 'lbry',
timeout: 30000,
//Get the lbrycrd config from the .lbrycrd folder.
function getClient () {
return new Promise((resolve, reject) => {
fileExists(path.join(os.homedir(), '.lbrycrd/lbrycrd.conf'), (err, exists) => {
if (err) { reject(err) };
let config = {'username': 'lbry', 'password': 'lbry', 'rpc_port': 9245};
if (exists) {
let prop = PropertiesReader(path.join(os.homedir(), '.lbrycrd/lbrycrd.conf'));
config.username = prop.get('rpcuser');
config.password = prop.get('rpcpassword');
config.rpc_port = prop.get('rpcport');
let client = new bitcoin.Client({
host : 'localhost',
port : config.rpc_port,
user : config.username,
pass : config.password,
timeout: 30000,
});
resolve(client);
} else {
let client = new bitcoin.Client({
host : 'localhost',
port : config.rpc_port,
user : config.username,
pass : config.password,
timeout: 30000,
});
resolve(client);
}
});
});
}
//Check that our cache file exist.
fileExists(path.join(appRoot.path, 'claimTrieCache.json'), (err, exists) => {
if (err) { throw err };
if (!exists) {
fs.writeFileSync(path.join(appRoot.path, 'claimTrieCache.json'), '[]');
}
});
let status = {};
export async function sync () {
try {
status.info = 'Grabbing the claimTrie...';
let client = await getClient();
status.info = 'gettingClaimTrie';
let claimTrie = await client.getClaimsInTrie().then(claimtrie => { return claimtrie }).catch(err => { throw err });
let txList = [];
let latestClaimTrie = [];
@ -47,10 +88,11 @@ export async function sync () {
latestClaimTrie.push(claimTrie[i].claims[o].claimId);
}
}
status.info = 'calculatingClaimTrie';
let oldClaimTrie = await getJSON(path.join(appRoot.path, 'claimTrieCache.json')); // get our old claimTrieCache....
let added = await getAddedClaims(oldClaimTrie, latestClaimTrie); // get all new that should be added
let removed = await getRemovedClaims(oldClaimTrie, latestClaimTrie); // get all old that should be removed
status.info = 'Adding/Removing Claims, please wait...';
status.info = 'updatingIndex';
for (let claimId of added) { // for all new get their tx info and add to database
let tx = txList.find(x => x.claimId === claimId);
if (typeof tx !== 'undefined') {
@ -82,7 +124,7 @@ export async function sync () {
}
// Done adding, update our claimTrie cache to latest and wait a bit...
await saveJSON(path.join(appRoot.path, 'claimTrieCache.json'), latestClaimTrie);
status.info = 'Done updating the claimTrieCache, waiting 5 minutes before doing a recheck..';
status.info = 'upToDate';
await sleep(300000);
sync();
} catch (err) {

7
server/yarn.lock Normal file
View file

@ -0,0 +1,7 @@
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
# yarn lockfile v1
prettysize@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/prettysize/-/prettysize-1.1.0.tgz#c6c52f87161ff172ea435f375f99831dd9a97bb0"

16
yarn.lock
View file

@ -1472,6 +1472,10 @@ file-entry-cache@^2.0.0:
flat-cache "^1.2.1"
object-assign "^4.0.1"
file-exists@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/file-exists/-/file-exists-4.0.0.tgz#104eacf25d3fd6b3e462951ae923533195ffb52b"
filename-regex@^2.0.0:
version "2.0.1"
resolved "https://registry.yarnpkg.com/filename-regex/-/filename-regex-2.0.1.tgz#c1c4b9bee3e09725ddb106b75c1e301fe2f18b26"
@ -2549,7 +2553,7 @@ minimist@^1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284"
mkdirp@0.5.1, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1:
mkdirp@0.5.1, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.1:
version "0.5.1"
resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903"
dependencies:
@ -2904,6 +2908,10 @@ preserve@^0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/preserve/-/preserve-0.2.0.tgz#815ed1f6ebc65926f865b310c0713bcb3315ce4b"
prettysize@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/prettysize/-/prettysize-1.1.0.tgz#c6c52f87161ff172ea435f375f99831dd9a97bb0"
private@^0.1.6, private@^0.1.7:
version "0.1.7"
resolved "https://registry.yarnpkg.com/private/-/private-0.1.7.tgz#68ce5e8a1ef0a23bb570cc28537b5332aba63ef1"
@ -2916,6 +2924,12 @@ progress@^1.1.8:
version "1.1.8"
resolved "https://registry.yarnpkg.com/progress/-/progress-1.1.8.tgz#e260c78f6161cdd9b0e56cc3e0a85de17c7a57be"
properties-reader@^0.0.16:
version "0.0.16"
resolved "https://registry.yarnpkg.com/properties-reader/-/properties-reader-0.0.16.tgz#a2cc0db9cdd7b81d977d0a59c1dc10c6d1d6ab98"
dependencies:
mkdirp "~0.5.1"
ps-tree@^1.0.1:
version "1.1.0"
resolved "https://registry.yarnpkg.com/ps-tree/-/ps-tree-1.1.0.tgz#b421b24140d6203f1ed3c76996b4427b08e8c014"