LBRYCommunity/lighthouse.js
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #25 from filipnyquist/master

Browse source 
This closes #21, closes #22, closes #23 and moves the api to the base path.
Created the first status API which gives some useful information and does also set the standard port to 5005 unless specified.
This commit is contained in:
filipnyquist 2017-09-26 20:35:12 +02:00 • committed by GitHub
commit 829930ea1e
12 changed files with 131 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
apidoc.json
View file

@ -3,8 +3,8 @@
"version": "0.0.1",
"description": "Lighthouse - Next-gen search api for LBRY",
"title": "Lighthouse API DOCS",
"url" : "http://localhost/api/lighthouse",
"sampleUrl": "http://localhost/api/lighthouse",
"url" : "http://localhost",
"sampleUrl": "http://localhost",
"json_body": true,
"template": {
"withCompare": true,

1
decoder/config.json
View file

@ -1 +0,0 @@
{"rpc_user": "lbry", "rpc_password": "lbry", "rpc_port": 9245, "rpc_url": "127.0.0.1"}

34
decoder/decoder.py
View file

@ -6,17 +6,25 @@ from lbryschema.decode import smart_decode
from flask import Flask, url_for
app = Flask(__name__)
def get_lbrycrdd_connection_details():
with open(os.path.dirname(os.path.realpath(__file__))+'/config.json', 'r') as f:
config = json.load(f)
rpc_user = config['rpc_user']
rpc_pass = config['rpc_password']
rpc_port = config['rpc_port']
rpc_url = config['rpc_url']
return 'http://%s:%s@%s:%i' % (rpc_user, rpc_pass, rpc_url,
rpc_port)
def get_lbrycrdd_connection_details(wallet_conf):
settings = {"username": "lbry",
"password": "lbry",
"rpc_port": 9245}
if wallet_conf and os.path.exists(wallet_conf):
with open(wallet_conf, "r") as conf:
conf_lines = conf.readlines()
for l in conf_lines:
if l.startswith("rpcuser="):
settings["username"] = l[8:].rstrip('\n')
if l.startswith("rpcpassword="):
settings["password"] = l[12:].rstrip('\n')
if l.startswith("rpcport="):
settings["rpc_port"] = int(l[8:].rstrip('\n'))
rpc_user = settings["username"]
rpc_pass = settings["password"]
rpc_port = settings["rpc_port"]
rpc_url = "127.0.0.1"
return "http://%s:%s@%s:%i" % (rpc_user, rpc_pass, rpc_url, rpc_port)
@app.errorhandler(500)
def internal_error(error):
@ -26,7 +34,7 @@ def internal_error(error):
@app.route('/claim_decode/<txid>/<nout>')
def api_decode(txid, nout):
connection_string = get_lbrycrdd_connection_details()
connection_string = get_lbrycrdd_connection_details(os.path.expanduser("~")+"/.lbrycrd/lbrycrd.conf")
rpc = AuthServiceProxy(connection_string)
result = rpc.getclaimsfortx(txid)
claim = None
@ -43,7 +51,7 @@ def api_decode(txid, nout):
@app.route('/claim_decodeinv/<claimid>')
def api_decodebyclaim(claimid):
connection_string = get_lbrycrdd_connection_details()
connection_string = get_lbrycrdd_connection_details(os.path.expanduser("~")+"/.lbrycrd/lbrycrd.conf")
rpc = AuthServiceProxy(connection_string)
claim = rpc.getvalueforname(claimid)
if claim:

10
gendb.sh
View file

@ -1,2 +1,10 @@
#!/bin/bash
curl -X PUT http://localhost:9200/claims -d '{ "settings" : { "number_of_shards" : 1 }, "mappings" : { "claim" : { "properties" : { "value" : { "type" : "nested" }, "suggest_name": { "type": "completion" }, "suggest_desc": { "type": "completion" } } } } }'
if [ "$(curl -IHEAD -w '%{http_code}' 'localhost:9200/claims' -o /dev/null --connect-timeout 3 --max-time 5)" == "200" ] ;
then
echo "Index already exists." ;
exit 1;
else
echo "Index did not exist, creating..." ;
curl -X PUT http://localhost:9200/claims -d '{ "settings" : { "number_of_shards" : 1 }, "mappings" : { "claim" : { "properties" : { "value" : { "type" : "nested" }, "suggest_name": { "type": "completion" }, "suggest_desc": { "type": "completion" } } } } }';
exit 0;
fi

3
package.json
View file

@ -39,6 +39,7 @@
"chalk": "^2.0.1",
"elastic-queue": "^0.3.0",
"elasticsearch": "^13.2.0",
"file-exists": "^4.0.0",
"glob": "^7.1.1",
"jsonfile": "^3.0.1",
"jsonwebtoken": "^7.2.1",
@ -50,6 +51,8 @@
"koa-router": "^7.0.0",
"limited-request-queue": "^3.0.4",
"ora": "^1.3.0",
"prettysize": "^1.1.0",
"properties-reader": "^0.0.16",
"request": "^2.81.0",
"request-promise": "^4.2.1",
"winston": "^2.3.1",

2
server/config.js
View file

@ -1,2 +1,2 @@
export const port = process.env.PORT || 80;
export const port = process.env.PORT || 50005;
export const baseApi = 'api';

19
server/controllers/lighthouse.js
View file

@ -3,6 +3,8 @@ import winston from 'winston';
import winstonStream from 'winston-stream';
import { sync, getStats } from '../utils/importer';
import elasticsearch from 'elasticsearch';
import rp from 'request-promise';
import pretty from 'prettysize';
const loggerStream = winstonStream(winston, 'info');
@ -71,6 +73,19 @@ function getAutoComplete (input) {
});
}
function getStatus () {
return new Promise((resolve, reject) => {
rp(`http://localhost:9200/claims/_stats`)
.then(function (data) {
data = JSON.parse(data);
resolve({status: getStats().info, spaceUsed: pretty(data._all.total.store.size_in_bytes, true), claimsInIndex: data._all.total.indexing.index_total, totSearches: data._all.total.search.query_total});
})
.catch(function (err) {
reject(err);
});
});
}
class LighthouseControllers {
/* eslint-disable no-param-reassign */
// Start syncing blocks...
@ -124,7 +139,7 @@ class LighthouseControllers {
* @param {ctx} Koa Context
*/
async info (ctx) {
ctx.body = 'Lighthouse';
ctx.body = 'Lighthouse search API';
}
/**
@ -132,7 +147,7 @@ class LighthouseControllers {
* @param {ctx} Koa Context
*/
async status (ctx) {
ctx.body = eclient.getStats();
ctx.body = await getStatus();
}
/* eslint-enable no-param-reassign */

5
server/package.json Normal file
View file

@ -0,0 +1,5 @@
{
"dependencies": {
"prettysize": "^1.1.0"
}
}

5
server/routes/lighthouse.js
View file

@ -1,16 +1,11 @@
import 'babel-polyfill';
import Router from 'koa-router';
import { baseApi } from '../config';
import LighthouseControllers from '../controllers/lighthouse';
LighthouseControllers.startSync();
const api = 'lighthouse';
const router = new Router();
router.prefix(`/${baseApi}/${api}`);
// GET /api/lighthouse
router.get('/', LighthouseControllers.info);

60
server/utils/importer/index.js
View file

@ -10,6 +10,10 @@ import jsonfile from 'jsonfile';
import path from 'path';
import rp from 'request-promise';
import appRoot from 'app-root-path';
import fs from 'fs';
import fileExists from 'file-exists';
import PropertiesReader from 'properties-reader';
import os from 'os';
const loggerStream = winstonStream(winston, 'info');
const eclient = new elasticsearch.Client({
@ -22,18 +26,55 @@ const eclient = new elasticsearch.Client({
},
});
const queue = new ElasticQueue({elastic: eclient});
const client = new bitcoin.Client({
host : 'localhost',
port : 9245,
user : 'lbry',
pass : 'lbry',
timeout: 30000,
//Get the lbrycrd config from the .lbrycrd folder.
function getClient () {
return new Promise((resolve, reject) => {
fileExists(path.join(os.homedir(), '.lbrycrd/lbrycrd.conf'), (err, exists) => {
if (err) { reject(err) };
let config = {'username': 'lbry', 'password': 'lbry', 'rpc_port': 9245};
if (exists) {
let prop = PropertiesReader(path.join(os.homedir(), '.lbrycrd/lbrycrd.conf'));
config.username = prop.get('rpcuser');
config.password = prop.get('rpcpassword');
config.rpc_port = prop.get('rpcport');
let client = new bitcoin.Client({
host : 'localhost',
port : config.rpc_port,
user : config.username,
pass : config.password,
timeout: 30000,
});
resolve(client);
} else {
let client = new bitcoin.Client({
host : 'localhost',
port : config.rpc_port,
user : config.username,
pass : config.password,
timeout: 30000,
});
resolve(client);
}
});
});
}
//Check that our cache file exist.
fileExists(path.join(appRoot.path, 'claimTrieCache.json'), (err, exists) => {
if (err) { throw err };
if (!exists) {
fs.writeFileSync(path.join(appRoot.path, 'claimTrieCache.json'), '[]');
}
});
let status = {};
export async function sync () {
try {
status.info = 'Grabbing the claimTrie...';
let client = await getClient();
status.info = 'gettingClaimTrie';
let claimTrie = await client.getClaimsInTrie().then(claimtrie => { return claimtrie }).catch(err => { throw err });
let txList = [];
let latestClaimTrie = [];
@ -47,10 +88,11 @@ export async function sync () {
latestClaimTrie.push(claimTrie[i].claims[o].claimId);
}
}
status.info = 'calculatingClaimTrie';
let oldClaimTrie = await getJSON(path.join(appRoot.path, 'claimTrieCache.json')); // get our old claimTrieCache....
let added = await getAddedClaims(oldClaimTrie, latestClaimTrie); // get all new that should be added
let removed = await getRemovedClaims(oldClaimTrie, latestClaimTrie); // get all old that should be removed
status.info = 'Adding/Removing Claims, please wait...';
status.info = 'updatingIndex';
for (let claimId of added) { // for all new get their tx info and add to database
let tx = txList.find(x => x.claimId === claimId);
if (typeof tx !== 'undefined') {
@ -82,7 +124,7 @@ export async function sync () {
}
// Done adding, update our claimTrie cache to latest and wait a bit...
await saveJSON(path.join(appRoot.path, 'claimTrieCache.json'), latestClaimTrie);
status.info = 'Done updating the claimTrieCache, waiting 5 minutes before doing a recheck..';
status.info = 'upToDate';
await sleep(300000);
sync();
} catch (err) {

7
server/yarn.lock Normal file
View file

@ -0,0 +1,7 @@
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
# yarn lockfile v1
prettysize@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/prettysize/-/prettysize-1.1.0.tgz#c6c52f87161ff172ea435f375f99831dd9a97bb0"

16
yarn.lock
View file

@ -1472,6 +1472,10 @@ file-entry-cache@^2.0.0:
flat-cache "^1.2.1"
object-assign "^4.0.1"
file-exists@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/file-exists/-/file-exists-4.0.0.tgz#104eacf25d3fd6b3e462951ae923533195ffb52b"
filename-regex@^2.0.0:
version "2.0.1"
resolved "https://registry.yarnpkg.com/filename-regex/-/filename-regex-2.0.1.tgz#c1c4b9bee3e09725ddb106b75c1e301fe2f18b26"
@ -2549,7 +2553,7 @@ minimist@^1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284"
mkdirp@0.5.1, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1:
mkdirp@0.5.1, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.1:
version "0.5.1"
resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903"
dependencies:
@ -2904,6 +2908,10 @@ preserve@^0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/preserve/-/preserve-0.2.0.tgz#815ed1f6ebc65926f865b310c0713bcb3315ce4b"
prettysize@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/prettysize/-/prettysize-1.1.0.tgz#c6c52f87161ff172ea435f375f99831dd9a97bb0"
private@^0.1.6, private@^0.1.7:
version "0.1.7"
resolved "https://registry.yarnpkg.com/private/-/private-0.1.7.tgz#68ce5e8a1ef0a23bb570cc28537b5332aba63ef1"
@ -2916,6 +2924,12 @@ progress@^1.1.8:
version "1.1.8"
resolved "https://registry.yarnpkg.com/progress/-/progress-1.1.8.tgz#e260c78f6161cdd9b0e56cc3e0a85de17c7a57be"
properties-reader@^0.0.16:
version "0.0.16"
resolved "https://registry.yarnpkg.com/properties-reader/-/properties-reader-0.0.16.tgz#a2cc0db9cdd7b81d977d0a59c1dc10c6d1d6ab98"
dependencies:
mkdirp "~0.5.1"
ps-tree@^1.0.1:
version "1.1.0"
resolved "https://registry.yarnpkg.com/ps-tree/-/ps-tree-1.1.0.tgz#b421b24140d6203f1ed3c76996b4427b08e8c014"