LBRYCommunity/lighthouse.js
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

add timestamp index

Browse source
This commit is contained in:
Niko Storni 2019-12-02 17:49:58 +01:00
parent 5c68e32abb
commit 08068a2a47
3 changed files with 65 additions and 19 deletions
gendb.sh
server
controllers
lighthouse.js
utils/chainquery
index.js

2
gendb.sh
View file

@ -5,6 +5,6 @@ then
exit 1; exit 1;
else else
echo "Index did not exist, creating..." ; echo "Index did not exist, creating..." ;
curl -H 'Content-Type: application/json' -H 'Accept: application/json' -X PUT -d '{ "settings" : { "number_of_shards" : 1 }, "mappings" : { "claim" : { "properties" : { "value" : { "type" : "nested" }, "suggest_name": { "type": "completion" }, "suggest_desc": { "type": "completion" } } } } }' http://localhost:9200/claims; curl -H 'Content-Type: application/json' -H 'Accept: application/json' -X PUT -d '{ "settings" : { "number_of_shards" : 1 }, "mappings" : { "claim" : { "properties" : { "value" : { "type" : "nested" }, "suggest_name": { "type": "completion" }, "suggest_desc": { "type": "completion" }, "transaction_time": { "type": "date" } } } } }' http://localhost:9200/claims;
exit 0; exit 0;
fi fi

76
server/controllers/lighthouse.js
View file

@ -80,6 +80,32 @@ function getResults (input) {
}, },
}, },
}; };
const newerBoost = [ // New claims should come up further up
{
'range': {
'transaction_time': {
'boost': 5,
'gte' : new Date().setDate(Date.now() - 30),
},
},
},
{
'range': {
'transaction_time': {
'boost': 4,
'gte' : new Date().setDate(Date.now() - 60),
},
},
},
{
'range': {
'transaction_time': {
'boost': 3,
'gte' : new Date().setDate(Date.now() - 90),
},
},
},
];
const funcScoreClaimWeight = { // 100 LBC adds 1 point to the score const funcScoreClaimWeight = { // 100 LBC adds 1 point to the score
'function_score': { 'function_score': {
'field_value_factor': { 'field_value_factor': {
@ -112,7 +138,9 @@ function getResults (input) {
let conCatTerm = ''; let conCatTerm = '';
let phraseTerm = ''; let phraseTerm = '';
escapedQuery.split(' ').every((term, index) => { escapedQuery.split(' ').every((term, index) => {
if (index === 4) { return false } if (index === 4) {
return false;
}
phraseTerm = phraseTerm + ' ' + term; phraseTerm = phraseTerm + ' ' + term;
conCatTerm = conCatTerm + term; conCatTerm = conCatTerm + term;
queries.push( queries.push(
@ -144,10 +172,10 @@ function getResults (input) {
}, },
}, },
{ {
'prefix': { 'name': { 'value': '@' + escapedQuery, 'boost': 10 } }, 'prefix': {'name': {'value': '@' + escapedQuery, 'boost': 10}},
}, },
{ {
'prefix': { 'name': { 'value': escapedQuery, 'boost': 10 } }, 'prefix': {'name': {'value': escapedQuery, 'boost': 10}},
}, },
); );
}); });
@ -156,7 +184,9 @@ function getResults (input) {
const splitATD = () => { const splitATD = () => {
let queries = []; let queries = [];
escapedQuery.split(' ').every((term, index) => { escapedQuery.split(' ').every((term, index) => {
if (index === 4) { return false } if (index === 4) {
return false;
}
queries.push({ // Contains search term in Author, Title, Description queries.push({ // Contains search term in Author, Title, Description
'query_string': { 'query_string': {
'query' : `*${term}*`, 'query' : `*${term}*`,
@ -277,6 +307,7 @@ function getResults (input) {
'bool': { 'bool': {
'should': [ 'should': [
conBoost, conBoost,
newerBoost,
funcScoreClaimWeight, funcScoreClaimWeight,
funcScoreChannelWeight, funcScoreChannelWeight,
channelIdentifier, channelIdentifier,
@ -360,7 +391,7 @@ function getAutoCompleteQuery (query) {
function getFilters (input) { function getFilters (input) {
var filters = []; var filters = [];
var bidStateFilter = {'bool': {'must_not': {'match': { 'bid_state': 'Accepted' }}}}; var bidStateFilter = {'bool': {'must_not': {'match': {'bid_state': 'Accepted'}}}};
if (input.nsfw === 'true' || input.nsfw === 'false') { if (input.nsfw === 'true' || input.nsfw === 'false') {
const nsfwFilter = {'match': {'value.stream.metadata.nsfw': input.nsfw}}; const nsfwFilter = {'match': {'value.stream.metadata.nsfw': input.nsfw}};
filters.push(nsfwFilter); filters.push(nsfwFilter);
@ -416,7 +447,7 @@ function getFilters (input) {
}, },
bidStateFilter]; bidStateFilter];
return filterQuery; return filterQuery;
} else { } else {
return [bidStateFilter]; return [bidStateFilter];
} }
} }
@ -446,7 +477,12 @@ function getStatus () {
rp(`http://localhost:9200/claims/_stats`) rp(`http://localhost:9200/claims/_stats`)
.then(function (data) { .then(function (data) {
data = JSON.parse(data); data = JSON.parse(data);
resolve({status: getStats(), spaceUsed: pretty(data._all.total.store.size_in_bytes, true), claimsInIndex: data._all.total.indexing.index_total, totSearches: data._all.total.search.query_total}); resolve({
status : getStats(),
spaceUsed : pretty(data._all.total.store.size_in_bytes, true),
claimsInIndex: data._all.total.indexing.index_total,
totSearches : data._all.total.search.query_total,
});
}) })
.catch(function (err) { .catch(function (err) {
reject(err); reject(err);
@ -459,11 +495,13 @@ function getWashedQuery (query) {
query = query.toLowerCase().replace(/ +/g, ' ').replace('lbry://', ''); query = query.toLowerCase().replace(/ +/g, ' ').replace('lbry://', '');
let splitBy = ['&', '$', ' ']; let splitBy = ['&', '$', ' '];
let regex = new RegExp(splitBy.join('|'), 'gi'); let regex = new RegExp(splitBy.join('|'), 'gi');
let badWords = [ 'from', 'with', 'not', 'can', 'all', 'are', 'for', 'but', 'and', 'the' ]; let badWords = ['from', 'with', 'not', 'can', 'all', 'are', 'for', 'but', 'and', 'the'];
let words = query.split(regex); let words = query.split(regex);
let sentence = []; let sentence = [];
words.forEach(w => { words.forEach(w => {
if (!badWords.includes(w)) { sentence.push(w) } if (!badWords.includes(w)) {
sentence.push(w);
}
}); });
query = sentence.join(' '); query = sentence.join(' ');
@ -474,7 +512,7 @@ function getWashedQuery (query) {
function getEscapedQuery (query) { function getEscapedQuery (query) {
// https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#_reserved_characters // https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#_reserved_characters
// The reserved characters are: + - = && || > < ! ( ) { } [ ] ^ " ~ * ? : \ / // The reserved characters are: + - = && || > < ! ( ) { } [ ] ^ " ~ * ? : \ /
let badCharacters = ['+', '-', '=', '&&', '||', '>', '<', '!', '(', ')', '{', '}', '[', ']', '^', '"', '~', '*', '?', ':', '\\', '/']; let badCharacters = ['+', '-', '=', '&&', '||', '>', '<', '!', '(', ')', '{', '}', '[', ']', '^', '"', '~', '*', '?', ':', '\\', '/'];
let escapedQuery = ''; let escapedQuery = '';
for (var i = 0; i < query.length; i++) { for (var i = 0; i < query.length; i++) {
let char1 = query.charAt(i); let char1 = query.charAt(i);
@ -502,12 +540,14 @@ async function update () {
class LighthouseControllers { class LighthouseControllers {
/* eslint-disable no-param-reassign */ /* eslint-disable no-param-reassign */
// Start syncing blocks... // Start syncing blocks...
startSync () { startSync () {
winston.log('info', '[Importer] Started importer, indexing claims.'); winston.log('info', '[Importer] Started importer, indexing claims.');
claimSync(); claimSync();
// sync(); // Old Sync // sync(); // Old Sync
} }
/** /**
* Search API Endpoint. * Search API Endpoint.
* @param {ctx} Koa Context * @param {ctx} Koa Context
@ -537,8 +577,8 @@ class LighthouseControllers {
cResults.push(name); cResults.push(name);
} }
if (pResult._source.value && if (pResult._source.value &&
pResult._source.value.stream && pResult._source.value.stream &&
pResult._source.value.stream !== undefined) { pResult._source.value.stream !== undefined) {
var title = pResult._source.value.stream.metadata.title; var title = pResult._source.value.stream.metadata.title;
var author = pResult._source.value.stream.metadata.author; var author = pResult._source.value.stream.metadata.author;
if (title.indexOf(ctx.query.s.trim()) > -1 && title.indexOf('http') === -1) { if (title.indexOf(ctx.query.s.trim()) > -1 && title.indexOf('http') === -1) {
@ -560,6 +600,7 @@ class LighthouseControllers {
ctx.body = clean; ctx.body = clean;
}); });
} }
/** /**
* Info about the api here * Info about the api here
* @param {ctx} Koa Context * @param {ctx} Koa Context
@ -583,7 +624,7 @@ class LighthouseControllers {
async autoUpdate (ctx) { async autoUpdate (ctx) {
let travisSignature = Buffer.from(ctx.request.headers.signature, 'base64'); let travisSignature = Buffer.from(ctx.request.headers.signature, 'base64');
let payload = ctx.request.body.payload; let payload = ctx.request.body.payload;
let travisResponse = await got('https://api.travis-ci.com/config', {timeout: 10000}); let travisResponse = await got('https://api.travis-ci.com/config', {timeout: 10000});
let travisPublicKey = JSON.parse(travisResponse.body).config.notifications.webhook.public_key; let travisPublicKey = JSON.parse(travisResponse.body).config.notifications.webhook.public_key;
let verifier = crypto.createVerify('sha1'); let verifier = crypto.createVerify('sha1');
verifier.update(payload); verifier.update(payload);
@ -597,15 +638,18 @@ class LighthouseControllers {
ctx.body = 'OK'; ctx.body = 'OK';
} else { } else {
ctx.status = 400; ctx.status = 400;
ctx.body = 'skip auto update: pull request'; logToSlack(ctx.body); ctx.body = 'skip auto update: pull request';
logToSlack(ctx.body);
} }
} else { } else {
ctx.status = 400; ctx.status = 400;
ctx.body = 'skip auto update: only deploys on master branch'; logToSlack(ctx.body); ctx.body = 'skip auto update: only deploys on master branch';
logToSlack(ctx.body);
} }
} else { } else {
ctx.status = 500; ctx.status = 500;
ctx.body = 'skip auto update: could not verify webhook'; logToSlack(ctx.body); ctx.body = 'skip auto update: could not verify webhook';
logToSlack(ctx.body);
} }
} }

6
server/utils/chainquery/index.js
View file

@ -19,7 +19,7 @@ import chainqueryConfig from '../../../chainquery-config.json';
let connection = null; let connection = null;
const esLogLevel = 'info'; const esLogLevel = 'info';
const MaxClaimsToProcessPerIteration = 100000; const MaxClaimsToProcessPerIteration = 100000000;
const BatchSize = 5000; const BatchSize = 5000;
const loggerStream = winstonStream(winston, esLogLevel); const loggerStream = winstonStream(winston, esLogLevel);
const eclient = new elasticsearch.Client({ const eclient = new elasticsearch.Client({
@ -32,7 +32,7 @@ const eclient = new elasticsearch.Client({
}, },
}); });
const queue = new ElasticQueue({elastic: eclient}); const queue = new ElasticQueue({batchSize: 5000, concurrency: 6, elastic: eclient});
queue.on('drain', function () { queue.on('drain', function () {
console.log('elasticsearch queue is drained'); console.log('elasticsearch queue is drained');
}); });
@ -211,6 +211,7 @@ function getClaimsSince (time, lastID, MaxClaimsInCall) {
p.claim_id as channel_id, p.claim_id as channel_id,
c.bid_state, c.bid_state,
c.effective_amount, c.effective_amount,
c.transaction_time,
COALESCE(p.effective_amount,1) as certificate_amount, COALESCE(p.effective_amount,1) as certificate_amount,
c.claim_id as claimId, c.claim_id as claimId,
c.value_as_json as value c.value_as_json as value
@ -245,6 +246,7 @@ function getClaimsSince (time, lastID, MaxClaimsInCall) {
bid_state : r.bid_state, bid_state : r.bid_state,
effective_amount : r.effective_amount, effective_amount : r.effective_amount,
certificate_amount: r.certificate_amount, certificate_amount: r.certificate_amount,
transaction_time : new Date(r.transaction_time * 1000),
claimId : r.claimId, claimId : r.claimId,
value : value, value : value,
}); });