Merge pull request #183 from ykris45/patch-1

Update LICENSE

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2017-2019 LBRY Inc
+Copyright (c) 2017-2020 LBRY Inc
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

gendb.sh

@@ -5,6 +5,6 @@ then
    exit 1;
 else
    echo "Index did not exist, creating..." ;
-   curl -H 'Content-Type: application/json' -H 'Accept: application/json' -X PUT -d '{ "settings" : { "number_of_shards" : 1 }, "mappings" : { "claim" : { "properties" : { "value" : { "type" : "nested" }, "suggest_name": { "type": "completion" }, "suggest_desc": { "type": "completion" }, "transaction_time": { "type": "date" } } } } }' http://localhost:9200/claims;
+   curl -H 'Content-Type: application/json' -H 'Accept: application/json' -X PUT -d '{ "settings" : { "number_of_shards" : 1 }, "mappings" : { "claim" : { "properties" : { "value" : { "type" : "nested" }, "suggest_name": { "type": "completion" }, "suggest_desc": { "type": "completion" } } } } }' http://localhost:9200/claims;
    exit 0;
 fi

package-lock.json

@@ -5336,9 +5336,9 @@
       "dev": true
     },
     "handlebars": {
-      "version": "4.1.2",
+      "version": "4.5.3",
-      "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.1.2.tgz",
+      "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.5.3.tgz",
-      "integrity": "sha512-nvfrjqvt9xQ8Z/w0ijewdD/vvWDTOweBUm96NTr66Wfvo1mJenBLwcYmPs3TIBP5ruzYGD7Hx/DaM9RmhroGPw==",
+      "integrity": "sha512-3yPecJoJHK/4c6aZhSvxOyG4vJKDshV36VHp0iVCDVh7o9w2vwi3NSnL2MMPj3YdduqaBcu7cGbggJQM0br9xA==",
       "requires": {
         "neo-async": "^2.6.0",
         "optimist": "^0.6.1",

server/controllers/lighthouse.js

@@ -80,64 +80,6 @@ function getResults (input) {
       },
     },
   };
-  const newerBoost = {
-    'function_score': {
-      'score_mode': 'sum', // All functions outputs get summed
-      'boost_mode': 'multiply', // The documents relevance is multiplied with the sum
-
-      'functions': [
-        {
-          // The relevancy of old posts is multiplied by at least one.
-          // Remove if you want to exclude old posts
-          'weight': 0.6,
-        },
-        {
-          // Published this week get a big boost
-          'weight': 85,
-          'gauss' : {
-            'transaction_time': { // <- Change to your date field name
-              'origin': Date.now(), // Change to current date
-              'scale' : '7d',
-              'decay' : 0.6,
-            },
-          },
-        },
-        {
-          // Published this month get a big boost
-          'weight': 60,
-          'gauss' : {
-            'transaction_time': { // <- Change to your date field name
-              'origin': Date.now(), // Change to current date
-              'scale' : '31d',
-              'decay' : 0.55,
-            },
-          },
-        },
-        {
-          // Published this month get a big boost
-          'weight': 50,
-          'gauss' : {
-            'transaction_time': { // <- Change to your date field name
-              'origin': Date.now(), // Change to current date
-              'scale' : '62d',
-              'decay' : 0.5,
-            },
-          },
-        },
-        {
-          // Published this year get a boost
-          'weight': 40,
-          'gauss' : {
-            'transaction_time': { // <- Change to your date field name
-              'origin': Date.now(), // Change to current date
-              'scale' : '356d',
-              'decay' : 0.2,
-            },
-          },
-        },
-      ],
-    },
-  };
   const funcScoreClaimWeight = { // 100 LBC adds 1 point to the score
     'function_score': {
       'field_value_factor': {
@@ -170,9 +112,7 @@ function getResults (input) {
     let conCatTerm = '';
     let phraseTerm = '';
     escapedQuery.split(' ').every((term, index) => {
-      if (index === 4) {
+      if (index === 4) { return false }
-        return false;
-      }
       phraseTerm = phraseTerm + ' ' + term;
       conCatTerm = conCatTerm + term;
       queries.push(
@@ -204,10 +144,10 @@ function getResults (input) {
           },
         },
         {
-          'prefix': {'name': {'value': '@' + escapedQuery, 'boost': 10}},
+          'prefix': { 'name': { 'value': '@' + escapedQuery, 'boost': 10 } },
         },
         {
-          'prefix': {'name': {'value': escapedQuery, 'boost': 10}},
+          'prefix': { 'name': { 'value': escapedQuery, 'boost': 10 } },
         },
       );
     });
@@ -216,9 +156,7 @@ function getResults (input) {
   const splitATD = () => {
     let queries = [];
     escapedQuery.split(' ').every((term, index) => {
-      if (index === 4) {
+      if (index === 4) { return false }
-        return false;
-      }
       queries.push({ // Contains search term in Author, Title, Description
         'query_string': {
           'query' : `*${term}*`,
@@ -339,7 +277,6 @@ function getResults (input) {
         'bool': {
           'should': [
             conBoost,
-            newerBoost,
             funcScoreClaimWeight,
             funcScoreChannelWeight,
             channelIdentifier,
@@ -423,7 +360,7 @@ function getAutoCompleteQuery (query) {
 
 function getFilters (input) {
   var filters = [];
-  var bidStateFilter = {'bool': {'must_not': {'match': {'bid_state': 'Accepted'}}}};
+  var bidStateFilter = {'bool': {'must_not': {'match': { 'bid_state': 'Accepted' }}}};
   if (input.nsfw === 'true' || input.nsfw === 'false') {
     const nsfwFilter = {'match': {'value.stream.metadata.nsfw': input.nsfw}};
     filters.push(nsfwFilter);
@@ -479,7 +416,7 @@ function getFilters (input) {
       },
       bidStateFilter];
     return filterQuery;
-  } else {
+  }  else {
     return [bidStateFilter];
   }
 }
@@ -509,12 +446,7 @@ function getStatus () {
     rp(`http://localhost:9200/claims/_stats`)
       .then(function (data) {
         data = JSON.parse(data);
-        resolve({
+        resolve({status: getStats(), spaceUsed: pretty(data._all.total.store.size_in_bytes, true), claimsInIndex: data._all.total.indexing.index_total, totSearches: data._all.total.search.query_total});
-          status       : getStats(),
-          spaceUsed    : pretty(data._all.total.store.size_in_bytes, true),
-          claimsInIndex: data._all.total.indexing.index_total,
-          totSearches  : data._all.total.search.query_total,
-        });
       })
       .catch(function (err) {
         reject(err);
@@ -527,13 +459,11 @@ function getWashedQuery (query) {
   query = query.toLowerCase().replace(/ +/g, ' ').replace('lbry://', '');
   let splitBy = ['&', '$', ' '];
   let regex = new RegExp(splitBy.join('|'), 'gi');
-  let badWords = ['from', 'with', 'not', 'can', 'all', 'are', 'for', 'but', 'and', 'the'];
+  let badWords  = [ 'from', 'with', 'not', 'can', 'all', 'are', 'for', 'but', 'and', 'the' ];
   let words = query.split(regex);
   let sentence = [];
   words.forEach(w => {
-    if (!badWords.includes(w)) {
+    if (!badWords.includes(w))      { sentence.push(w) }
-      sentence.push(w);
-    }
   });
   query = sentence.join(' ');
 
@@ -544,7 +474,7 @@ function getWashedQuery (query) {
 function getEscapedQuery (query) {
   // https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#_reserved_characters
   // The reserved characters are: + - = && || > < ! ( ) { } [ ] ^ " ~ * ? : \ /
-  let badCharacters = ['+', '-', '=', '&&', '||', '>', '<', '!', '(', ')', '{', '}', '[', ']', '^', '"', '~', '*', '?', ':', '\\', '/'];
+  let badCharacters  = ['+', '-', '=', '&&', '||', '>', '<', '!', '(', ')', '{', '}', '[', ']', '^', '"', '~', '*', '?', ':', '\\', '/'];
   let escapedQuery = '';
   for (var i = 0; i < query.length; i++) {
     let char1 = query.charAt(i);
@@ -572,14 +502,12 @@ async function update () {
 
 class LighthouseControllers {
   /* eslint-disable no-param-reassign */
-
   // Start syncing blocks...
   startSync () {
     winston.log('info', '[Importer] Started importer, indexing claims.');
     claimSync();
     // sync(); // Old Sync
   }
-
   /**
    * Search API Endpoint.
    * @param {ctx} Koa Context
@@ -587,7 +515,6 @@ class LighthouseControllers {
   async search (ctx) {
     await getResults(ctx.query).then(function (result) {
       let results = result.hits.hits;
-      console.log(results);
       let cResults = [];
       for (let pResult of results) {
         cResults.push(pResult._source);
@@ -610,8 +537,8 @@ class LighthouseControllers {
           cResults.push(name);
         }
         if (pResult._source.value &&
-          pResult._source.value.stream &&
+            pResult._source.value.stream &&
-          pResult._source.value.stream !== undefined) {
+            pResult._source.value.stream !== undefined) {
           var title = pResult._source.value.stream.metadata.title;
           var author = pResult._source.value.stream.metadata.author;
           if (title.indexOf(ctx.query.s.trim()) > -1 && title.indexOf('http') === -1) {
@@ -633,7 +560,6 @@ class LighthouseControllers {
       ctx.body = clean;
     });
   }
-
   /**
    * Info about the api here
    * @param {ctx} Koa Context
@@ -657,7 +583,7 @@ class LighthouseControllers {
   async autoUpdate (ctx) {
     let travisSignature = Buffer.from(ctx.request.headers.signature, 'base64');
     let payload = ctx.request.body.payload;
-    let travisResponse = await got('https://api.travis-ci.com/config', {timeout: 10000});
+    let travisResponse  = await got('https://api.travis-ci.com/config', {timeout: 10000});
     let travisPublicKey = JSON.parse(travisResponse.body).config.notifications.webhook.public_key;
     let verifier = crypto.createVerify('sha1');
     verifier.update(payload);
@@ -671,18 +597,15 @@ class LighthouseControllers {
           ctx.body = 'OK';
         } else {
           ctx.status = 400;
-          ctx.body = 'skip auto update: pull request';
+          ctx.body = 'skip auto update: pull request'; logToSlack(ctx.body);
-          logToSlack(ctx.body);
         }
       } else {
         ctx.status = 400;
-        ctx.body = 'skip auto update: only deploys on master branch';
+        ctx.body = 'skip auto update: only deploys on master branch'; logToSlack(ctx.body);
-        logToSlack(ctx.body);
       }
     } else {
       ctx.status = 500;
-      ctx.body = 'skip auto update: could not verify webhook';
+      ctx.body = 'skip auto update: could not verify webhook'; logToSlack(ctx.body);
-      logToSlack(ctx.body);
     }
   }
 

server/utils/chainquery/index.js

@@ -19,7 +19,7 @@ import chainqueryConfig from '../../../chainquery-config.json';
 let connection = null;
 
 const esLogLevel = 'info';
-const MaxClaimsToProcessPerIteration = 100000000;
+const MaxClaimsToProcessPerIteration = 100000;
 const BatchSize = 5000;
 const loggerStream = winstonStream(winston, esLogLevel);
 const eclient = new elasticsearch.Client({
@@ -32,7 +32,7 @@ const eclient = new elasticsearch.Client({
   },
 });
 
-const queue = new ElasticQueue({batchSize: 5000, concurrency: 6, elastic: eclient});
+const queue = new ElasticQueue({elastic: eclient});
 queue.on('drain', function () {
   console.log('elasticsearch queue is drained');
 });
@@ -211,7 +211,6 @@ function getClaimsSince (time, lastID, MaxClaimsInCall) {
      p.claim_id as channel_id,
      c.bid_state,
      c.effective_amount,
-     c.transaction_time,
      COALESCE(p.effective_amount,1) as certificate_amount,
      c.claim_id as claimId,
      c.value_as_json as value
@@ -246,7 +245,6 @@ function getClaimsSince (time, lastID, MaxClaimsInCall) {
           bid_state         : r.bid_state,
           effective_amount  : r.effective_amount,
           certificate_amount: r.certificate_amount,
-          transaction_time  : new Date(r.transaction_time * 1000),
           claimId           : r.claimId,
           value             : value,
         });