From e9ee65be338183b404d89a45d31617bfa548de93 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Sat, 26 May 2018 01:26:17 -0400 Subject: [PATCH 1/4] -Added content type to gendb to support elastic search 6.+ comms. --- gendb.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gendb.sh b/gendb.sh index 8ee119d..c94d802 100755 --- a/gendb.sh +++ b/gendb.sh @@ -1,10 +1,10 @@ #!/bin/bash -if [ "$(curl -IHEAD -w '%{http_code}' 'localhost:9200/claims' -o /dev/null --connect-timeout 3 --max-time 5)" == "200" ] ; +if [ "$(curl -IHEAD -w '%{http_code}' 'localhost:9200/claims' -o /dev/null --connect-timeout 3 --max-time 5)" == "200" ] ; then echo "Index already exists." ; exit 1; else echo "Index did not exist, creating..." ; - curl -X PUT http://localhost:9200/claims -d '{ "settings" : { "number_of_shards" : 1 }, "mappings" : { "claim" : { "properties" : { "value" : { "type" : "nested" }, "suggest_name": { "type": "completion" }, "suggest_desc": { "type": "completion" } } } } }'; + curl -X PUT http://localhost:9200/claims -H '"Content-Type: application/json"'-d '{ "settings" : { "number_of_shards" : 1 }, "mappings" : { "claim" : { "properties" : { "value" : { "type" : "nested" }, "suggest_name": { "type": "completion" }, "suggest_desc": { "type": "completion" } } } } }'; exit 0; -fi \ No newline at end of file +fi -- 2.45.2 From bb8525f69ea8c62375b7d9156f0158c43f7c69ad Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Sat, 26 May 2018 22:47:26 -0400 Subject: [PATCH 2/4] -Fixed bug that causes completionmapper parse error in elastic search. This causes an entire batch to fail. Batches are up to 500 so if 1 fails they all fail. The impact is that many claims were not indexed in elastic search. --- server/utils/chainquery/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/utils/chainquery/index.js b/server/utils/chainquery/index.js index 91ee49f..756bbb7 100644 --- a/server/utils/chainquery/index.js +++ b/server/utils/chainquery/index.js @@ -56,8 +56,8 @@ export async function claimSync () { // console.log('--------------------------------------------'); if (claim.name && claim.value) { claim.suggest_name = { - input : claim.name, - weight: 30, + input : '' + claim.name + '', + weight: '30', }; } pushElastic(claim); -- 2.45.2 From 78e26b2e6e7ef66fd56ae52074ebe5a6cd4ff418 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Sat, 26 May 2018 22:49:29 -0400 Subject: [PATCH 3/4] -Parameterized elastic search logging to a const. --- server/utils/chainquery/index.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/utils/chainquery/index.js b/server/utils/chainquery/index.js index 756bbb7..05ae1ea 100644 --- a/server/utils/chainquery/index.js +++ b/server/utils/chainquery/index.js @@ -13,12 +13,13 @@ import fs from 'fs'; import fileExists from 'file-exists'; import * as util from '../../utils/importer/util'; -const loggerStream = winstonStream(winston, 'info'); +const elasticsearchloglevel = 'info'; +const loggerStream = winstonStream(winston, elasticsearchloglevel); const eclient = new elasticsearch.Client({ host: 'http://localhost:9200', log: { - level : 'info', + level : elasticsearchloglevel, type : 'stream', stream: loggerStream, }, -- 2.45.2 From cd0f9809042011630bb7875e1314c481954c6029 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Sat, 26 May 2018 22:56:09 -0400 Subject: [PATCH 4/4] -Refactored handling of blocked claims. -Changed deletes to be queued and refactored. -Added logging. -Removed commented debug code. -Added Spent and Expired bid_state handling. --- server/utils/chainquery/index.js | 57 +++++++++++++++++--------------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/server/utils/chainquery/index.js b/server/utils/chainquery/index.js index 05ae1ea..61afa6a 100644 --- a/server/utils/chainquery/index.js +++ b/server/utils/chainquery/index.js @@ -26,7 +26,7 @@ const eclient = new elasticsearch.Client({ }); const queue = new ElasticQueue({elastic: eclient}); -// Check that our cache file exist. +// Check that our syncState file exist. fileExists(path.join(appRoot.path, 'syncState.json'), (err, exists) => { if (err) { throw err } if (!exists) { @@ -48,38 +48,20 @@ export async function claimSync () { status.info = 'addingClaimsToElastic'; for (let claim of claims) { claim.value = JSON.parse(claim.value).Claim; - // console.log('NEW: ' + JSON.stringify(claim)); - // console.log('--------------------------------------------'); - // var imprtr = require('../importer'); - // let oldvalue = await imprtr.getValue(claim.transaction_by_hash_id, claim.vout); - // oldvalue = JSON.parse(oldvalue); - // console.log('OLD: ' + JSON.stringify(oldvalue)); - // console.log('--------------------------------------------'); if (claim.name && claim.value) { claim.suggest_name = { input : '' + claim.name + '', weight: '30', }; } - pushElastic(claim); + if (claim.bid_state === 'Spent' || claim.bid_state === 'Expired') { + deleteFromElastic(claim.claimId); + } else { + pushElastic(claim); + } } winston.log('info', '[Importer] Pushed ' + claims.length + ' claims to elastic search'); - winston.log('info', '[Importer] Removing blocked claims from search!'); - let blockedOutputsResponse = await getBlockedOutpoints(); - let outpointlist = JSON.parse(blockedOutputsResponse); - for (let outpoint of outpointlist.data.outpoints) { - let claimid = util.OutpointToClaimId(outpoint); - console.log('Deleting ClaimId: ' + claimid); - eclient.delete({ - index: 'claims', - type : 'claim', - id : claimid, - }, function (error, response) { - if (error) { - winston.log(error, response); - } - }); - } + deleteBlockedClaims(); syncState.LastSyncTime = new Date().toISOString().slice(0, 19).replace('T', ' '); await saveJSON(path.join(appRoot.path, 'syncState.json'), syncState); status.info = 'upToDate'; @@ -97,8 +79,29 @@ export function getStats () { return status; } +async function deleteBlockedClaims () { + winston.log('info', '[Importer] Removing blocked claims from search!'); + let blockedOutputsResponse = await getBlockedOutpoints(); + let outpointlist = JSON.parse(blockedOutputsResponse); + for (let outpoint of outpointlist.data.outpoints) { + let claimid = util.OutpointToClaimId(outpoint); + deleteFromElastic(claimid); + } + winston.log('info', '[Importer] Done processing blocked claims!'); +} + +async function deleteFromElastic (claimid) { + return new Promise(async(resolve, reject) => { + queue.push({ + index: 'claims', + type : 'claim', + id : claimid, + body : {}, + }); + }); +} + async function pushElastic (claim) { - console.log('Pushing To Elastic Search claimId:' + claim.claimId); return new Promise(async(resolve, reject) => { queue.push({ index: 'claims', @@ -143,6 +146,7 @@ function getBlockedOutpoints () { resolve(htmlString); }) .catch(function (err) { + winston.log('error', '[Importer] Error getting blocked outpoints. ' + err); reject(err); }); }); @@ -168,6 +172,7 @@ function getClaimsSince (time) { resolve(htmlString); }) .catch(function (err) { + winston.log('error', '[Importer] Error getting updated claims. ' + err); reject(err); }); }); -- 2.45.2