LBRYCommunity/lighthouse.js
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Batched the chainquery calls for lighthouse in case it needs to grab all 700K in a redo.

Browse source
This commit is contained in:
Mark Beamer Jr 2019-02-24 16:57:56 -05:00
parent b04f2df1e1
commit 81986315cb
No known key found for this signature in database
GPG key ID: 1C314FB89AD76973
1 changed files with 29 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

49
server/utils/chainquery/index.js
View file

@ -44,24 +44,32 @@ export async function claimSync () {
syncState.LastSyncTime = '0001-01-01 00:00:00'; syncState.LastSyncTime = '0001-01-01 00:00:00';
} }
status.info = 'gettingClaimsToUpdate'; status.info = 'gettingClaimsToUpdate';
let claimsResponse = await getClaimsSince(syncState.LastSyncTime); let finished = false;
let claims = JSON.parse(claimsResponse).data; let lastID = 0;
status.info = 'addingClaimsToElastic'; let groupSize = 5000;
for (let claim of claims) { while (!finished) {
claim.value = JSON.parse(claim.value).Claim; let claimsResponse = await getClaimsSince(syncState.LastSyncTime, lastID, groupSize);
if (claim.name && claim.value) { let claims = JSON.parse(claimsResponse).data;
claim.suggest_name = { status.info = 'addingClaimsToElastic';
input : '' + claim.name + '', for (let claim of claims) {
weight: '30', claim.value = JSON.parse(claim.value).Claim;
}; if (claim.name && claim.value) {
} claim.suggest_name = {
if (claim.bid_state === 'Spent' || claim.bid_state === 'Expired') { input : '' + claim.name + '',
deleteFromElastic(claim.claimId); weight: '30',
} else { };
pushElastic(claim); }
if (claim.bid_state === 'Spent' || claim.bid_state === 'Expired') {
deleteFromElastic(claim.claimId);
} else {
pushElastic(claim);
}
lastID = claim.id;
} }
winston.log('info', '[Importer] Pushed ' + claims.length + ' claims to elastic search [LastID]' + lastID);
finished = claims.length < groupSize;
} }
winston.log('info', '[Importer] Pushed ' + claims.length + ' claims to elastic search');
deleteBlockedClaims(); deleteBlockedClaims();
syncState.LastSyncTime = new Date().toISOString().slice(0, 19).replace('T', ' '); syncState.LastSyncTime = new Date().toISOString().slice(0, 19).replace('T', ' ');
await saveJSON(path.join(appRoot.path, 'syncState.json'), syncState); await saveJSON(path.join(appRoot.path, 'syncState.json'), syncState);
@ -155,21 +163,22 @@ function getBlockedOutpoints () {
}); });
} }
function getClaimsSince (time) { function getClaimsSince (time, lastID, MaxClaimsInCall) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let query = `` + let query = `` +
`SELECT ` + `SELECT ` +
`c.id, ` +
`c.name,` + `c.name,` +
`p.name as channel,` + `p.name as channel,` +
`c.bid_state,` + `c.bid_state,` +
`c.effective_amount,` + `c.effective_amount,` +
`c.claim_id as claimId,` + `c.claim_id as claimId,` +
`c.value_as_json as value ` + `c.value_as_json as value ` +
// `,transaction_by_hash_id, ` + // txhash and vout needed to leverage old format for comparison.
// `vout ` +
`FROM claim c ` + `FROM claim c ` +
`LEFT JOIN claim p on p.claim_id = c.publisher_id ` + `LEFT JOIN claim p on p.claim_id = c.publisher_id ` +
`WHERE c.modified_at >='` + time + `'`; `WHERE c.id >` + lastID + ` ` +
`AND c.modified_at >='` + time + `' ` +
`LIMIT ` + MaxClaimsInCall;
// Outputs full query to console for copy/paste into chainquery (debugging) // Outputs full query to console for copy/paste into chainquery (debugging)
// console.log(query); // console.log(query);
rp(`https://chainquery.lbry.io/api/sql?query=` + query) rp(`https://chainquery.lbry.io/api/sql?query=` + query)