diff --git a/server/utils/importer/index.js b/server/utils/importer/index.js index f20dc0d..79547fd 100644 --- a/server/utils/importer/index.js +++ b/server/utils/importer/index.js @@ -121,6 +121,24 @@ export async function sync () { } }); } + winston.log('info', '[Importer] Removing blocked claims from search!'); + var util = require('./util.js'); + var blockedOutputsResponse = await getBlockedOutpoints(); + var outpointlist = JSON.parse(blockedOutputsResponse); + for (let outpoint of outpointlist.data.outpoints) { + var claimid = util.OutpointToClaimId(outpoint); + console.log('Deleting ClaimId: ' + claimid); + eclient.delete({ + index: 'claims', + type : 'claim', + id : claimid, + }, function (error, response) { + if (error) { + winston.log(error); + } + }); + } + // Done adding, update our claimTrie cache to latest and wait a bit... await saveJSON(path.join(appRoot.path, 'claimTrieCache.json'), latestClaimTrie); status.info = 'upToDate'; @@ -152,6 +170,18 @@ function getRemovedClaims (oldClaimTrie, newClaimTrie) { }); } +function getBlockedOutpoints () { + return new Promise((resolve, reject) => { + rp(`http://api.lbry.io/file/list_blocked`) + .then(function (htmlString) { + resolve(htmlString); + }) + .catch(function (err) { + reject(err); + }); + }); +} + function getValue (tx, i) { return new Promise((resolve, reject) => { rp(`http://localhost:5000/claim_decode/${tx}/${i}`) diff --git a/server/utils/importer/util.js b/server/utils/importer/util.js new file mode 100644 index 0000000..05a8089 --- /dev/null +++ b/server/utils/importer/util.js @@ -0,0 +1,57 @@ + +// taken from https://github.com/crypto-browserify/buffer-reverse/blob/master/inplace.js +function reverseInplace (buffer) { + for (var i = 0, j = buffer.length - 1; i < j; ++i, --j) { + var t = buffer[j]; + + buffer[j] = buffer[i]; + buffer[i] = t; + } + + return buffer; +} + +function ripemd160 (bytearray) { + const crypto = require('crypto'); + + const secret = 'abcdefg'; + const hash = crypto.createHash('ripemd160', secret) + .update(bytearray, 'binary', 'binary') + .digest(); + return hash; +} + +function sha256 (bytearray) { + const crypto = require('crypto'); + + const secret = 'abcdefg'; + const hash = crypto.createHash('sha256', secret) + .update(bytearray, 'binary', 'binary') + .digest(); + return hash; +} + +export function OutpointToClaimId (outpointstr) { + var outpointsplit = outpointstr.split(':'); + var outpoint = {txid: outpointsplit[0], vout: outpointsplit[1]}; + // Assuming endianess is LittleEndian - Javacsript endianess depends on hardware. + // Can check with os.endianness(). Possible values are "BE" or "LE" as of Node.js v0.10.0 + // convert txid to byte array then reverse bytes to get BigEndian + var txidbytes = Buffer.from(outpoint.txid, 'hex'); + var txidrevbytes = reverseInplace(txidbytes); + // Get big endian of vout(uint32) + var voutbytes = Buffer.allocUnsafe(4); + voutbytes.writeInt32BE(outpoint.vout); + // Merge arrays + var claimidbytes = Buffer.concat([txidrevbytes, voutbytes]); + // Hash - Sha256 + claimidbytes = sha256(claimidbytes); + // Hash - RipeMD160 + claimidbytes = ripemd160(claimidbytes); + // Return to little endian. + claimidbytes = reverseInplace(claimidbytes); + // Encode to hex string + var claimid = claimidbytes.toString('hex'); + + return claimid; +}