From affa46e0f687d4fb36300fb1e126eb053cfb408b Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Thu, 23 May 2019 23:55:57 -0400 Subject: [PATCH] initial import of canonical url via the new sql wallet server --- .../extras/daemon/json_response_encoder.py | 1 + lbrynet/schema/result.py | 2 + lbrynet/wallet/server/canonical.py | 28 +++++++ lbrynet/wallet/server/db.py | 28 +++++-- tests/unit/wallet/server/test_sqldb.py | 77 +++++++++++++++++-- 5 files changed, 123 insertions(+), 13 deletions(-) create mode 100644 lbrynet/wallet/server/canonical.py diff --git a/lbrynet/extras/daemon/json_response_encoder.py b/lbrynet/extras/daemon/json_response_encoder.py index 644991c98..c4bc0e4d7 100644 --- a/lbrynet/extras/daemon/json_response_encoder.py +++ b/lbrynet/extras/daemon/json_response_encoder.py @@ -175,6 +175,7 @@ class JSONResponseEncoder(JSONEncoder): }) if include_meta: output['meta'] = self.encode_claim_meta(txo.meta) + output['canonical_url'] = output['meta'].pop('canonical_url', None) if txo.script.is_claim_name or txo.script.is_update_claim: try: output['value'] = txo.claim diff --git a/lbrynet/schema/result.py b/lbrynet/schema/result.py index c263f023e..c2c316d03 100644 --- a/lbrynet/schema/result.py +++ b/lbrynet/schema/result.py @@ -20,6 +20,7 @@ class Outputs: def _inflate_claim(self, txo, message): txo.meta = { + 'canonical_url': message.canonical_url, 'is_controlling': message.is_controlling, 'activation_height': message.activation_height, 'effective_amount': message.effective_amount, @@ -91,6 +92,7 @@ class Outputs: txo_message.height = txo['height'] txo_message.tx_hash = txo['txo_hash'][:32] txo_message.nout, = struct.unpack(' len(self.short_id)-1: + self.short_id = self.new_id[:i+1] + break + + def finalize(self): + if self.short_id: + return '#'+self.short_id.decode() + return '' + + +def register_canonical_functions(connection): + connection.create_aggregate("shortest_id", 2, FindShortestID) diff --git a/lbrynet/wallet/server/db.py b/lbrynet/wallet/server/db.py index 41b3669ac..8b198ba89 100644 --- a/lbrynet/wallet/server/db.py +++ b/lbrynet/wallet/server/db.py @@ -9,6 +9,7 @@ from torba.client.basedatabase import query, constraints_to_sql from lbrynet.schema.url import URL, normalize_name from lbrynet.wallet.transaction import Transaction, Output +from lbrynet.wallet.server.canonical import register_canonical_functions from lbrynet.wallet.server.trending import ( CREATE_TREND_TABLE, calculate_trending, register_trending_functions ) @@ -66,8 +67,9 @@ class SQLDB: CREATE_CLAIM_TABLE = """ create table if not exists claim ( claim_hash bytes primary key, - normalized text not null, claim_name text not null, + normalized text not null, + canonical text not null, is_channel bool not null, txo_hash bytes not null, tx_position integer not null, @@ -150,6 +152,7 @@ class SQLDB: self.db = sqlite3.connect(self._db_path, isolation_level=None, check_same_thread=False) self.db.row_factory = sqlite3.Row self.db.executescript(self.CREATE_TABLES_QUERY) + register_canonical_functions(self.db) register_trending_functions(self.db) def close(self): @@ -250,11 +253,19 @@ class SQLDB: self.db.executemany(""" INSERT INTO claim ( claim_hash, normalized, claim_name, is_channel, txo_hash, tx_position, - height, amount, channel_hash, release_time, publish_time, activation_height) + height, amount, channel_hash, release_time, publish_time, activation_height, + canonical) VALUES ( :claim_hash, :normalized, :claim_name, :is_channel, :txo_hash, :tx_position, :height, :amount, :channel_hash, :release_time, :publish_time, - CASE WHEN :normalized NOT IN (SELECT normalized FROM claimtrie) THEN :height END + CASE WHEN :normalized NOT IN (SELECT normalized FROM claimtrie) THEN :height END, + CASE WHEN :channel_hash IS NOT NULL + THEN (SELECT canonical FROM claim WHERE claim_hash=:channel_hash)||'/'|| + :normalized||COALESCE((SELECT shortest_id(claim_hash, :claim_hash) + FROM claim WHERE normalized = :normalized), '') + ELSE :normalized||COALESCE((SELECT shortest_id(claim_hash, :claim_hash) + FROM claim WHERE normalized = :normalized), '') + END )""", claims) def update_claims(self, txos: Set[Output], header): @@ -561,7 +572,7 @@ class SQLDB: return self.get_claims( """ claimtrie.claim_hash as is_controlling, - claim.claim_hash, claim.txo_hash, claim.height, + claim.claim_hash, claim.txo_hash, claim.height, claim.canonical, claim.activation_height, claim.effective_amount, claim.support_amount, claim.trending_group, claim.trending_mixed, claim.trending_local, claim.trending_global, @@ -614,7 +625,10 @@ class SQLDB: continue channel = None if url.has_channel: - matches = self._search(is_controlling=True, **url.channel.to_dict()) + query = url.channel.to_dict() + if set(query) == {'name'}: + query['is_controlling'] = True + matches = self._search(**query) if matches: channel = matches[0] else: @@ -624,7 +638,9 @@ class SQLDB: query = url.stream.to_dict() if channel is not None: query['channel_hash'] = channel['claim_hash'] - matches = self._search(is_controlling=True, **query) + if set(query) == {'name'}: + query['is_controlling'] = True + matches = self._search(**query) if matches: result.append(matches[0]) else: diff --git a/tests/unit/wallet/server/test_sqldb.py b/tests/unit/wallet/server/test_sqldb.py index 7422171e8..c8bc41b92 100644 --- a/tests/unit/wallet/server/test_sqldb.py +++ b/tests/unit/wallet/server/test_sqldb.py @@ -1,10 +1,13 @@ import unittest -from binascii import hexlify +import ecdsa +import hashlib +from binascii import hexlify, unhexlify from torba.client.constants import COIN, NULL_HASH32 from lbrynet.schema.claim import Claim from lbrynet.wallet.server.db import SQLDB from lbrynet.wallet.server.trending import TRENDING_WINDOW +from lbrynet.wallet.server.canonical import FindShortestID from lbrynet.wallet.server.block_processor import Timer from lbrynet.wallet.transaction import Transaction, Input, Output @@ -23,10 +26,6 @@ def get_tx(): return Transaction().add_inputs([get_input()]) -def claim_id(claim_hash): - return hexlify(claim_hash[::-1]).decode() - - class OldWalletServerTransaction: def __init__(self, tx): self.tx = tx @@ -57,7 +56,10 @@ class TestSQLDB(unittest.TestCase): claim = Claim() claim.channel.title = title channel = Output.pay_claim_name_pubkey_hash(amount, name, claim, b'abc') - channel.generate_channel_private_key() + private_key = ecdsa.SigningKey.from_string(b'c'*32, curve=ecdsa.SECP256k1, hashfunc=hashlib.sha256) + channel.private_key = private_key.to_pem().decode() + channel.claim.channel.public_key_bytes = private_key.get_verifying_key().to_der() + channel.script.generate() return self._make_tx(channel) def get_stream(self, title, amount, name='foo'): @@ -284,8 +286,69 @@ class TestSQLDB(unittest.TestCase): self.get_support(up_biggly, (20+(window*(3 if window == 7 else 1)))*COIN), ]) results = self.sql._search(order_by=['trending_local']) - self.assertEqual([c.claim_id for c in claims], [claim_id(c['claim_hash']) for c in results]) + self.assertEqual([c.claim_id for c in claims], [hexlify(c['claim_hash'][::-1]).decode() for c in results]) self.assertEqual([10, 6, 2, 0, -2], [int(c['trending_local']) for c in results]) self.assertEqual([53, 38, -32, 0, -6], [int(c['trending_global']) for c in results]) self.assertEqual([4, 4, 2, 0, 1], [int(c['trending_group']) for c in results]) self.assertEqual([53, 38, 2, 0, -6], [int(c['trending_mixed']) for c in results]) + + @staticmethod + def _get_x_with_claim_id_prefix(getter, prefix, cached_iteration=None): + iterations = 100 + for i in range(cached_iteration or 1, iterations): + stream = getter(f'claim #{i}', COIN) + if stream[0].tx.outputs[0].claim_id.startswith(prefix): + print(f'Found "{prefix}" in {i} iterations.') + return stream + raise ValueError(f'Failed to find "{prefix}" in {iterations} iterations.') + + def get_channel_with_claim_id_prefix(self, prefix, cached_iteration): + return self._get_x_with_claim_id_prefix(self.get_channel, prefix, cached_iteration) + + def get_stream_with_claim_id_prefix(self, prefix, cached_iteration): + return self._get_x_with_claim_id_prefix(self.get_stream, prefix, cached_iteration) + + def test_canonical_name(self): + advance = self.advance + tx_abc = self.get_stream_with_claim_id_prefix('abc', 65) + tx_ab = self.get_stream_with_claim_id_prefix('ab', 42) + tx_a = self.get_stream_with_claim_id_prefix('a', 2) + advance(1, [tx_a]) + advance(2, [tx_ab]) + advance(3, [tx_abc]) + r_a, r_ab, r_abc = self.sql._search(order_by=['^height']) + self.assertEqual("foo", r_a['canonical']) + self.assertEqual(f"foo#ab", r_ab['canonical']) + self.assertEqual(f"foo#abc", r_abc['canonical']) + + tx_ab = self.get_channel_with_claim_id_prefix('ab', 72) + tx_a = self.get_channel_with_claim_id_prefix('a', 1) + advance(4, [tx_a]) + advance(5, [tx_ab]) + + tx_c = self.get_stream_with_claim_id_prefix('c', 2) + tx_cd = self.get_stream_with_claim_id_prefix('cd', 2) + advance(6, [tx_c]) + advance(7, [tx_cd]) + + r_a, r_ab, r_abc = self.sql._search(order_by=['^height']) + self.assertEqual("foo", r_a['canonical']) + self.assertEqual(f"foo#ab", r_ab['canonical']) + self.assertEqual(f"foo#abc", r_abc['canonical']) + + def test_canonical_find_shortest_id(self): + new_hash = unhexlify('abcdef0123456789beef')[::-1] + other0 = unhexlify('1bcdef0123456789beef')[::-1] + other1 = unhexlify('ab1def0123456789beef')[::-1] + other2 = unhexlify('abc1ef0123456789beef')[::-1] + other3 = unhexlify('abcdef0123456789bee1')[::-1] + f = FindShortestID() + self.assertEqual('', f.finalize()) + f.step(other0, new_hash) + self.assertEqual('#a', f.finalize()) + f.step(other1, new_hash) + self.assertEqual('#abc', f.finalize()) + f.step(other2, new_hash) + self.assertEqual('#abcd', f.finalize()) + f.step(other3, new_hash) + self.assertEqual('#abcdef0123456789beef', f.finalize())