working sync

lbrynet/extras/daemon/Daemon.py

@@ -14,8 +14,9 @@ from binascii import hexlify, unhexlify
 from traceback import format_exc
 from aiohttp import web
 from functools import wraps
+from torba.client.wallet import Wallet
 from torba.client.baseaccount import SingleKey, HierarchicalDeterministic
-from torba.client.hash import aes_encrypt, sha256
+from torba.client.hash import sha256
 
 from lbrynet import __version__, utils
 from lbrynet.conf import Config, Setting, SLACK_WEBHOOK
@@ -1252,42 +1253,61 @@ class Daemon(metaclass=JSONRPCServerType):
         return result
 
     @requires("wallet")
-    def jsonrpc_account_manifest(self, password, account_ids=None):
+    def jsonrpc_sync_hash(self):
         """
-        Generate a manifest for all of the accounts or only for limited set of accounts.
+        Deterministic hash of the wallet.
 
         Usage:
-            account manifest <password> [<account_ids>...]
+            sync hash
 
         Options:
-            --password=<password>         : (str) password to use for encrypting values
-            --account-ids=<account_ids>   : (list) list of accounts ids to limit manifest
 
         Returns:
-            (map) manifest
+            (str) sha256 hash of wallet
+        """
+        return hexlify(self.default_wallet.hash).decode()
+
+    @requires("wallet")
+    def jsonrpc_sync_apply(self, password, data=None, encrypt_password=None):
+        """
+        Apply incoming synchronization data, if provided, and then produce a sync hash and
+        an encrypted wallet.
+
+        Usage:
+            sync apply <password> [--data=<data>] [--encrypt-password=<encrypt_password>]
+
+        Options:
+            --password=<password>         : (str) password to decrypt incoming and encrypt outgoing data
+            --data=<data>                 : (str) incoming sync data, if any
+            --encrypt-password=<encrypt_password> : (str) password to encrypt outgoing data if different
+                                                    from the decrypt password, used during password changes
+
+        Returns:
+            (map) sync hash and data
 
         """
-        init_vector = b'!\tT\xef\x0c\x85j\x9elp\xf1\xa6\xe0\xe8\x188'
+        if data is not None:
-        accounts = self.get_accounts_or_all(account_ids)
+            decrypted_data = Wallet.unpack(password, data)
-        manifest = []
+            for account_data in decrypted_data['accounts']:
-        status = []
+                _, _, pubkey = LBCAccount.keys_from_dict(self.ledger, account_data)
-        for account in accounts:
+                account_id = pubkey.address
-            encrypted_data = aes_encrypt(password, json.dumps(account.to_dict(False)), init_vector).encode()
+                local_match = None
-            manifest.append({
+                for local_account in self.default_wallet.accounts:
-                'account_id': account.id,
+                    if account_id == local_account.id:
-                'timestamp': time.time(),
+                        local_match = local_account
-                'hash-data': hexlify(sha256(encrypted_data)),
+                        break
-                'hash-certificates': [
+                if local_match is not None:
-                    aes_encrypt(password, cert, init_vector).encode() for cert in account.certificates.keys()
+                    local_match.name = account_data.get('name', local_match.name)
-                ]
+                    local_match.certificates.update(account_data.get('certificates', {}))
-            })
+                else:
-            status.append(manifest[-1]['hash-data'])
+                    new_account = LBCAccount.from_dict(self.ledger, self.default_wallet, account_data)
-            status.extend(manifest[-1]['hash-certificates'])
+                    if self.ledger.network.is_connected:
+                        asyncio.create_task(self.ledger.subscribe_account(new_account))
+
+        encrypted = self.default_wallet.pack(encrypt_password or password)
         return {
-            'type': 'manifest',
+            'hash': self.jsonrpc_sync_hash(),
-            'generated': time.time(),
+            'data': encrypted.decode()
-            'status': hexlify(sha256(b''.join(status))),
-            'accounts': manifest
         }
 
     ADDRESS_DOC = """

lbrynet/extras/wallet/account.py

@@ -1,6 +1,7 @@
 import json
 import logging
 import binascii
+from hashlib import sha256
 
 from lbrynet.schema.validator import validate_claim_id
 from torba.client.baseaccount import BaseAccount
@@ -24,6 +25,13 @@ class Account(BaseAccount):
         super().__init__(*args, **kwargs)
         self.certificates = {}
 
+    @property
+    def hash(self) -> bytes:
+        h = sha256(json.dumps(self.to_dict(False)).encode())
+        for cert in sorted(self.certificates.keys()):
+            h.update(cert.encode())
+        return h.digest()
+
     def add_certificate_private_key(self, ref: TXORef, private_key):
         assert ref.id not in self.certificates, 'Trying to add a duplicate certificate.'
         self.certificates[ref.id] = private_key
@@ -176,9 +184,9 @@ class Account(BaseAccount):
         account.certificates = d.get('certificates', {})
         return account
 
-    def to_dict(self, with_certificates=True):
+    def to_dict(self, include_certificates=True):
         d = super().to_dict()
-        if with_certificates:
+        if include_certificates:
             d['certificates'] = self.certificates
         return d
 

tests/integration/test_sync.py

@@ -62,18 +62,29 @@ class AccountSynchronization(AsyncioTestCase):
         await self.daemon.stop()
 
     @mock.patch('time.time', mock.Mock(return_value=12345))
-    def test_manifest(self):
+    def test_sync(self):
-        self.account.certificates['abcdefg1234:0'] = '---PRIVATE KEY---'
+        starting_hash = 'fcafbb9bd3943d8d425a4c00d3982a4c6aaff763d5289f7852296f8ea882214f'
-        self.assertEqual({
+        self.assertEqual(self.daemon.jsonrpc_sync_hash(), starting_hash)
-            'type': 'manifest',
+        self.assertEqual(self.daemon.jsonrpc_sync_apply('password')['hash'], starting_hash)
-            'generated': 12345,
+        self.assertFalse(self.account.certificates)
-            'status': b'880fd3bef17c02d02710af94dcb69877407636432ad8cf17db2689be36fc52e4',
+
-            'accounts': [{
+        hash_w_cert = 'ef748d7777bb01ce9be6f87d3e46aeb31abbbe1648b1f2ddfa5aa9bcf0736a2d'
-                'account_id': 'n4ZRwP4QjKwsmXCfqUPqnx133i83Ha7GbW',
+        add_cert = (
-                'timestamp': 12345,
+            '5sx5Q4ruPFDSftJ3+5l0rKDEacDE7npsee2Pz+jsYTiNSBtDXt/fbvpKELpn6BWYDM1rqDCHDgZoy6609KbTCu'
-                'hash-data': b'c9a0e30c9cccd995e0c241a5d6e34308a291581dd858ffe51b307094fa621f8a',
+            'TqlYnrtMVpSz8QXc/Gzry2zXgtuuG6CAAvhntfELfwiJW4r1wvKDq30+IDrX8HIM5TiErLsLqfvfhc4t9Qfn5Y'
-                'hash-certificates': [
+            'IgJk9pYxu+xC7rJh+kYra+zu6JtEI9hdq+peXX6uAnqEKlRQCTLDPA6Z9Pk9Hdbhl9QJ3TVTNeTkMQyCZZ49SJ'
-                    b'IQlU7wyFap5scPGm4OgYOBa5bXx9Fy0KJOfeX2QbTN4='
+            'PtOghGXIA9Gtkp86nKvuzV7rKpVEJEe/mcUsBkQ/W9W/7bok3tOXBs7SCis0MMyYFbCQ1LVDy6RUD28UHp/P5O'
-                ]
+            '4kbxptuRzGKrkrQX00QEqzPuQwbbxuOMarGWUBP4USX6GmtK0e3AL1bUJzdJEuy937DdcvbhrzfxT0Jphjal5s'
-            }]
+            'BSDufxZaQcHLHOhjQ8DDnFscjbAChcjxCLgcYMtdxYGM0WmCU7vdKyWK7sULi+LSqPTf/75lYoW1FxXt3v/blX'
-        }, self.daemon.jsonrpc_account_manifest('password'))
+            'I3nJF5owVEZPx/5dNy95WDVCpQyDNd/Zw9ke2P+4d6hyMXbsz9Oei0q4BlKDM3MNGHd+MNSiX23xZq+FtTQdbw'
+            'ZOBhRTcQRB8VoR9M27acQApcdd2AXj0ZKrj/T+p8O0tuM0kWYOOAt6P/WxbU16im+WoR+4OTPggxu8r8SFFsXZ'
+            'EXYXT3tUSNzpU32OH2jXzo7P4Wa69s8u+X8RgA=='
+        )
+        self.assertEqual(self.daemon.jsonrpc_sync_apply('password', data=add_cert)['hash'], hash_w_cert)
+        self.assertEqual(self.daemon.jsonrpc_sync_hash(), hash_w_cert)
+        self.assertEqual(self.account.certificates, {'abcdefg1234:0': '---PRIVATE KEY---'})
+
+        # applying the same diff is idempotent
+        self.assertEqual(self.daemon.jsonrpc_sync_apply('password', data=add_cert)['hash'], hash_w_cert)
+        self.assertEqual(self.daemon.jsonrpc_sync_hash(), hash_w_cert)
+        self.assertEqual(self.account.certificates, {'abcdefg1234:0': '---PRIVATE KEY---'})