From 0dff82c31cb8032d6ae7465efd29bf95fe7255ec Mon Sep 17 00:00:00 2001
From: Lex Berezhny <lex@damoti.com>
Date: Mon, 13 Jul 2020 23:58:49 -0400
Subject: [PATCH] save all claims, even bad ones, also fix cli

---
 lbry/blockchain/sync/claims.py   | 22 ++++++++++------------
 lbry/blockchain/sync/supports.py |  2 +-
 lbry/console.py                  |  7 +++++--
 lbry/db/query_context.py         |  2 +-
 4 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/lbry/blockchain/sync/claims.py b/lbry/blockchain/sync/claims.py
index 2086020f1..a7b256e07 100644
--- a/lbry/blockchain/sync/claims.py
+++ b/lbry/blockchain/sync/claims.py
@@ -127,19 +127,16 @@ def claims_insert(
             )
             i = 0
             for row in rows:
-                metadata = claim_metadata[i] if i < len(claim_metadata) else None
-                if metadata is None:
-                    break
-                if metadata['claim_hash'] != row.claim_hash:
-                    continue
-                i += 1
+                metadata = claim_metadata[i] if i < len(claim_metadata) else {}
+                if metadata and metadata['claim_hash'] == row.claim_hash:
+                    i += 1
                 txo, extra = row_to_claim_for_saving(row)
                 extra.update({
-                    'short_url': metadata['short_url'],
-                    'creation_height': metadata['creation_height'],
-                    'activation_height': metadata['activation_height'],
-                    'expiration_height': metadata['expiration_height'],
-                    'takeover_height': metadata['takeover_height'],
+                    'short_url': metadata.get('short_url'),
+                    'creation_height': metadata.get('creation_height'),
+                    'activation_height': metadata.get('activation_height'),
+                    'expiration_height': metadata.get('expiration_height'),
+                    'takeover_height': metadata.get('takeover_height'),
                 })
                 loader.add_claim(txo, **extra)
             if len(loader.claims) >= 25_000:
@@ -154,10 +151,11 @@ def claims_constraints_and_indexes(p: ProgressContext):
         with p.ctx.engine.connect() as c:
             c.execute(text("COMMIT;"))
             c.execute(text("VACUUM ANALYZE claim;"))
+            c.execute(text("VACUUM ANALYZE tag;"))
     p.step()
     for constraint in pg_add_claim_and_tag_constraints_and_indexes:
         if p.ctx.is_postgres:
-            p.ctx.execute(constraint)
+            p.ctx.execute(text(constraint))
         p.step()
 
 
diff --git a/lbry/blockchain/sync/supports.py b/lbry/blockchain/sync/supports.py
index 6db46a262..670b13b7e 100644
--- a/lbry/blockchain/sync/supports.py
+++ b/lbry/blockchain/sync/supports.py
@@ -73,7 +73,7 @@ def supports_constraints_and_indexes(p: ProgressContext):
     p.step()
     for constraint in pg_add_support_constraints_and_indexes:
         if p.ctx.is_postgres:
-            p.ctx.execute(constraint)
+            p.ctx.execute(text(constraint))
         p.step()
 
 
diff --git a/lbry/console.py b/lbry/console.py
index b5b6bc1e5..cc78644df 100644
--- a/lbry/console.py
+++ b/lbry/console.py
@@ -366,6 +366,7 @@ class Advanced(Basic):
                 bar.update(d['done'][0] - bar.n)
             if d['done'][0] == -1 or d['done'][0] == bar.total:
                 bar.close()
+            self.bars.pop(bar_name)
 
     def sync_main(self, name, d):
         bar = self.bars.get(name)
@@ -382,9 +383,10 @@ class Advanced(Basic):
                     if child_name.startswith(base_name):
                         child_bar.close()
                 bar.close()
+                self.bars.pop(name)
             else:
-                if len(d['done']) > 1:
-                    bar.update(d['done']-bar.n)
+                if len(d['done']) == 2:
+                    bar.update((d['done'][0]-bar.n, d['done'][1]-bar.n2))
                 else:
                     bar.update(d['done'][0]-bar.n)
 
@@ -412,6 +414,7 @@ class Advanced(Basic):
                 bar.update(diff)
             if d['done'][0] == -1 or d['done'][0] == bar.total:
                 bar.close()
+                self.bars.pop(bar_name)
 
     def update_other_bars(self, e, d):
         if d['total'] == 0:
diff --git a/lbry/db/query_context.py b/lbry/db/query_context.py
index e6b73ff07..777ca3bbe 100644
--- a/lbry/db/query_context.py
+++ b/lbry/db/query_context.py
@@ -626,7 +626,7 @@ class BulkLoader:
             claim_name = txo.claim_name.replace('\x00', '')
             normalized_name = txo.normalized_name
         except UnicodeDecodeError:
-            return self
+            claim_name = normalized_name = ''
         d, tags = self.claim_to_rows(txo, **extra)
         d['claim_hash'] = txo.claim_hash
         d['claim_id'] = txo.claim_id