save all claims, even bad ones, also fix cli

lbry/blockchain/sync/claims.py

@@ -127,19 +127,16 @@ def claims_insert(
             )
             i = 0
             for row in rows:
-                metadata = claim_metadata[i] if i < len(claim_metadata) else None
+                metadata = claim_metadata[i] if i < len(claim_metadata) else {}
-                if metadata is None:
+                if metadata and metadata['claim_hash'] == row.claim_hash:
-                    break
+                    i += 1
-                if metadata['claim_hash'] != row.claim_hash:
-                    continue
-                i += 1
                 txo, extra = row_to_claim_for_saving(row)
                 extra.update({
-                    'short_url': metadata['short_url'],
+                    'short_url': metadata.get('short_url'),
-                    'creation_height': metadata['creation_height'],
+                    'creation_height': metadata.get('creation_height'),
-                    'activation_height': metadata['activation_height'],
+                    'activation_height': metadata.get('activation_height'),
-                    'expiration_height': metadata['expiration_height'],
+                    'expiration_height': metadata.get('expiration_height'),
-                    'takeover_height': metadata['takeover_height'],
+                    'takeover_height': metadata.get('takeover_height'),
                 })
                 loader.add_claim(txo, **extra)
             if len(loader.claims) >= 25_000:
@@ -154,10 +151,11 @@ def claims_constraints_and_indexes(p: ProgressContext):
         with p.ctx.engine.connect() as c:
             c.execute(text("COMMIT;"))
             c.execute(text("VACUUM ANALYZE claim;"))
+            c.execute(text("VACUUM ANALYZE tag;"))
     p.step()
     for constraint in pg_add_claim_and_tag_constraints_and_indexes:
         if p.ctx.is_postgres:
-            p.ctx.execute(constraint)
+            p.ctx.execute(text(constraint))
         p.step()
 
 

lbry/blockchain/sync/supports.py

@@ -73,7 +73,7 @@ def supports_constraints_and_indexes(p: ProgressContext):
     p.step()
     for constraint in pg_add_support_constraints_and_indexes:
         if p.ctx.is_postgres:
-            p.ctx.execute(constraint)
+            p.ctx.execute(text(constraint))
         p.step()
 
 

lbry/console.py

@@ -366,6 +366,7 @@ class Advanced(Basic):
                 bar.update(d['done'][0] - bar.n)
             if d['done'][0] == -1 or d['done'][0] == bar.total:
                 bar.close()
+            self.bars.pop(bar_name)
 
     def sync_main(self, name, d):
         bar = self.bars.get(name)
@@ -382,9 +383,10 @@ class Advanced(Basic):
                     if child_name.startswith(base_name):
                         child_bar.close()
                 bar.close()
+                self.bars.pop(name)
             else:
-                if len(d['done']) > 1:
+                if len(d['done']) == 2:
-                    bar.update(d['done']-bar.n)
+                    bar.update((d['done'][0]-bar.n, d['done'][1]-bar.n2))
                 else:
                     bar.update(d['done'][0]-bar.n)
 
@@ -412,6 +414,7 @@ class Advanced(Basic):
                 bar.update(diff)
             if d['done'][0] == -1 or d['done'][0] == bar.total:
                 bar.close()
+                self.bars.pop(bar_name)
 
     def update_other_bars(self, e, d):
         if d['total'] == 0:

lbry/db/query_context.py

@@ -626,7 +626,7 @@ class BulkLoader:
             claim_name = txo.claim_name.replace('\x00', '')
             normalized_name = txo.normalized_name
         except UnicodeDecodeError:
-            return self
+            claim_name = normalized_name = ''
         d, tags = self.claim_to_rows(txo, **extra)
         d['claim_hash'] = txo.claim_hash
         d['claim_id'] = txo.claim_id