moved executing create index queries outside of a tx

lbry/lbry/wallet/server/block_processor.py

@@ -79,6 +79,10 @@ class LBRYBlockProcessor(BlockProcessor):
             raise
         finally:
             self.sql.commit()
+        if self.db.first_sync and blocks == self.daemon.cached_height():
+            self.timer.run(self.db.executescript, self.sql.SEARCH_INDEXES, timer_name='executing SEARCH_INDEXES')
+            if self.env.individual_tag_indexes:
+                self.timer.run(self.db.executescript, self.sql.TAG_INDEXES, timer_name='executing TAG_INDEXES')
         for cache in self.search_cache.values():
             cache.clear()
 

lbry/lbry/wallet/server/db/writer.py

@@ -116,7 +116,6 @@ class SQLDB:
     SEARCH_INDEXES = """
         -- used by any tag clouds
         create index if not exists tag_tag_idx on tag (tag, claim_hash);
-        {custom_tags_indexes}
 
         -- common ORDER BY
         create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash, release_time);
@@ -139,10 +138,12 @@ class SQLDB:
         create index if not exists claim_fee_currency_idx on claim (fee_currency);
 
         create index if not exists claim_signature_valid_idx on claim (signature_valid);
-    """.format(custom_tags_indexes='\n'.join(
-        f'create unique index if not exists tag_{tag_key}_idx on tag (tag, claim_hash) WHERE tag="{tag_value}";'
+    """
+
+    TAG_INDEXES = '\n'.join(
+        f"create unique index if not exists tag_{tag_key}_idx on tag (tag, claim_hash) WHERE tag='{tag_value}';"
         for tag_value, tag_key in COMMON_TAGS.items()
-    ))
+    )
 
     CREATE_TABLES_QUERY = (
         PRAGMAS +
@@ -699,9 +700,6 @@ class SQLDB:
         r(self.update_claimtrie, height, recalculate_claim_hashes, deleted_claim_names, forward_timer=True)
         r(calculate_trending, self.db, height, self.main.first_sync, daemon_height)
 
-        if self.main.first_sync and height == daemon_height:
-            self.db.executescript(self.SEARCH_INDEXES)
-
 
 class LBRYDB(DB):
 

torba/torba/orchstr8/node.py

@@ -206,7 +206,8 @@ class SPVNode:
             'HOST': self.hostname,
             'TCP_PORT': str(self.port),
             'SESSION_TIMEOUT': str(self.session_timeout),
-            'MAX_QUERY_WORKERS': '0'
+            'MAX_QUERY_WORKERS': '0',
+            'INDIVIDUAL_TAG_INDEXES': '',
         }
         # TODO: don't use os.environ
         os.environ.update(conf)

torba/torba/server/env.py

@@ -38,6 +38,7 @@ class Env:
         self.db_dir = self.required('DB_DIRECTORY')
         self.db_engine = self.default('DB_ENGINE', 'leveldb')
         self.max_query_workers = self.integer('MAX_QUERY_WORKERS', None)
+        self.individual_tag_indexes = self.boolean('INDIVIDUAL_TAG_INDEXES', True)
         self.track_metrics = self.boolean('TRACK_METRICS', False)
         self.websocket_host = self.default('WEBSOCKET_HOST', self.host)
         self.websocket_port = self.integer('WEBSOCKET_PORT', None)