good stuff

lbrynet/extras/daemon/Daemon.py

@@ -1674,6 +1674,10 @@ class Daemon(metaclass=JSONRPCServerType):
         Usage:
             claim_search [<name> | --name=<name>] [--claim_id=<claim_id>] [--txid=<txid> --nout=<nout>]
                          [--channel_id=<channel_id>] [--channel_name=<channel_name>] [--is_controlling]
+                         [--order_by=<order_by>...]
+                         [--published_since=<published_since>] [--released_since=<released_since>]
+                         [--block_height=<block_height>] [--after_block_height=<after_block_height>]
+                         [--before_block_height=<before_block_height>]
                          [--any_tags=<any_tags>...] [--all_tags=<all_tags>...] [--not_tags=<not_tags>...]
                          [--any_languages=<any_languages>...] [--all_languages=<all_languages>...]
                          [--not_languages=<not_languages>...]
@@ -1689,6 +1693,20 @@ class Daemon(metaclass=JSONRPCServerType):
             --channel_id=<channel_id>       : (str) limit search to specific channel claim id (returns stream claims)
             --channel_name=<channel_name>   : (str) limit search to specific channel name (returns stream claims)
             --is_controlling                : (bool) limit to controlling claims for their respective name
+            --order_by=<order_by>           : (str) field to order by, default is descending order, to do an
+                                                    ascending order prepend ^ to the field name, eg. '^amount'
+                                                    available fields: 'name', 'block_height', 'release_time',
+                                                    'publish_time', 'amount', 'effective_amount', 'support_amount',
+                                                    'trending_amount'
+        --published_since=<published_since> : (int) limit to claims confirmed into blocks on or after
+                                                    this UTC timestamp
+          --released_since=<released_since> : (int) limit to claims self-described as having been
+                                                    released to the public on or after this UTC
+                                                    timestamp, when claim does not provide
+                                                    a release time the block time is used instead
+            --block_height=<block_height>   : (int) limit to claims at specific block height
+        --after_block_height=<after_block_height> : (int) limit to claims after specific block height
+        --before_block_height=<before_block_height> : (int) limit to claims before specific block height
             --any_tags=<any_tags>           : (list) find claims containing any of the tags
             --all_tags=<all_tags>           : (list) find claims containing every tag
             --not_tags=<not_tags>           : (list) find claims not containing any of these tags

lbrynet/extras/daemon/json_response_encoder.py

@@ -138,7 +138,7 @@ class JSONResponseEncoder(JSONEncoder):
             'hex': hexlify(tx.raw).decode(),
         }
 
-    def encode_output(self, txo, check_signature=True):
+    def encode_output(self, txo, check_signature=True, include_meta=True):
         tx_height = txo.tx_ref.height
         best_height = self.ledger.headers.height
         output = {
@@ -171,9 +171,10 @@ class JSONResponseEncoder(JSONEncoder):
                 'name': txo.claim_name,
                 'normalized': txo.normalized_name,
                 'claim_id': txo.claim_id,
-                'permanent_url': txo.permanent_url,
+                'permanent_url': txo.permanent_url
-                'meta': self.encode_claim_meta(txo.meta)
             })
+            if include_meta:
+                output['meta'] = self.encode_claim_meta(txo.meta)
             if txo.script.is_claim_name or txo.script.is_update_claim:
                 try:
                     output['value'] = txo.claim
@@ -181,7 +182,7 @@ class JSONResponseEncoder(JSONEncoder):
                     if self.include_protobuf:
                         output['protobuf'] = hexlify(txo.claim.to_bytes())
                     if txo.channel is not None:
-                        output['signing_channel'] = txo.channel
+                        output['signing_channel'] = self.encode_output(txo.channel, include_meta=False)
                         if check_signature and txo.claim.is_signed:
                             output['is_channel_signature_valid'] = False
                             if txo.channel:
@@ -191,10 +192,10 @@ class JSONResponseEncoder(JSONEncoder):
         return output
 
     def encode_claim_meta(self, meta):
-        if isinstance(meta.get('effective_amount'), int):
+        for key, value in meta.items():
-            meta['effective_amount'] = dewies_to_lbc(meta['effective_amount'])
+            if key.endswith('_amount') or key.startswith('trending_'):
-        if isinstance(meta.get('trending_amount'), int):
+                if isinstance(value, int):
-            meta['trending_amount'] = dewies_to_lbc(meta['trending_amount'])
+                    meta[key] = dewies_to_lbc(value)
         return meta
 
     def encode_input(self, txi):

lbrynet/schema/result.py

@@ -23,7 +23,14 @@ class Outputs:
             'is_controlling': message.is_controlling,
             'activation_height': message.activation_height,
             'effective_amount': message.effective_amount,
-            'trending_amount': message.trending_amount,
+            'support_amount': message.support_amount,
+            'claims_in_channel': message.claims_in_channel,
+            'trending_daily': message.trending_daily,
+            'trending_day_one': message.trending_day_one,
+            'trending_day_two': message.trending_day_two,
+            'trending_weekly': message.trending_weekly,
+            'trending_week_one': message.trending_week_one,
+            'trending_week_two': message.trending_week_two,
         }
         try:
             if txo.claim.is_channel:
@@ -89,14 +96,17 @@ class Outputs:
             txo_message.claim.is_controlling = bool(txo['is_controlling'])
             txo_message.claim.activation_height = txo['activation_height']
             txo_message.claim.effective_amount = txo['effective_amount']
-            txo_message.claim.trending_amount = txo['trending_amount']
+            txo_message.claim.support_amount = txo['support_amount']
             txo_message.claim.claims_in_channel = txo['claims_in_channel']
+            txo_message.claim.trending_daily = txo['trending_daily']
+            txo_message.claim.trending_day_one = txo['trending_day_one']
+            txo_message.claim.trending_day_two = txo['trending_day_two']
+            txo_message.claim.trending_weekly = txo['trending_weekly']
+            txo_message.claim.trending_week_one = txo['trending_week_one']
+            txo_message.claim.trending_week_two = txo['trending_week_two']
             if txo['channel_txo_hash']:
                 channel = txo_message.claim.channel
                 channel.height = txo['channel_height']
                 channel.tx_hash = txo['channel_txo_hash'][:32]
                 channel.nout, = struct.unpack('<I', txo['channel_txo_hash'][32:])
-                channel.claim.activation_height = txo['channel_activation_height']
-                channel.claim.effective_amount = txo['channel_effective_amount']
-                channel.claim.trending_amount = txo['channel_trending_amount']
         return page.SerializeToString()

lbrynet/schema/types/v2/result_pb2.py

@@ -19,7 +19,7 @@ DESCRIPTOR = _descriptor.FileDescriptor(
   package='pb',
   syntax='proto3',
   serialized_options=None,
-  serialized_pb=_b('\n\x0cresult.proto\x12\x02pb\"B\n\x07Outputs\x12\x18\n\x04txos\x18\x01 \x03(\x0b\x32\n.pb.Output\x12\r\n\x05total\x18\x02 \x01(\r\x12\x0e\n\x06offset\x18\x03 \x01(\r\"{\n\x06Output\x12\x0f\n\x07tx_hash\x18\x01 \x01(\x0c\x12\x0c\n\x04nout\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\x1e\n\x05\x63laim\x18\x07 \x01(\x0b\x32\r.pb.ClaimMetaH\x00\x12\x1a\n\x05\x65rror\x18\x0f \x01(\x0b\x32\t.pb.ErrorH\x00\x42\x06\n\x04meta\"\xa9\x01\n\tClaimMeta\x12\x1b\n\x07\x63hannel\x18\x01 \x01(\x0b\x32\n.pb.Output\x12\x16\n\x0eis_controlling\x18\x02 \x01(\x08\x12\x19\n\x11\x61\x63tivation_height\x18\x03 \x01(\r\x12\x18\n\x10\x65\x66\x66\x65\x63tive_amount\x18\x04 \x01(\x04\x12\x17\n\x0ftrending_amount\x18\x05 \x01(\x04\x12\x19\n\x11\x63laims_in_channel\x18\x06 \x01(\r\"i\n\x05\x45rror\x12\x1c\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x0e.pb.Error.Code\x12\x0c\n\x04text\x18\x02 \x01(\t\"4\n\x04\x43ode\x12\x10\n\x0cUNKNOWN_CODE\x10\x00\x12\r\n\tNOT_FOUND\x10\x01\x12\x0b\n\x07INVALID\x10\x02\x62\x06proto3')
+  serialized_pb=_b('\n\x0cresult.proto\x12\x02pb\"B\n\x07Outputs\x12\x18\n\x04txos\x18\x01 \x03(\x0b\x32\n.pb.Output\x12\r\n\x05total\x18\x02 \x01(\r\x12\x0e\n\x06offset\x18\x03 \x01(\r\"{\n\x06Output\x12\x0f\n\x07tx_hash\x18\x01 \x01(\x0c\x12\x0c\n\x04nout\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\x1e\n\x05\x63laim\x18\x07 \x01(\x0b\x32\r.pb.ClaimMetaH\x00\x12\x1a\n\x05\x65rror\x18\x0f \x01(\x0b\x32\t.pb.ErrorH\x00\x42\x06\n\x04meta\"\xc3\x02\n\tClaimMeta\x12\x1b\n\x07\x63hannel\x18\x01 \x01(\x0b\x32\n.pb.Output\x12\x16\n\x0eis_controlling\x18\x02 \x01(\x08\x12\x19\n\x11\x61\x63tivation_height\x18\x03 \x01(\r\x12\x18\n\x10\x65\x66\x66\x65\x63tive_amount\x18\x04 \x01(\x04\x12\x16\n\x0esupport_amount\x18\x05 \x01(\x04\x12\x19\n\x11\x63laims_in_channel\x18\x06 \x01(\r\x12\x16\n\x0etrending_daily\x18\x07 \x01(\x12\x12\x18\n\x10trending_day_one\x18\x08 \x01(\x04\x12\x18\n\x10trending_day_two\x18\t \x01(\x04\x12\x17\n\x0ftrending_weekly\x18\n \x01(\x12\x12\x19\n\x11trending_week_one\x18\x0b \x01(\x04\x12\x19\n\x11trending_week_two\x18\x0c \x01(\x04\"i\n\x05\x45rror\x12\x1c\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x0e.pb.Error.Code\x12\x0c\n\x04text\x18\x02 \x01(\t\"4\n\x04\x43ode\x12\x10\n\x0cUNKNOWN_CODE\x10\x00\x12\r\n\tNOT_FOUND\x10\x01\x12\x0b\n\x07INVALID\x10\x02\x62\x06proto3')
 )
 
 
@@ -45,8 +45,8 @@ _ERROR_CODE = _descriptor.EnumDescriptor(
   ],
   containing_type=None,
   serialized_options=None,
-  serialized_start=438,
+  serialized_start=592,
-  serialized_end=490,
+  serialized_end=644,
 )
 _sym_db.RegisterEnumDescriptor(_ERROR_CODE)
 
@@ -194,7 +194,7 @@ _CLAIMMETA = _descriptor.Descriptor(
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
-      name='trending_amount', full_name='pb.ClaimMeta.trending_amount', index=4,
+      name='support_amount', full_name='pb.ClaimMeta.support_amount', index=4,
       number=5, type=4, cpp_type=4, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
@@ -207,6 +207,48 @@ _CLAIMMETA = _descriptor.Descriptor(
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='trending_daily', full_name='pb.ClaimMeta.trending_daily', index=6,
+      number=7, type=18, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='trending_day_one', full_name='pb.ClaimMeta.trending_day_one', index=7,
+      number=8, type=4, cpp_type=4, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='trending_day_two', full_name='pb.ClaimMeta.trending_day_two', index=8,
+      number=9, type=4, cpp_type=4, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='trending_weekly', full_name='pb.ClaimMeta.trending_weekly', index=9,
+      number=10, type=18, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='trending_week_one', full_name='pb.ClaimMeta.trending_week_one', index=10,
+      number=11, type=4, cpp_type=4, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='trending_week_two', full_name='pb.ClaimMeta.trending_week_two', index=11,
+      number=12, type=4, cpp_type=4, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
   ],
   extensions=[
   ],
@@ -220,7 +262,7 @@ _CLAIMMETA = _descriptor.Descriptor(
   oneofs=[
   ],
   serialized_start=214,
-  serialized_end=383,
+  serialized_end=537,
 )
 
 
@@ -258,8 +300,8 @@ _ERROR = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=385,
+  serialized_start=539,
-  serialized_end=490,
+  serialized_end=644,
 )
 
 _OUTPUTS.fields_by_name['txos'].message_type = _OUTPUT

lbrynet/wallet/server/block_processor.py

@@ -79,11 +79,11 @@ class LBRYBlockProcessor(BlockProcessor):
         finally:
             self.sql.commit()
 
-    def advance_txs(self, height, txs):
+    def advance_txs(self, height, txs, header):
         timer = self.timer.sub_timers['advance_blocks']
-        undo = timer.run(super().advance_txs, height, txs, timer_name='super().advance_txs')
+        undo = timer.run(super().advance_txs, height, txs, header, timer_name='super().advance_txs')
-        timer.run(self.sql.advance_txs, height, txs, forward_timer=True)
+        timer.run(self.sql.advance_txs, height, txs, header, forward_timer=True)
-        if height % 10000 == 0:
+        if (height % 10000 == 0 or not self.db.first_sync) and self.logger.isEnabledFor(20):
             self.timer.show(height=height)
         return undo
 

lbrynet/wallet/server/db.py

@@ -23,8 +23,8 @@ def _apply_constraints_for_array_attributes(constraints, attr):
         values = ', '.join(
             f':$any_{attr}{i}' for i in range(len(any_items))
         )
-        constraints[f'claim.txo_hash__in#_any_{attr}'] = f"""
+        constraints[f'claim.claim_hash__in#_any_{attr}'] = f"""
-            SELECT DISTINCT txo_hash FROM {attr} WHERE {attr} IN ({values})
+            SELECT DISTINCT claim_hash FROM {attr} WHERE {attr} IN ({values})
         """
 
     all_items = constraints.pop(f'all_{attr}s', [])[:ATTRIBUTE_ARRAY_MAX_LENGTH]
@@ -36,9 +36,9 @@ def _apply_constraints_for_array_attributes(constraints, attr):
         values = ', '.join(
             f':$all_{attr}{i}' for i in range(len(all_items))
         )
-        constraints[f'claim.txo_hash__in#_all_{attr}'] = f"""
+        constraints[f'claim.claim_hash__in#_all_{attr}'] = f"""
-            SELECT txo_hash FROM {attr} WHERE {attr} IN ({values})
+            SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
-            GROUP BY txo_hash HAVING COUNT({attr}) = :$all_{attr}_count
+            GROUP BY claim_hash HAVING COUNT({attr}) = :$all_{attr}_count
         """
 
     not_items = constraints.pop(f'not_{attr}s', [])[:ATTRIBUTE_ARRAY_MAX_LENGTH]
@@ -49,14 +49,15 @@ def _apply_constraints_for_array_attributes(constraints, attr):
         values = ', '.join(
             f':$not_{attr}{i}' for i in range(len(not_items))
         )
-        constraints[f'claim.txo_hash__not_in#_not_{attr}'] = f"""
+        constraints[f'claim.claim_hash__not_in#_not_{attr}'] = f"""
-            SELECT DISTINCT txo_hash FROM {attr} WHERE {attr} IN ({values})
+            SELECT DISTINCT claim_hash FROM {attr} WHERE {attr} IN ({values})
         """
 
 
 class SQLDB:
 
-    TRENDING_BLOCKS = 300  # number of blocks over which to calculate trending
+    TRENDING_24_HOURS = 720
+    TRENDING_WEEK = TRENDING_24_HOURS * 7
 
     PRAGMAS = """
         pragma journal_mode=WAL;
@@ -71,17 +72,29 @@ class SQLDB:
             txo_hash bytes not null,
             tx_position integer not null,
             height integer not null,
-            amount integer not null,
             channel_hash bytes,
+            release_time integer,
+            publish_time integer,
             activation_height integer,
+            amount integer not null,
             effective_amount integer not null default 0,
             support_amount integer not null default 0,
-            trending_amount integer not null default 0
+            trending_daily integer not null default 0,
+            trending_day_one integer not null default 0,
+            trending_day_two integer not null default 0,
+            trending_weekly integer not null default 0,
+            trending_week_one integer not null default 0,
+            trending_week_two integer not null default 0
         );
         create index if not exists claim_normalized_idx on claim (normalized);
         create index if not exists claim_txo_hash_idx on claim (txo_hash);
         create index if not exists claim_channel_hash_idx on claim (channel_hash);
+        create index if not exists claim_release_time_idx on claim (release_time);
+        create index if not exists claim_publish_time_idx on claim (publish_time);
+        create index if not exists claim_height_idx on claim (height);
         create index if not exists claim_activation_height_idx on claim (activation_height);
+        create index if not exists claim_trending_daily_idx on claim (trending_daily);
+        create index if not exists claim_trending_weekly_idx on claim (trending_weekly);
     """
 
     CREATE_SUPPORT_TABLE = """
@@ -174,7 +187,7 @@ class SQLDB:
     def commit(self):
         self.execute('commit;')
 
-    def _upsertable_claims(self, txos: Set[Output], clear_first=False):
+    def _upsertable_claims(self, txos: Set[Output], header, clear_first=False):
         claim_hashes, claims, tags = [], [], []
         for txo in txos:
             tx = txo.tx_ref.tx
@@ -198,6 +211,8 @@ class SQLDB:
                 'height': tx.height,
                 'amount': txo.amount,
                 'channel_hash': None,
+                'publish_time': header['timestamp'],
+                'release_time': header['timestamp']
             }
             claims.append(claim_record)
 
@@ -207,6 +222,8 @@ class SQLDB:
                 #self.logger.exception(f"Could not parse claim protobuf for {tx.id}:{txo.position}.")
                 continue
 
+            if claim.is_stream and claim.stream.release_time:
+                claim_record['release_time'] = claim.stream.release_time
             claim_record['is_channel'] = claim.is_channel
             if claim.signing_channel_hash:
                 claim_record['channel_hash'] = sqlite3.Binary(claim.signing_channel_hash)
@@ -223,26 +240,27 @@ class SQLDB:
 
         return claims
 
-    def insert_claims(self, txos: Set[Output]):
+    def insert_claims(self, txos: Set[Output], header):
-        claims = self._upsertable_claims(txos)
+        claims = self._upsertable_claims(txos, header)
         if claims:
-            self.db.executemany(
+            self.db.executemany("""
-                "INSERT INTO claim ("
+                INSERT INTO claim (
-                "   claim_hash, normalized, claim_name, is_channel, txo_hash,"
+                    claim_hash, normalized, claim_name, is_channel, txo_hash, tx_position,
-                "   tx_position, height, amount, channel_hash) "
+                    height, amount, channel_hash, release_time, publish_time, activation_height)
-                "VALUES ("
+                VALUES (
-                "   :claim_hash, :normalized, :claim_name, :is_channel, :txo_hash,"
+                    :claim_hash, :normalized, :claim_name, :is_channel, :txo_hash, :tx_position,
-                "   :tx_position, :height, :amount, :channel_hash) ",
+                    :height, :amount, :channel_hash, :release_time, :publish_time,
-                claims
+                    CASE WHEN :normalized NOT IN (SELECT normalized FROM claimtrie) THEN :height END
-            )
+                )""", claims)
 
-    def update_claims(self, txos: Set[Output]):
+    def update_claims(self, txos: Set[Output], header):
-        claims = self._upsertable_claims(txos, clear_first=True)
+        claims = self._upsertable_claims(txos, header, clear_first=True)
         if claims:
             self.db.executemany(
                 "UPDATE claim SET "
                 "   is_channel=:is_channel, txo_hash=:txo_hash, tx_position=:tx_position,"
-                "   height=:height, amount=:amount, channel_hash=:channel_hash "
+                "   height=:height, amount=:amount, channel_hash=:channel_hash,"
+                "   release_time=:release_time, publish_time=:publish_time "
                 "WHERE claim_hash=:claim_hash;",
                 claims
             )
@@ -302,13 +320,37 @@ class SQLDB:
             ))
 
     def _update_trending_amount(self, height):
+        day_ago = height-self.TRENDING_24_HOURS
+        two_day_ago = height-self.TRENDING_24_HOURS*2
+        week_ago = height-self.TRENDING_WEEK
+        two_week_ago = height-self.TRENDING_WEEK*2
         self.execute(f"""
             UPDATE claim SET
-                trending_amount = COALESCE(
+                trending_day_one = COALESCE(
-                    (SELECT SUM(amount) FROM support WHERE support.claim_hash=claim.claim_hash
+                    (SELECT SUM(amount) FROM support WHERE claim_hash=claim.claim_hash
-                     AND support.height > {height-self.TRENDING_BLOCKS}), 0
+                     AND height >= {day_ago}), 0
+                ),
+                trending_day_two = COALESCE(
+                    (SELECT SUM(amount) FROM support WHERE claim_hash=claim.claim_hash
+                     AND {day_ago} > height and height >= {two_day_ago}
+                     ), 0
+                ),
+                trending_week_one = COALESCE(
+                    (SELECT SUM(amount) FROM support WHERE claim_hash=claim.claim_hash
+                     AND height >= {week_ago}
+                     ), 0
+                ),
+                trending_week_two = COALESCE(
+                    (SELECT SUM(amount) FROM support WHERE claim_hash=claim.claim_hash
+                     AND {week_ago} > height and height >= {two_week_ago}
+                     ), 0
                 )
         """)
+        self.execute(f"""
+            UPDATE claim SET
+                trending_daily = trending_day_one - trending_day_two,
+                trending_weekly = trending_week_one - trending_week_two
+        """)
 
     def _update_support_amount(self, claim_hashes):
         if claim_hashes:
@@ -320,106 +362,92 @@ class SQLDB:
                 WHERE claim_hash IN ({','.join('?' for _ in claim_hashes)})
             """, claim_hashes)
 
-    def _make_claims_without_competition_become_controlling(self, height, changed, timer):
+    def _update_effective_amount(self, height, claim_hashes=None):
-        if not changed:
+        self.execute(
-            return
+            f"UPDATE claim SET effective_amount = amount + support_amount "
+            f"WHERE activation_height = {height}"
+        )
+        if claim_hashes:
+            self.execute(
+                f"UPDATE claim SET effective_amount = amount + support_amount "
+                f"WHERE activation_height < {height} "
+                f"  AND claim_hash IN ({','.join('?' for _ in claim_hashes)})",
+                claim_hashes
+            )
 
-        t = timer.add_timer('insert into claimtrie')
+    def _calculate_activation_height(self, height):
-        t.start()
+        last_take_over_height = f"""COALESCE(
-        self.execute(f"""
+            (SELECT last_take_over_height FROM claimtrie
-            INSERT INTO claimtrie (normalized, claim_hash, last_take_over_height)
+            WHERE claimtrie.normalized=claim.normalized),
-            SELECT claim.normalized, claim.claim_hash, {height} FROM claim
+            {height}
-                WHERE normalized IN ({','.join('?' for _ in changed)}) AND
+        )
-                      normalized NOT IN (SELECT normalized FROM claimtrie)
+        """
-                GROUP BY normalized HAVING COUNT(*) = 1
-        """, list(changed))
-        t.stop()
-
-        t = timer.add_timer('set activation_height to current height for default winner')
-        t.start()
-        self.execute(f"""
-            UPDATE claim SET activation_height = {height}
-            WHERE (activation_height IS NULL OR activation_height > {height})
-              AND EXISTS(SELECT * FROM claimtrie WHERE claimtrie.claim_hash=claim.claim_hash)
-        """)
-        t.stop()
-
-        t = timer.add_timer('calculate activation_height for contentious claim name')
-        t.start()
         self.execute(f"""
             UPDATE claim SET activation_height = 
-                {height} +
+                {height} + min(4032, cast(({height} - {last_take_over_height}) / 32 AS INT))
-                min(4032, cast(({height} - (SELECT last_take_over_height FROM claimtrie
-                                            WHERE claimtrie.normalized=claim.normalized)) / 32 AS INT))
             WHERE activation_height IS NULL
         """)
-        t.stop()
 
-    def _update_effective_amount(self, constraints):
+    def _perform_overtake(self, height, changed):
-        where, values = constraints_to_sql(constraints)
+        constraint = f"normalized IN ({','.join('?' for _ in changed)}) OR " if changed else ""
-        self.execute("UPDATE claim SET effective_amount = amount + support_amount WHERE "+where, values)
-
-    def _perform_overtake(self, height, constraints):
-        where, values = constraints_to_sql(constraints)
         overtakes = self.execute(f"""
-            SELECT winner.normalized, winner.claim_hash FROM claimtrie JOIN (
+            SELECT winner.normalized, winner.claim_hash, claimtrie.claim_hash AS current_winner FROM (
-                SELECT normalized, claim_hash, MAX(effective_amount)
+                SELECT normalized, claim_hash FROM claim
-                FROM claim WHERE normalized IN (SELECT normalized FROM claim WHERE {where})
+                WHERE {constraint}
-                GROUP BY normalized
+                    normalized IN (SELECT normalized FROM claim WHERE activation_height={height})
-            ) AS winner USING (normalized) WHERE claimtrie.claim_hash <> winner.claim_hash
+                ORDER BY effective_amount, height, tx_position DESC
-        """, values)
+            ) AS winner LEFT JOIN claimtrie USING (normalized)
+            GROUP BY winner.normalized
+            HAVING current_winner IS NULL
+                OR current_winner <> winner.claim_hash
+        """, changed)
         for overtake in overtakes:
-            self.execute(
+            if overtake['current_winner']:
-                f"UPDATE claim SET activation_height = {height} WHERE normalized = ? "
-                f"AND (activation_height IS NULL OR activation_height > {height})",
-                (overtake['normalized'],)
-            )
                 self.execute(
                     f"UPDATE claimtrie SET claim_hash = ?, last_take_over_height = {height} "
                     f"WHERE normalized = ?",
                     (sqlite3.Binary(overtake['claim_hash']), overtake['normalized'])
                 )
+            else:
+                self.execute(
+                    f"INSERT INTO claimtrie (claim_hash, normalized, last_take_over_height) "
+                    f"VALUES (?, ?, {height})",
+                    (sqlite3.Binary(overtake['claim_hash']), overtake['normalized'])
+                )
+            self.execute(
+                f"UPDATE claim SET activation_height = {height} WHERE normalized = ? "
+                f"AND (activation_height IS NULL OR activation_height > {height})",
+                (overtake['normalized'],)
+            )
 
     def _copy(self, height):
         if height > 50:
             self.execute(f"DROP TABLE claimtrie{height-50}")
         self.execute(f"CREATE TABLE claimtrie{height} AS SELECT * FROM claimtrie")
 
-    def update_claimtrie(self, height, changed_names, recalc_claims, timer):
+    def update_claimtrie(self, height, changed_names, amount_affected_claim_hashes, timer):
-        binary_recalc_claims = [sqlite3.Binary(claim_hash) for claim_hash in recalc_claims]
+        binary_claim_hashes = [
+            sqlite3.Binary(claim_hash) for claim_hash in amount_affected_claim_hashes
+        ]
         r = timer.run
-        r(self._make_claims_without_competition_become_controlling, height, changed_names, forward_timer=True)
+        r(self._calculate_activation_height, height)
-        r(self._update_support_amount, binary_recalc_claims)
+        r(self._update_support_amount, binary_claim_hashes)
+
+        r(self._update_effective_amount, height, binary_claim_hashes)
+        r(self._perform_overtake, height, list(changed_names))
+
+        r(self._update_effective_amount, height)
+        r(self._perform_overtake, height, [])
+
         if not self.main.first_sync:
             r(self._update_trending_amount, height)
 
-        if binary_recalc_claims:
+    def advance_txs(self, height, all_txs, header, timer):
-            claims_filter = {
-                '__or': {
-                    'activation_height': height,
-                    '__and': {
-                        'claim_hash__in': binary_recalc_claims,
-                        'activation_height__lte': height
-                    }
-                }
-            }
-        else:
-            claims_filter = {
-                'activation_height': height
-            }
-
-        r(self._update_effective_amount, claims_filter)
-        r(self._perform_overtake, height, claims_filter)
-        r(self._update_effective_amount, {'activation_height': height})
-        r(self._perform_overtake, height, {'activation_height': height})
-        #r(self._copy, height)
-
-    def advance_txs(self, height, all_txs, timer):
         insert_claims = set()
         update_claims = set()
-        delete_claims = set()
+        delete_claim_hashes = set()
-        changed_names = set()
+        deleted_and_inserted_names = set()
-        recalc_claims = set()
+        amount_affected_claim_hashes = set()
         insert_supports = set()
         delete_supports = set()
         body_timer = timer.add_timer('body')
@@ -431,34 +459,69 @@ class SQLDB:
                 self.split_inputs_into_claims_supports_and_other, tx.inputs
             )
             body_timer.start()
-            delete_claims.update(spent_claims.keys())
+            delete_claim_hashes.update(spent_claims.keys())
-            changed_names.update(spent_claims.values())
+            deleted_and_inserted_names.update(spent_claims.values())
-            recalc_claims.update(spent_supports.values())
+            amount_affected_claim_hashes.update(spent_supports.values())
             delete_supports.update(spent_supports)
             for output in tx.outputs:
                 if output.is_support:
                     insert_supports.add(output)
-                    recalc_claims.add(output.claim_hash)
+                    amount_affected_claim_hashes.add(output.claim_hash)
                 elif output.script.is_claim_name:
                     insert_claims.add(output)
-                    recalc_claims.add(output.claim_hash)
+                    try:
-                    changed_names.add(output.normalized_name)
+                        deleted_and_inserted_names.add(output.normalized_name)
+                    except:
+                        self.logger.exception(
+                            f"Could not decode claim name for claim_id: {output.claim_id}, "
+                            f"txid: {output.tx_ref.id}, nout: {output.position}.")
+                        print(output.script.values['claim_name'])
+                        continue
                 elif output.script.is_update_claim:
                     claim_hash = output.claim_hash
-                    if claim_hash in delete_claims:
+                    if claim_hash in delete_claim_hashes:
-                        delete_claims.remove(claim_hash)
+                        delete_claim_hashes.remove(claim_hash)
                     update_claims.add(output)
-                    recalc_claims.add(claim_hash)
+                    amount_affected_claim_hashes.add(claim_hash)
             body_timer.stop()
         r = timer.run
-        r(self.delete_claims, delete_claims)
+        r(self.delete_claims, delete_claim_hashes)
         r(self.delete_supports, delete_supports)
-        r(self.insert_claims, insert_claims)
+        r(self.insert_claims, insert_claims, header)
-        r(self.update_claims, update_claims)
+        r(self.update_claims, update_claims, header)
         r(self.insert_supports, insert_supports)
-        r(self.update_claimtrie, height, changed_names, recalc_claims, forward_timer=True)
+        r(self.update_claimtrie, height,
+          deleted_and_inserted_names,
+          amount_affected_claim_hashes,
+          forward_timer=True)
 
     def get_claims(self, cols, **constraints):
+        if 'order_by' in constraints:
+            sql_order_by = []
+            for order_by in constraints['order_by']:
+                is_asc = order_by.startswith('^')
+                column = order_by[1:] if is_asc else order_by
+                if column not in self.ORDER_FIELDS:
+                    raise NameError(f'{column} is not a valid order_by field')
+                elif column == 'name':
+                    column = 'normalized'
+                sql_order_by.append(
+                    f"claim.{column} ASC" if is_asc else f"claim.{column} DESC"
+                )
+            constraints['order_by'] = sql_order_by
+
+        ops = {'<=': '__lte', '>=': '__gte', '<': '__lt', '>': '__gt'}
+        for constraint in self.INTEGER_PARAMS:
+            if constraint in constraints:
+                value = constraints.pop(constraint)
+                postfix = ''
+                if isinstance(value, str):
+                    if len(value) >= 2 and value[:2] in ops:
+                        postfix, value = ops[value[:2]], int(value[2:])
+                    elif len(value) >= 1 and value[0] in ops:
+                        postfix, value = ops[value[0]], int(value[1:])
+                constraints[f'claim.{constraint}{postfix}'] = value
+
         if 'is_controlling' in constraints:
             if {'sequence', 'amount_order'}.isdisjoint(constraints):
                 constraints['claimtrie.claim_hash__is_not_null'] = ''
@@ -503,6 +566,7 @@ class SQLDB:
         _apply_constraints_for_array_attributes(constraints, 'language')
         _apply_constraints_for_array_attributes(constraints, 'location')
 
+        try:
             return self.db.execute(*query(
                 f"""
                 SELECT {cols} FROM claim
@@ -510,6 +574,16 @@ class SQLDB:
                 LEFT JOIN claim as channel ON (claim.channel_hash=channel.claim_hash)
                 """, **constraints
             )).fetchall()
+        except:
+            self.logger.exception('Failed to execute claim search query:')
+            print(query(
+                f"""
+            SELECT {cols} FROM claim
+            LEFT JOIN claimtrie USING (claim_hash)
+            LEFT JOIN claim as channel ON (claim.channel_hash=channel.claim_hash)
+            """, **constraints
+            ))
+            raise
 
     def get_claims_count(self, **constraints):
         constraints.pop('offset', None)
@@ -523,26 +597,36 @@ class SQLDB:
             """
             claimtrie.claim_hash as is_controlling,
             claim.claim_hash, claim.txo_hash, claim.height,
-            claim.activation_height, claim.effective_amount, claim.trending_amount,
+            claim.activation_height, claim.effective_amount, claim.support_amount,
-            channel.txo_hash as channel_txo_hash, channel.height as channel_height,
+            claim.trending_daily, claim.trending_day_one, claim.trending_day_two,
-            channel.activation_height as channel_activation_height,
+            claim.trending_weekly, claim.trending_week_one, claim.trending_week_two,
-            channel.effective_amount as channel_effective_amount,
-            channel.trending_amount as channel_trending_amount,
             CASE WHEN claim.is_channel=1 THEN (
                 SELECT COUNT(*) FROM claim as claim_in_channel
                 WHERE claim_in_channel.channel_hash=claim.claim_hash
-             ) ELSE 0 END AS claims_in_channel
+             ) ELSE 0 END AS claims_in_channel,
+            channel.txo_hash as channel_txo_hash, channel.height as channel_height
             """, **constraints
         )
 
+    INTEGER_PARAMS = {
+        'height', 'release_time', 'publish_time',
+        'amount', 'effective_amount', 'support_amount',
+        'trending_daily', 'trending_day_one', 'trending_day_two',
+        'trending_weekly', 'trending_week_one', 'trending_week_two'
+    }
+
     SEARCH_PARAMS = {
         'name', 'claim_id', 'txid', 'nout',
         'channel', 'channel_id', 'channel_name',
         'any_tags', 'all_tags', 'not_tags',
         'any_locations', 'all_locations', 'not_locations',
         'any_languages', 'all_languages', 'not_languages',
-        'is_controlling', 'limit', 'offset'
+        'is_controlling', 'limit', 'offset', 'order_by'
-    }
+    } | INTEGER_PARAMS
+
+    ORDER_FIELDS = {
+        'name',
+    } | INTEGER_PARAMS
 
     def search(self, constraints) -> Tuple[List, int, int]:
         assert set(constraints).issubset(self.SEARCH_PARAMS), \
@@ -550,7 +634,8 @@ class SQLDB:
         total = self.get_claims_count(**constraints)
         constraints['offset'] = abs(constraints.get('offset', 0))
         constraints['limit'] = min(abs(constraints.get('limit', 10)), 50)
-        constraints['order_by'] = ["claim.height DESC", "claim.normalized ASC"]
+        if 'order_by' not in constraints:
+            constraints['order_by'] = ["height", "^name"]
         txo_rows = self._search(**constraints)
         return txo_rows, constraints['offset'], total
 

tests/integration/test_claim_commands.py

@@ -148,6 +148,22 @@ class ClaimSearchCommand(CommandTestCase):
         await self.assertFindsClaims([claim3], all_tags=['abc', 'ghi'], any_tags=['jkl'], not_tags=['mno'])
         await self.assertFindsClaims([claim4, claim3, claim2], all_tags=['abc'], any_tags=['def', 'ghi'])
 
+    async def test_order_by(self):
+        height = await self.ledger.network.get_server_height()
+        claims = [await self.stream_create(f'claim{i}') for i in range(5)]
+
+        await self.assertFindsClaims(claims, order_by=["^height"])
+        await self.assertFindsClaims(list(reversed(claims)), order_by=["height"])
+
+        await self.assertFindsClaims([claims[0]], height=height+1)
+        await self.assertFindsClaims([claims[4]], height=height+5)
+        await self.assertFindsClaims(claims[:1], height=f'<{height+2}', order_by=["^height"])
+        await self.assertFindsClaims(claims[:2], height=f'<={height+2}', order_by=["^height"])
+        await self.assertFindsClaims(claims[2:], height=f'>{height+2}', order_by=["^height"])
+        await self.assertFindsClaims(claims[1:], height=f'>={height+2}', order_by=["^height"])
+
+        await self.assertFindsClaims(claims, order_by=["^name"])
+
 
 class ChannelCommands(CommandTestCase):
 

tests/unit/wallet/server/test_sqldb.py

@@ -67,6 +67,13 @@ class TestSQLDB(unittest.TestCase):
             Input.spend(claim)
         )
 
+    def get_stream_abandon(self, tx):
+        claim = Transaction(tx[0].serialize()).outputs[0]
+        return self._make_tx(
+            Output.pay_pubkey_hash(claim.amount, b'abc'),
+            Input.spend(claim)
+        )
+
     def get_support(self, tx, amount):
         claim = Transaction(tx[0].serialize()).outputs[0]
         return self._make_tx(
@@ -101,10 +108,8 @@ class TestSQLDB(unittest.TestCase):
         return accepted
 
     def advance(self, height, txs):
-        #for skipped_height in range(self._current_height+1, height):
-        #    self.sql.advance_txs(skipped_height, [], self.timer)
         self._current_height = height
-        self.sql.advance_txs(height, txs, self.timer)
+        self.sql.advance_txs(height, txs, {'timestamp': 1}, self.timer)
 
     def state(self, controlling=None, active=None, accepted=None):
         self.assertEqual(controlling or [], self.get_controlling())
@@ -175,3 +180,59 @@ class TestSQLDB(unittest.TestCase):
             ],
             accepted=[]
         )
+
+    def test_competing_claims_in_single_block_height_wins(self):
+        advance, state = self.advance, self.state
+        stream = self.get_stream('Claim A', 10*COIN)
+        stream2 = self.get_stream('Claim B', 10*COIN)
+        advance(13, [stream, stream2])
+        state(
+            controlling=('Claim A', 10*COIN, 10*COIN, 13),
+            active=[('Claim B', 10*COIN, 10*COIN, 13)],
+            accepted=[]
+        )
+
+    def test_competing_claims_in_single_block_effective_amount_wins(self):
+        advance, state = self.advance, self.state
+        stream = self.get_stream('Claim A', 10*COIN)
+        stream2 = self.get_stream('Claim B', 11*COIN)
+        advance(13, [stream, stream2])
+        state(
+            controlling=('Claim B', 11*COIN, 11*COIN, 13),
+            active=[('Claim A', 10*COIN, 10*COIN, 13)],
+            accepted=[]
+        )
+
+    def test_winning_claim_deleted(self):
+        advance, state = self.advance, self.state
+        stream = self.get_stream('Claim A', 10*COIN)
+        stream2 = self.get_stream('Claim B', 11*COIN)
+        advance(13, [stream, stream2])
+        state(
+            controlling=('Claim B', 11*COIN, 11*COIN, 13),
+            active=[('Claim A', 10*COIN, 10*COIN, 13)],
+            accepted=[]
+        )
+        advance(14, [self.get_stream_abandon(stream2)])
+        state(
+            controlling=('Claim A', 10*COIN, 10*COIN, 13),
+            active=[],
+            accepted=[]
+        )
+
+    def test_winning_claim_deleted_and_new_claim_becomes_winner(self):
+        advance, state = self.advance, self.state
+        stream = self.get_stream('Claim A', 10*COIN)
+        stream2 = self.get_stream('Claim B', 11*COIN)
+        advance(13, [stream, stream2])
+        state(
+            controlling=('Claim B', 11*COIN, 11*COIN, 13),
+            active=[('Claim A', 10*COIN, 10*COIN, 13)],
+            accepted=[]
+        )
+        advance(15, [self.get_stream_abandon(stream2), self.get_stream('Claim C', 12*COIN)])
+        state(
+            controlling=('Claim C', 12*COIN, 12*COIN, 15),
+            active=[('Claim A', 10*COIN, 10*COIN, 13)],
+            accepted=[]
+        )