From 39996d761209e3b099904d6c44d432e7a39bbd4d Mon Sep 17 00:00:00 2001 From: "Brendon J. Brewer" Date: Wed, 5 Feb 2020 07:47:38 +1300 Subject: [PATCH 1/4] Only create file during install if logging is turned on --- lbry/wallet/server/db/trending/ar.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lbry/wallet/server/db/trending/ar.py b/lbry/wallet/server/db/trending/ar.py index 748a57ad4..bc20da4c7 100644 --- a/lbry/wallet/server/db/trending/ar.py +++ b/lbry/wallet/server/db/trending/ar.py @@ -28,6 +28,11 @@ TRENDING_LOG = True def install(connection): check_trending_values(connection) + if TRENDING_LOG: + f = open("trending_ar.log", "w") + f.close() + + CREATE_TREND_TABLE = "" @@ -53,8 +58,6 @@ def check_trending_values(connection): COMMIT;""") print("done.") - # Create the index - c.execute("create index if not exists claim_id_only_idx on claim (claim_id);") def spike_height(trending_score, x, x_old, time_boost=1.0): @@ -170,8 +173,6 @@ def test_trending(): # One global instance # pylint: disable=C0103 trending_data = TrendingData() -f = open("trending_ar.log", "w") -f.close() def run(db, height, final_height, recalculate_claim_hashes): From 17d8a3e5d6691de8f6988522eddbe07a89d7eae2 Mon Sep 17 00:00:00 2001 From: "Brendon J. Brewer" Date: Wed, 5 Feb 2020 07:51:29 +1300 Subject: [PATCH 2/4] Use claim_hash not claim_id --- lbry/wallet/server/db/trending/ar.py | 31 +++++++++++++++------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/lbry/wallet/server/db/trending/ar.py b/lbry/wallet/server/db/trending/ar.py index bc20da4c7..3987c3d07 100644 --- a/lbry/wallet/server/db/trending/ar.py +++ b/lbry/wallet/server/db/trending/ar.py @@ -24,21 +24,23 @@ DECAY_PER_RENORM = DECAY**(RENORM_INTERVAL) TRENDING_LOG = True -# Stubs def install(connection): + """ + Install the AR trending algorithm. + """ check_trending_values(connection) if TRENDING_LOG: f = open("trending_ar.log", "w") f.close() - +# Stub CREATE_TREND_TABLE = "" def check_trending_values(connection): """ - If the trending values appear to be based on the standard algorithm, + If the trending values appear to be based on the zscore algorithm, reset them. This will allow resyncing from a standard snapshot. """ c = connection.cursor() @@ -109,14 +111,14 @@ class TrendingData: # Have all claims been read from db yet? self.initialised = False - def insert_claim_from_load(self, claim_id, trending_score, total_amount): + def insert_claim_from_load(self, claim_hash, trending_score, total_amount): assert not self.initialised - self.claims[claim_id] = {"trending_score": trending_score, + self.claims[claim_hash] = {"trending_score": trending_score, "total_amount": total_amount, "changed": False} - def update_claim(self, claim_id, total_amount, time_boost=1.0): + def update_claim(self, claim_hash, total_amount, time_boost=1.0): """ Update trending data for a claim, given its new total amount. """ @@ -124,8 +126,8 @@ class TrendingData: # Extract existing total amount and trending score # or use starting values if the claim is new - if claim_id in self.claims: - old_state = copy.deepcopy(self.claims[claim_id]) + if claim_hash in self.claims: + old_state = copy.deepcopy(self.claims[claim_hash]) else: old_state = {"trending_score": 0.0, "total_amount": 0.0, @@ -141,9 +143,9 @@ class TrendingData: old_state["total_amount"], time_boost) trending_score = old_state["trending_score"] + spike - self.claims[claim_id] = {"total_amount": total_amount, - "trending_score": trending_score, - "changed": True} + self.claims[claim_hash] = {"total_amount": total_amount, + "trending_score": trending_score, + "changed": True} @@ -178,6 +180,7 @@ def run(db, height, final_height, recalculate_claim_hashes): if height < final_height - 5*HALF_LIFE: trending_log("Skipping AR trending at block {h}.\n".format(h=height)) + return start = time.time() @@ -213,7 +216,7 @@ def run(db, height, final_height, recalculate_claim_hashes): if not trending_data.initialised: # On fresh launch for row in db.execute(""" - SELECT claim_id, trending_mixed, + SELECT claim_hash, trending_mixed, (amount + support_amount) AS total_amount FROM claim; @@ -222,7 +225,7 @@ def run(db, height, final_height, recalculate_claim_hashes): trending_data.initialised = True else: for row in db.execute(f""" - SELECT claim_id, + SELECT claim_hash, (amount + support_amount) AS total_amount FROM claim @@ -249,7 +252,7 @@ def run(db, height, final_height, recalculate_claim_hashes): trending_log("{n} scores to write...".format(n=len(the_list))) - db.executemany("UPDATE claim SET trending_mixed=? WHERE claim_id=?;", + db.executemany("UPDATE claim SET trending_mixed=? WHERE claim_hash=?;", the_list) trending_log("done.\n") From 94524f7330393284770499031255b1eaf659b2bc Mon Sep 17 00:00:00 2001 From: "Brendon J. Brewer" Date: Wed, 5 Feb 2020 09:19:09 +1300 Subject: [PATCH 3/4] lint --- lbry/wallet/server/db/trending/ar.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lbry/wallet/server/db/trending/ar.py b/lbry/wallet/server/db/trending/ar.py index 3987c3d07..265e54fa1 100644 --- a/lbry/wallet/server/db/trending/ar.py +++ b/lbry/wallet/server/db/trending/ar.py @@ -114,8 +114,8 @@ class TrendingData: def insert_claim_from_load(self, claim_hash, trending_score, total_amount): assert not self.initialised self.claims[claim_hash] = {"trending_score": trending_score, - "total_amount": total_amount, - "changed": False} + "total_amount": total_amount, + "changed": False} def update_claim(self, claim_hash, total_amount, time_boost=1.0): From 3697d9e1f0566a49d7e2264445c5f6ae8358d039 Mon Sep 17 00:00:00 2001 From: "Brendon J. Brewer" Date: Wed, 5 Feb 2020 14:07:41 +1300 Subject: [PATCH 4/4] Tweaked some parameters, back to 134 --- lbry/wallet/server/db/trending/ar.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lbry/wallet/server/db/trending/ar.py b/lbry/wallet/server/db/trending/ar.py index 265e54fa1..e53579005 100644 --- a/lbry/wallet/server/db/trending/ar.py +++ b/lbry/wallet/server/db/trending/ar.py @@ -31,7 +31,7 @@ def install(connection): check_trending_values(connection) if TRENDING_LOG: - f = open("trending_ar.log", "w") + f = open("trending_ar.log", "a") f.close() # Stub @@ -61,7 +61,6 @@ def check_trending_values(connection): print("done.") - def spike_height(trending_score, x, x_old, time_boost=1.0): """ Compute the size of a trending spike. @@ -76,8 +75,9 @@ def spike_height(trending_score, x, x_old, time_boost=1.0): # Softened change in amount counts more for minnows if delta > 0.0: - multiplier = 1.0/math.sqrt(x + 1.0) - softened_change_in_amount *= multiplier + if trending_score >= 0.0: + multiplier = 0.1/((trending_score/time_boost + softened_change_in_amount) + 1.0) + softened_change_in_amount *= multiplier else: softened_change_in_amount *= -1.0