forked from LBRYCommunity/lbry-sdk
add timestamps to checkpoints and estimate from chunk boundaries
This commit is contained in:
Victor Shyba
parent
37129bce58
commit
a2102b91a0
4 changed files with 121 additions and 2 deletions
|
|
@ -740,4 +740,100 @@ HASHES = {
|
|||
738000: 'aebdf15b23eb7a37600f67d45bf6586b1d5bff3d5f3459adc2f6211ab3dd0bcb',
|
||||
739000: '3f5a894ac42f95f7d54ce25c42ea0baf1a05b2da0e9406978de0dc53484d8b04',
|
||||
740000: '55debc22f995d844eafa0a90296c9f4f433e2b7f38456fff45dd3c66cef04e37',
|
||||
741000: '927b47fc909b4b55c067bbd75d8638af1400fac076cb642e9500a747d849e458',
|
||||
742000: '97fa3d83eb94114496e418c118f549ebfb8f6d123d0b40a12ecb093239557646',
|
||||
743000: '482b66d8d5084703079c28e3ae69e5dee735f762d6fcf9743e75f04e139fd181',
|
||||
744000: 'f406890d5c70808a58fb14429bad812a3185bdb9dace1aa57de76663f92b5013',
|
||||
745000: '2bd0802cbb8aa4441a159104d39515a4ff6fc8dfe616bc83e88197847c78bcff',
|
||||
746000: '24d090a7b6359db3d5d714a69ddc9a6f2e8ff8f044b723220a8ba32df785fd54',
|
||||
747000: '07c4ce9ce5310ee472cf753ddb03c39c5fee6c910d491daffd38615205411633',
|
||||
748000: 'ea913798c0f09d0a27eae7c852954c2c88b8c3b7f23f8fba26b68a3952d0ffde',
|
||||
749000: '23f256adebfe35d49ba84ad49f3f71fc67f7745091c91f22e65f1cc2e23b8f2c',
|
||||
750000: '96db12ee3a295f3d5c56d244e6e7493f58c08d3427e379940e5d4f891a41ec26',
|
||||
}
|
||||
CHUNK_TIMES = [
|
||||
1446058291, 1466796062, 1466957160, 1467112807, 1467272478, 1467432743, 1467593826, 1467753971, 1467911127,
|
||||
1468070719, 1468224507, 1468384171, 1468543503, 1468703272, 1468863520, 1469023822, 1469183471, 1469342714,
|
||||
1469501778, 1469667089, 1469827045, 1469989357, 1470149385, 1470308706, 1470467480, 1470628399, 1470789513,
|
||||
1470949409, 1471110369, 1471271233, 1471432048, 1471589330, 1471749739, 1471910365, 1472071525, 1472235982,
|
||||
1472398890, 1472558207, 1472721074, 1472885421, 1473046344, 1473205609, 1473368205, 1473533441, 1473692268,
|
||||
1473853245, 1474016588, 1474178224, 1474340711, 1474502952, 1474662243, 1474821799, 1474981569, 1475145919,
|
||||
1475307424, 1475467443, 1475630023, 1475791989, 1475952599, 1476111994, 1476271960, 1476433228, 1476595751,
|
||||
1476756365, 1476918643, 1477080426, 1477244266, 1477404425, 1477564644, 1477729818, 1477890614, 1478052281,
|
||||
1478213943, 1478372873, 1478531715, 1478694298, 1478856902, 1479021042, 1479181793, 1479344484, 1479506486,
|
||||
1479663432, 1479825937, 1479985117, 1480148599, 1480310388, 1480474658, 1480639077, 1480800300, 1480960405,
|
||||
1481122737, 1481282632, 1481445451, 1481605639, 1481767227, 1481926963, 1482091620, 1482252291, 1482411821,
|
||||
1482573703, 1482736912, 1482897678, 1483058633, 1483218595, 1483378674, 1483541215, 1483702499, 1483863896,
|
||||
1484022728, 1484180459, 1484342882, 1484507416, 1484670823, 1484829895, 1484993458, 1485156047, 1485317583,
|
||||
1485481218, 1485643245, 1485805928, 1485967452, 1486133206, 1486293958, 1486456274, 1486617735, 1486778872,
|
||||
1486940299, 1487107013, 1487273582, 1487438229, 1487599911, 1487759331, 1487921967, 1488080929, 1488241128,
|
||||
1488404512, 1488565344, 1488725701, 1488887005, 1489047968, 1489207675, 1489371326, 1489535725, 1489703338,
|
||||
1489866064, 1490029222, 1490191595, 1490351274, 1490512577, 1490671939, 1490829835, 1490994376, 1491154639,
|
||||
1491317939, 1491481515, 1491643279, 1491804437, 1491964639, 1492124263, 1492284273, 1492445688, 1492605884,
|
||||
1492765009, 1492927220, 1493088404, 1493251395, 1493412278, 1493573461, 1493732746, 1493893060, 1494053987,
|
||||
1494216326, 1494376404, 1494540073, 1494702026, 1494862527, 1495024065, 1495185322, 1495344568, 1495506617,
|
||||
1495666173, 1495826079, 1495987151, 1496148961, 1496309855, 1496473442, 1496634935, 1496796598, 1496961204,
|
||||
1497121769, 1497283066, 1497443548, 1497603288, 1497767555, 1497933761, 1498094921, 1498256760, 1498420931,
|
||||
1498584545, 1498749559, 1498913792, 1499077566, 1499239970, 1499404091, 1499566804, 1499731157, 1499895613,
|
||||
1500059235, 1500222621, 1500388811, 1500554678, 1500721542, 1500884471, 1501044844, 1501209905, 1501371165,
|
||||
1501530890, 1501694185, 1501857382, 1502018129, 1502184292, 1502350341, 1502511801, 1502676243, 1502838811,
|
||||
1503002137, 1503163638, 1503325023, 1503486684, 1503650746, 1503812386, 1503973244, 1504133666, 1504297628,
|
||||
1504459991, 1504623358, 1504784832, 1504946961, 1505110359, 1505273975, 1505437560, 1505598954, 1505757098,
|
||||
1505917242, 1506077971, 1506237591, 1506398935, 1506559037, 1506720796, 1506885747, 1507050316, 1507212971,
|
||||
1507375409, 1507537815, 1507702332, 1507864531, 1508025612, 1508185476, 1508345744, 1508508500, 1508670141,
|
||||
1508832224, 1508992717, 1509152807, 1509310842, 1509473481, 1509633634, 1509796495, 1509954849, 1510116079,
|
||||
1510277697, 1510439798, 1510600524, 1510761373, 1510921787, 1511081683, 1511243190, 1511403276, 1511563575,
|
||||
1511723540, 1511884418, 1512045789, 1512207376, 1512368713, 1512530546, 1512691400, 1512852021, 1513012694,
|
||||
1513173368, 1513334988, 1513495509, 1513657679, 1513817134, 1513978820, 1514142957, 1514304599, 1514465234,
|
||||
1514624591, 1514785874, 1514945051, 1515108188, 1515268796, 1515430685, 1515590234, 1515751217, 1515910476,
|
||||
1516068662, 1516228676, 1516387357, 1516547616, 1516707229, 1516867094, 1517026140, 1517186251, 1517345155,
|
||||
1517505393, 1517664993, 1517824788, 1517983415, 1518144196, 1518303260, 1518464979, 1518625788, 1518785521,
|
||||
1518945093, 1519103570, 1519263344, 1519423481, 1519584426, 1519746582, 1519907472, 1520066218, 1520226493,
|
||||
1520385895, 1520543541, 1520702453, 1520861359, 1521018892, 1521177734, 1521341622, 1521502375, 1521662214,
|
||||
1521822716, 1521982142, 1522142729, 1522301856, 1522459931, 1522620631, 1522780689, 1522940183, 1523099791,
|
||||
1523258399, 1523420479, 1523579443, 1523739555, 1523899500, 1524059916, 1524219172, 1524378005, 1524537757,
|
||||
1524701031, 1524861748, 1525021162, 1525180528, 1525341200, 1525501213, 1525662981, 1525823008, 1525982484,
|
||||
1526142364, 1526300429, 1526459717, 1526620687, 1526781127, 1526940119, 1527099136, 1527259200, 1527418437,
|
||||
1527579937, 1527739330, 1527898324, 1528059125, 1528218736, 1528379347, 1528540251, 1528700525, 1528859406,
|
||||
1529018377, 1529178147, 1529339191, 1529498669, 1529656834, 1529815415, 1529977050, 1530135713, 1530295787,
|
||||
1530456752, 1530616682, 1530776198, 1530937477, 1531097196, 1531259398, 1531419197, 1531579213, 1531739816,
|
||||
1531899928, 1532060554, 1532220789, 1532378546, 1532537193, 1532695877, 1532852995, 1533013229, 1533170401,
|
||||
1533329105, 1533488406, 1533646944, 1533807289, 1533965434, 1534125093, 1534284340, 1534442620, 1534600547,
|
||||
1534759292, 1534920427, 1535082093, 1535241742, 1535404226, 1535566494, 1535726910, 1535884899, 1536042729,
|
||||
1536201423, 1536361908, 1536520607, 1536679327, 1536840381, 1536999216, 1537157660, 1537318284, 1537477213,
|
||||
1537635506, 1537793639, 1537953780, 1538117759, 1538278913, 1538439538, 1538597977, 1538756382, 1538915421,
|
||||
1539077166, 1539240317, 1539400708, 1539562594, 1539722205, 1539881920, 1540042540, 1540202519, 1540363464,
|
||||
1540521096, 1540681043, 1540838897, 1541000630, 1541159754, 1541320837, 1541479768, 1541640101, 1541798960,
|
||||
1541957879, 1542114653, 1542274838, 1542433802, 1542592336, 1542752146, 1542913026, 1543072942, 1543231105,
|
||||
1543390483, 1543548911, 1543705884, 1543864964, 1544025570, 1544185854, 1544343817, 1544503407, 1544661411,
|
||||
1544819366, 1544980849, 1545142380, 1545303819, 1545464301, 1545624146, 1545783061, 1545942620, 1546102174,
|
||||
1546262550, 1546421421, 1546580475, 1546741261, 1546901871, 1547061606, 1547221600, 1547379723, 1547538785,
|
||||
1547697419, 1547857490, 1548017070, 1548176553, 1548335542, 1548495290, 1548656463, 1548815906, 1548976481,
|
||||
1549134901, 1549293055, 1549453525, 1549615159, 1549775541, 1549934636, 1550095275, 1550255002, 1550416229,
|
||||
1550577697, 1550739302, 1550897867, 1551057080, 1551218892, 1551379080, 1551538880, 1551700931, 1551860435,
|
||||
1552018109, 1552178077, 1552337707, 1552497354, 1552659923, 1552821230, 1552980036, 1553138910, 1553298180,
|
||||
1553457714, 1553616756, 1553777086, 1553937586, 1554098177, 1554257525, 1554415929, 1554574675, 1554734734,
|
||||
1554895337, 1555054185, 1555214104, 1555375518, 1555532785, 1555694268, 1555851788, 1556013303, 1556174215,
|
||||
1556332761, 1556494320, 1556655155, 1556816169, 1556975478, 1557132393, 1557293583, 1557451564, 1557612602,
|
||||
1557772604, 1557931689, 1558091515, 1558250159, 1558409668, 1558568765, 1558726947, 1558886991, 1559046885,
|
||||
1559205383, 1559364094, 1559524528, 1559686227, 1559844138, 1560003768, 1560163656, 1560324916, 1560484561,
|
||||
1560645681, 1560808735, 1560971999, 1561135851, 1561297483, 1561455206, 1561614235, 1561777714, 1561945864,
|
||||
1562118106, 1562278703, 1562437641, 1562603277, 1562767483, 1562927551, 1563087487, 1563248549, 1563410262,
|
||||
1563573257, 1563733848, 1563894208, 1564054018, 1564212252, 1564374334, 1564532357, 1564693415, 1564854450,
|
||||
1565015965, 1565175795, 1565339141, 1565497044, 1565655699, 1565818978, 1565981118, 1566141639, 1566302976,
|
||||
1566465005, 1566625379, 1566786170, 1566946313, 1567108918, 1567270890, 1567430354, 1567588684, 1567749712,
|
||||
1567908836, 1568068607, 1568229349, 1568390117, 1568549698, 1568709328, 1568867513, 1569025318, 1569185425,
|
||||
1569344544, 1569503324, 1569664889, 1569823995, 1569986419, 1570146118, 1570306012, 1570467311, 1570628581,
|
||||
1570787594, 1570948886, 1571110097, 1571270823, 1571430280, 1571588101, 1571750508, 1571911503, 1572071265,
|
||||
1572232449, 1572392837, 1572553922, 1572713730, 1572875606, 1573036488, 1573196968, 1573362152, 1573521448,
|
||||
1573683075, 1573843849, 1574003422, 1574162840, 1574323813, 1574484633, 1574643754, 1574807310, 1574966691,
|
||||
1575125631, 1575285152, 1575444009, 1575605511, 1575765796, 1575924265, 1576087060, 1576246324, 1576404707,
|
||||
1576562942, 1576724468, 1576885925, 1577045396, 1577204522, 1577365567, 1577524358, 1577684296, 1577847223,
|
||||
1578008388, 1578166426, 1578327792, 1578489088, 1578650676, 1578809573, 1578967948, 1579125782, 1579285861,
|
||||
1579449027, 1579608917, 1579768221, 1579927859, 1580086975, 1580247978, 1580408360, 1580569255, 1580728734,
|
||||
1580888441, 1581047850, 1581208990, 1581369823, 1581528564, 1581689593, 1581848711, 1582008320, 1582169619,
|
||||
1582329254, 1582490606, 1582653889, 1582813888, 1582976432, 1583137185, 1583299285, 1583463034, 1583625464,
|
||||
1583786150, 1583949616, 1584109878, 1584268907, 1584428799, 1584589342, 1584749487, 1584911825, 1585073681,
|
||||
1585234424, 1585394114, 1585556112, 1585716022, 1585876422, 1586036811, 1586196631, 1586356275, 1586519756,
|
||||
1586678956, 1586838190, 1587004938, 1587160029
|
||||
]
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ from binascii import hexlify, unhexlify
|
|||
|
||||
from lbry.crypto.hash import sha512, double_sha256, ripemd160
|
||||
from lbry.wallet.util import ArithUint256, date_to_julian_day
|
||||
from .checkpoints import HASHES
|
||||
from .checkpoints import HASHES, CHUNK_TIMES
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
|
@ -138,7 +138,13 @@ class Headers:
|
|||
def estimated_timestamp(self, height):
|
||||
if height <= 0:
|
||||
return
|
||||
return int(self.first_block_timestamp + (height * self.timestamp_average_offset))
|
||||
if height >= max(self.checkpoints.keys()):
|
||||
return int(self.first_block_timestamp + (height * self.timestamp_average_offset))
|
||||
chunk_index = min(height // 1000, len(CHUNK_TIMES) - 1)
|
||||
offset = height - chunk_index * 1000
|
||||
chunk_start_time = CHUNK_TIMES[chunk_index]
|
||||
chunk_end_time = CHUNK_TIMES[chunk_index + 1]
|
||||
return int(offset * ((chunk_end_time - chunk_start_time) / 1000) + chunk_start_time)
|
||||
|
||||
def estimated_julian_day(self, height):
|
||||
return date_to_julian_day(date.fromtimestamp(self.estimated_timestamp(height)))
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import asyncio
|
||||
import textwrap
|
||||
import os
|
||||
|
||||
from lbry.extras.cli import ensure_directory_exists
|
||||
|
|
@ -24,11 +25,18 @@ async def main():
|
|||
return
|
||||
print(f"Headers file at {headers.height}, checkpointing up to {target}."
|
||||
f"Current checkpoint at {current_checkpoint_tip}.")
|
||||
timestamps = []
|
||||
with open(outpath, 'w') as outfile:
|
||||
print('HASHES = {', file=outfile)
|
||||
for height in range(0, target, 1000):
|
||||
timestamps.append((await headers.get(height))['timestamp'])
|
||||
print(f" {height}: '{headers.chunk_hash(height, 1000)}',", file=outfile)
|
||||
print('}', file=outfile)
|
||||
print('CHUNK_TIMES = [', file=outfile)
|
||||
for line in textwrap.wrap(f'{", ".join(map(str, timestamps))}', 120, initial_indent=' '*4, subsequent_indent=' '*4):
|
||||
print(line, file=outfile)
|
||||
print(']', file=outfile)
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -153,6 +153,15 @@ class TestHeaders(AsyncioTestCase):
|
|||
self.assertIsNone(headers.estimated_timestamp(0))
|
||||
self.assertIsNotNone(headers.estimated_timestamp(1))
|
||||
|
||||
def test_estimate_from_checkpoints(self):
|
||||
headers = _Headers(':memory:') # here we use the real version, with checkpoints
|
||||
# check estimations matches real timestamps (LBRY, main net) by at most 20 minutes
|
||||
self.assertAlmostEqual(headers.estimated_timestamp(101010), 1482899706, delta=60 * 20)
|
||||
self.assertAlmostEqual(headers.estimated_timestamp(505050), 1547865484, delta=60 * 20)
|
||||
self.assertAlmostEqual(headers.estimated_timestamp(707070), 1580258295, delta=60 * 20)
|
||||
# tip uses old algorithm as the range doesnt exist
|
||||
self.assertIsNotNone(headers.estimated_timestamp(max(headers.checkpoints.keys())))
|
||||
|
||||
async def test_misalignment_triggers_repair_on_open(self):
|
||||
headers = Headers(':memory:')
|
||||
headers.io.seek(0)
|
||||
|
|
|
|||
Loading…
Reference in a new issue