LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

contrib/linearize: split block files based on year-month, not just year

Browse source
This commit is contained in:
Jeff Garzik 2014-08-24 01:40:40 -04:00
parent 75400a2a41
commit 8f5a423344
1 changed files with 15 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
contrib/linearize/linearize-data.py
View file

@ -58,10 +58,11 @@ def calc_hash_str(blk_hdr):
hash_str = hash.encode('hex') hash_str = hash.encode('hex')
return hash_str return hash_str
def get_blk_year(blk_hdr): def get_blk_dt(blk_hdr):
members = struct.unpack("<I", blk_hdr[68:68+4]) members = struct.unpack("<I", blk_hdr[68:68+4])
dt = datetime.datetime.fromtimestamp(members[0]) dt = datetime.datetime.fromtimestamp(members[0])
return dt.year dt_ym = datetime.datetime(dt.year, dt.month, 1)
return dt_ym
def get_block_hashes(settings): def get_block_hashes(settings):
blkindex = [] blkindex = []
@ -88,14 +89,14 @@ def copydata(settings, blkindex, blkset):
outF = None outF = None
blkCount = 0 blkCount = 0
lastYear = 0 lastDate = datetime.datetime(2000, 1, 1)
splitYear = False timestampSplit = False
fileOutput = True fileOutput = True
maxOutSz = settings['max_out_sz'] maxOutSz = settings['max_out_sz']
if 'output' in settings: if 'output' in settings:
fileOutput = False fileOutput = False
if settings['split_year'] != 0: if settings['split_timestamp'] != 0:
splitYear = True timestampSplit = True
while True: while True:
if not inF: if not inF:
@ -137,11 +138,11 @@ def copydata(settings, blkindex, blkset):
outFn = outFn + 1 outFn = outFn + 1
outsz = 0 outsz = 0
if splitYear: if timestampSplit:
blkYear = get_blk_year(blk_hdr) blkDate = get_blk_dt(blk_hdr)
if blkYear > lastYear: if blkDate > lastDate:
print("New year " + str(blkYear) + " @ " + hash_str) print("New month " + blkDate.strftime("%Y-%m") + " @ " + hash_str)
lastYear = blkYear lastDate = blkDate
if outF: if outF:
outF.close() outF.close()
outF = None outF = None
@ -190,13 +191,13 @@ if __name__ == '__main__':
settings['input'] = 'input' settings['input'] = 'input'
if 'hashlist' not in settings: if 'hashlist' not in settings:
settings['hashlist'] = 'hashlist.txt' settings['hashlist'] = 'hashlist.txt'
if 'split_year' not in settings: if 'split_timestamp' not in settings:
settings['split_year'] = 0 settings['split_timestamp'] = 0
if 'max_out_sz' not in settings: if 'max_out_sz' not in settings:
settings['max_out_sz'] = 1000L * 1000 * 1000 settings['max_out_sz'] = 1000L * 1000 * 1000
settings['max_out_sz'] = long(settings['max_out_sz']) settings['max_out_sz'] = long(settings['max_out_sz'])
settings['split_year'] = int(settings['split_year']) settings['split_timestamp'] = int(settings['split_timestamp'])
settings['netmagic'] = settings['netmagic'].decode('hex') settings['netmagic'] = settings['netmagic'].decode('hex')
if 'output_file' not in settings and 'output' not in settings: if 'output_file' not in settings and 'output' not in settings: