null in claim description

This commit is contained in:
Lex Berezhny 2020-06-27 23:14:28 -04:00
parent 635aebfeeb
commit 2495df8859
3 changed files with 9 additions and 4 deletions

View file

@ -436,9 +436,9 @@ class BulkLoader:
claim_record['claim_type'] = TXO_TYPES['stream'] claim_record['claim_type'] = TXO_TYPES['stream']
claim_record['stream_type'] = STREAM_TYPES[guess_stream_type(claim_record['media_type'])] claim_record['stream_type'] = STREAM_TYPES[guess_stream_type(claim_record['media_type'])]
claim_record['media_type'] = claim.stream.source.media_type claim_record['media_type'] = claim.stream.source.media_type
claim_record['title'] = claim.stream.title claim_record['title'] = claim.stream.title.replace('\x00', '')
claim_record['description'] = claim.stream.description claim_record['description'] = claim.stream.description.replace('\x00', '')
claim_record['author'] = claim.stream.author claim_record['author'] = claim.stream.author.replace('\x00', '')
if claim.stream.video and claim.stream.video.duration: if claim.stream.video and claim.stream.video.duration:
claim_record['duration'] = claim.stream.video.duration claim_record['duration'] = claim.stream.video.duration
if claim.stream.audio and claim.stream.audio.duration: if claim.stream.audio and claim.stream.audio.duration:

View file

@ -6,7 +6,11 @@ WEIRD_CHARS_RE = re.compile(r"[#!~]")
def normalize_tag(tag: str): def normalize_tag(tag: str):
return MULTI_SPACE_RE.sub(' ', WEIRD_CHARS_RE.sub(' ', tag.lower().replace("'", ""))).strip() return MULTI_SPACE_RE.sub(
' ', WEIRD_CHARS_RE.sub(
' ', tag.lower().replace("'", "").replace('\x00', '')
)
).strip()
def clean_tags(tags: List[str]): def clean_tags(tags: List[str]):

View file

@ -14,6 +14,7 @@ class TestTagNormalization(unittest.TestCase):
tag('tag', 'T\'ag') tag('tag', 'T\'ag')
tag('t ag', '\tT \nAG ') tag('t ag', '\tT \nAG ')
tag('tag hash', '#tag~#hash!') tag('tag hash', '#tag~#hash!')
tag('foobar', 'foo\x00bar')
def test_clean_tags(self): def test_clean_tags(self):
self.assertEqual(['tag'], clean_tags([' \t #!~', '!taG', '\t'])) self.assertEqual(['tag'], clean_tags([' \t #!~', '!taG', '\t']))