LBRYCommunity/lbry-sdk
1
0
Fork 1
Code Issues Pull requests Projects Releases 282 Packages Wiki Activity

added ffmpeg status, addressed items from code review

Browse source 
linter
This commit is contained in:
Brannon King 2020-02-03 15:53:27 -07:00 committed by Lex Berezhny
parent 85ad972ca8
commit 1780ddd329
6 changed files with 67 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
lbry/conf.py
View file

@ -1,6 +1,5 @@
import os
import re
import platform
import sys
import typing
import logging
@ -462,13 +461,6 @@ class BaseConfig:
if self.persisted.upgrade():
self.persisted.save()
@property
def needs_proactor(self):
major, minor, _ = platform.python_version_tuple()
if int(major) > 3 or (int(major) == 3 and int(minor) > 7):
return False
return platform.system() == "Windows"
class TranscodeConfig(BaseConfig):

4
lbry/extras/cli.py
View file

@ -3,6 +3,7 @@ import sys
import shutil
import signal
import pathlib
import platform
import json
import asyncio
import argparse
@ -262,7 +263,8 @@ def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config
def run_daemon(args: argparse.Namespace, conf: Config):
if conf.needs_proactor:
if sys.version_info < (3, 8) and platform.system() == "Windows":
# TODO: remove after we move to requiring Python 3.8
asyncio.set_event_loop(asyncio.ProactorEventLoop())
loop = asyncio.get_event_loop()
if args.verbose is not None:

2
lbry/extras/daemon/daemon.py
View file

@ -853,6 +853,7 @@ class Daemon(metaclass=JSONRPCServerType):
"""
connection_code = await self.get_connection_status()
ffmpeg_status = await self._video_file_analyzer.status()
response = {
'installation_id': self.installation_id,
@ -863,6 +864,7 @@ class Daemon(metaclass=JSONRPCServerType):
'code': connection_code,
'message': CONNECTION_MESSAGES[connection_code],
},
'ffmpeg_status': ffmpeg_status
}
for component in self.component_manager.components:
status = await component.get_status()

59
lbry/file_analysis.py
View file

@ -13,12 +13,6 @@ log = logging.getLogger(__name__)
class VideoFileAnalyzer:
@staticmethod
def _matches(needles: list, haystack: list):
for needle in needles:
if needle in haystack:
return True
return False
async def _execute(self, command, arguments):
args = shlex.split(arguments)
@ -44,34 +38,55 @@ class VideoFileAnalyzer:
return
await self._verify_executable("ffprobe")
version = await self._verify_executable("ffmpeg")
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0],
shutil.which(f"{self._conf.ffmpeg_folder}ffmpeg"))
self._which = shutil.which(f"{self._conf.ffmpeg_folder}ffmpeg")
self._ffmpeg_installed = True
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which)
def __init__(self, conf: TranscodeConfig):
self._conf = conf
self._available_encoders = ""
self._ffmpeg_installed = False
self._which = None
def _verify_container(self, scan_data: json):
async def status(self, reset=False):
if reset:
self._available_encoders = ""
self._ffmpeg_installed = False
self._which = None
installed = True
try:
await self._verify_ffmpeg_installed()
except FileNotFoundError:
installed = False
return {
"available": installed,
"which": self._which,
"analyze_audio_volume": int(self._conf.volume_analysis_time) > 0
}
@staticmethod
def _verify_container(scan_data: json):
container = scan_data["format"]["format_name"]
log.debug(" Detected container is %s", container)
if not self._matches(container.split(","), ["webm", "mp4", "3gp", "ogg"]):
if not {"webm", "mp4", "3gp", "ogg"}.intersection(container.split(",")):
return "Container format is not in the approved list of WebM, MP4. " \
f"Actual: {container} [{scan_data['format']['format_long_name']}]"
return ""
def _verify_video_encoding(self, scan_data: json):
@staticmethod
def _verify_video_encoding(scan_data: json):
for stream in scan_data["streams"]:
if stream["codec_type"] != "video":
continue
codec = stream["codec_name"]
log.debug(" Detected video codec is %s, format is %s", codec, stream["pix_fmt"])
if not self._matches(codec.split(","), ["h264", "vp8", "vp9", "av1", "theora"]):
if not {"h264", "vp8", "vp9", "av1", "theora"}.intersection(codec.split(",")):
return "Video codec is not in the approved list of H264, VP8, VP9, AV1, Theora. " \
f"Actual: {codec} [{stream['codec_long_name']}]"
if self._matches(codec.split(","), ["h264"]) and stream["pix_fmt"] != "yuv420p":
if "h264" in codec.split(",") and stream["pix_fmt"] != "yuv420p":
return "Video codec is H264, but its pixel format does not match the approved yuv420p. " \
f"Actual: {stream['pix_fmt']}"
@ -100,7 +115,7 @@ class VideoFileAnalyzer:
async def _verify_fast_start(self, scan_data: json, video_file):
container = scan_data["format"]["format_name"]
if self._matches(container.split(","), ["webm", "ogg"]):
if {"webm", "ogg"}.intersection(container.split(",")):
return ""
result, _ = await self._execute("ffprobe", f'-v debug "{video_file}"')
@ -110,13 +125,14 @@ class VideoFileAnalyzer:
return "Video stream descriptors are not at the start of the file (the faststart flag was not used)."
return ""
def _verify_audio_encoding(self, scan_data: json):
@staticmethod
def _verify_audio_encoding(scan_data: json):
for stream in scan_data["streams"]:
if stream["codec_type"] != "audio":
continue
codec = stream["codec_name"]
log.debug(" Detected audio codec is %s", codec)
if not self._matches(codec.split(","), ["aac", "mp3", "flac", "vorbis", "opus"]):
if not {"aac", "mp3", "flac", "vorbis", "opus"}.intersection(codec.split(",")):
return "Audio codec is not in the approved list of AAC, FLAC, MP3, Vorbis, and Opus. " \
f"Actual: {codec} [{stream['codec_long_name']}]"
@ -126,7 +142,7 @@ class VideoFileAnalyzer:
try:
validate_volume = int(seconds) > 0
except ValueError:
validate_volume = 0
validate_volume = False
if not validate_volume:
return ""
@ -224,7 +240,8 @@ class VideoFileAnalyzer:
async def _get_volume_filter(self):
return self._conf.volume_filter if self._conf.volume_filter else "-af loudnorm"
def _get_best_container_extension(self, scan_data, video_encoder):
@staticmethod
def _get_best_container_extension(scan_data, video_encoder):
# the container is chosen by the video format
# if we are theora-encoded, we want ogg
# if we are vp8/vp9/av1 we want webm
@ -235,9 +252,9 @@ class VideoFileAnalyzer:
if stream["codec_type"] != "video":
continue
codec = stream["codec_name"].split(",")
if self._matches(codec, ["theora"]):
if "theora" in codec:
return "ogg"
if self._matches(codec, ["vp8", "vp9", "av1"]):
if {"vp8", "vp9", "av1"}.intersection(codec):
return "webm"
if "theora" in video_encoder:
@ -260,7 +277,7 @@ class VideoFileAnalyzer:
if "format" not in scan_data:
if validate:
raise Exception(f'Unexpected video file contents in: {file_path}')
raise FileNotFoundError(f'Unexpected or absent video file contents at: {file_path}')
log.info("Unable to optimize %s . FFmpeg output is missing the format section.", file_path)
return

7
scripts/check_video.py
View file

@ -2,6 +2,7 @@
import asyncio
import logging
import platform
import sys
# noinspection PyUnresolvedReferences
@ -48,9 +49,11 @@ def main():
video_file = sys.argv[1]
conf = TranscodeConfig()
analyzer = VideoFileAnalyzer(conf)
loop = asyncio.ProactorEventLoop() if conf.needs_proactor else asyncio.get_event_loop()
if sys.version_info < (3, 8) and platform.system() == "Windows":
# TODO: remove after we move to requiring Python 3.8
asyncio.set_event_loop(asyncio.ProactorEventLoop())
try:
loop.run_until_complete(process_video(analyzer, video_file))
asyncio.run(process_video(analyzer, video_file))
except KeyboardInterrupt:
pass

28
tests/integration/blockchain/test_transcoding.py
View file

@ -34,6 +34,7 @@ class TranscodeValidation(ClaimTestCase):
self.conf.volume_analysis_time = 0 # disable it as the test file isn't very good here
self.analyzer = VideoFileAnalyzer(self.conf)
file_ogg = self.make_name("ogg", ".ogg")
self.video_file_ogg = str(file_ogg)
if not file_ogg.exists():
command = f'-i "{self.video_file_name}" -c:v libtheora -q:v 4 -c:a libvorbis -q:a 4 ' \
f'-c:s copy -c:d copy "{file_ogg}"'
@ -42,6 +43,7 @@ class TranscodeValidation(ClaimTestCase):
self.assertEqual(code, 0, output)
file_webm = self.make_name("webm", ".webm")
self.video_file_webm = str(file_webm)
if not file_webm.exists():
command = f'-i "{self.video_file_name}" -c:v libvpx-vp9 -crf 36 -b:v 0 -cpu-used 2 ' \
f'-c:a libopus -b:a 128k -c:s copy -c:d copy "{file_webm}"'
@ -49,12 +51,13 @@ class TranscodeValidation(ClaimTestCase):
output, code = await self.analyzer._execute("ffmpeg", command)
self.assertEqual(code, 0, output)
self.should_work = [self.video_file_name, str(file_ogg), str(file_webm)]
async def test_should_work(self):
for should_work_file_name in self.should_work:
new_file_name = await self.analyzer.verify_or_repair(True, False, should_work_file_name)
self.assertEqual(should_work_file_name, new_file_name)
new_file_name = await self.analyzer.verify_or_repair(True, False, self.video_file_name)
self.assertEqual(self.video_file_name, new_file_name)
new_file_name = await self.analyzer.verify_or_repair(True, False, self.video_file_ogg)
self.assertEqual(self.video_file_ogg, new_file_name)
new_file_name = await self.analyzer.verify_or_repair(True, False, self.video_file_webm)
self.assertEqual(self.video_file_webm, new_file_name)
async def test_volume(self):
try:
@ -125,10 +128,17 @@ class TranscodeValidation(ClaimTestCase):
async def test_extension_choice(self):
for file_name in self.should_work:
scan_data = await self.analyzer._get_scan_data(True, file_name)
extension = self.analyzer._get_best_container_extension(scan_data, "")
self.assertEqual(extension, pathlib.Path(file_name).suffix[1:])
scan_data = await self.analyzer._get_scan_data(True, self.video_file_name)
extension = self.analyzer._get_best_container_extension(scan_data, "")
self.assertEqual(extension, pathlib.Path(self.video_file_name).suffix[1:])
scan_data = await self.analyzer._get_scan_data(True, self.video_file_ogg)
extension = self.analyzer._get_best_container_extension(scan_data, "")
self.assertEqual(extension, "ogg")
scan_data = await self.analyzer._get_scan_data(True, self.video_file_webm)
extension = self.analyzer._get_best_container_extension(scan_data, "")
self.assertEqual(extension, "webm")
extension = self.analyzer._get_best_container_extension("", "libx264 -crf 23")
self.assertEqual("mp4", extension)