From 1780ddd32900d4fa449ba3532ee2b47e86ecd08a Mon Sep 17 00:00:00 2001 From: Brannon King Date: Mon, 3 Feb 2020 15:53:27 -0700 Subject: [PATCH] added ffmpeg status, addressed items from code review linter --- lbry/conf.py | 8 --- lbry/extras/cli.py | 4 +- lbry/extras/daemon/daemon.py | 2 + lbry/file_analysis.py | 59 ++++++++++++------- scripts/check_video.py | 7 ++- .../blockchain/test_transcoding.py | 28 ++++++--- 6 files changed, 67 insertions(+), 41 deletions(-) diff --git a/lbry/conf.py b/lbry/conf.py index 5ab16951f..09810d6a3 100644 --- a/lbry/conf.py +++ b/lbry/conf.py @@ -1,6 +1,5 @@ import os import re -import platform import sys import typing import logging @@ -462,13 +461,6 @@ class BaseConfig: if self.persisted.upgrade(): self.persisted.save() - @property - def needs_proactor(self): - major, minor, _ = platform.python_version_tuple() - if int(major) > 3 or (int(major) == 3 and int(minor) > 7): - return False - return platform.system() == "Windows" - class TranscodeConfig(BaseConfig): diff --git a/lbry/extras/cli.py b/lbry/extras/cli.py index 8c3699826..e0f6de36a 100644 --- a/lbry/extras/cli.py +++ b/lbry/extras/cli.py @@ -3,6 +3,7 @@ import sys import shutil import signal import pathlib +import platform import json import asyncio import argparse @@ -262,7 +263,8 @@ def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config def run_daemon(args: argparse.Namespace, conf: Config): - if conf.needs_proactor: + if sys.version_info < (3, 8) and platform.system() == "Windows": + # TODO: remove after we move to requiring Python 3.8 asyncio.set_event_loop(asyncio.ProactorEventLoop()) loop = asyncio.get_event_loop() if args.verbose is not None: diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index cd80d7698..d7333e967 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -853,6 +853,7 @@ class Daemon(metaclass=JSONRPCServerType): """ connection_code = await self.get_connection_status() + ffmpeg_status = await self._video_file_analyzer.status() response = { 'installation_id': self.installation_id, @@ -863,6 +864,7 @@ class Daemon(metaclass=JSONRPCServerType): 'code': connection_code, 'message': CONNECTION_MESSAGES[connection_code], }, + 'ffmpeg_status': ffmpeg_status } for component in self.component_manager.components: status = await component.get_status() diff --git a/lbry/file_analysis.py b/lbry/file_analysis.py index 5f837c532..75d119647 100644 --- a/lbry/file_analysis.py +++ b/lbry/file_analysis.py @@ -13,12 +13,6 @@ log = logging.getLogger(__name__) class VideoFileAnalyzer: - @staticmethod - def _matches(needles: list, haystack: list): - for needle in needles: - if needle in haystack: - return True - return False async def _execute(self, command, arguments): args = shlex.split(arguments) @@ -44,34 +38,55 @@ class VideoFileAnalyzer: return await self._verify_executable("ffprobe") version = await self._verify_executable("ffmpeg") - log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], - shutil.which(f"{self._conf.ffmpeg_folder}ffmpeg")) + self._which = shutil.which(f"{self._conf.ffmpeg_folder}ffmpeg") self._ffmpeg_installed = True + log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which) def __init__(self, conf: TranscodeConfig): self._conf = conf self._available_encoders = "" self._ffmpeg_installed = False + self._which = None - def _verify_container(self, scan_data: json): + async def status(self, reset=False): + if reset: + self._available_encoders = "" + self._ffmpeg_installed = False + self._which = None + + installed = True + try: + await self._verify_ffmpeg_installed() + except FileNotFoundError: + installed = False + + return { + "available": installed, + "which": self._which, + "analyze_audio_volume": int(self._conf.volume_analysis_time) > 0 + } + + @staticmethod + def _verify_container(scan_data: json): container = scan_data["format"]["format_name"] log.debug(" Detected container is %s", container) - if not self._matches(container.split(","), ["webm", "mp4", "3gp", "ogg"]): + if not {"webm", "mp4", "3gp", "ogg"}.intersection(container.split(",")): return "Container format is not in the approved list of WebM, MP4. " \ f"Actual: {container} [{scan_data['format']['format_long_name']}]" return "" - def _verify_video_encoding(self, scan_data: json): + @staticmethod + def _verify_video_encoding(scan_data: json): for stream in scan_data["streams"]: if stream["codec_type"] != "video": continue codec = stream["codec_name"] log.debug(" Detected video codec is %s, format is %s", codec, stream["pix_fmt"]) - if not self._matches(codec.split(","), ["h264", "vp8", "vp9", "av1", "theora"]): + if not {"h264", "vp8", "vp9", "av1", "theora"}.intersection(codec.split(",")): return "Video codec is not in the approved list of H264, VP8, VP9, AV1, Theora. " \ f"Actual: {codec} [{stream['codec_long_name']}]" - if self._matches(codec.split(","), ["h264"]) and stream["pix_fmt"] != "yuv420p": + if "h264" in codec.split(",") and stream["pix_fmt"] != "yuv420p": return "Video codec is H264, but its pixel format does not match the approved yuv420p. " \ f"Actual: {stream['pix_fmt']}" @@ -100,7 +115,7 @@ class VideoFileAnalyzer: async def _verify_fast_start(self, scan_data: json, video_file): container = scan_data["format"]["format_name"] - if self._matches(container.split(","), ["webm", "ogg"]): + if {"webm", "ogg"}.intersection(container.split(",")): return "" result, _ = await self._execute("ffprobe", f'-v debug "{video_file}"') @@ -110,13 +125,14 @@ class VideoFileAnalyzer: return "Video stream descriptors are not at the start of the file (the faststart flag was not used)." return "" - def _verify_audio_encoding(self, scan_data: json): + @staticmethod + def _verify_audio_encoding(scan_data: json): for stream in scan_data["streams"]: if stream["codec_type"] != "audio": continue codec = stream["codec_name"] log.debug(" Detected audio codec is %s", codec) - if not self._matches(codec.split(","), ["aac", "mp3", "flac", "vorbis", "opus"]): + if not {"aac", "mp3", "flac", "vorbis", "opus"}.intersection(codec.split(",")): return "Audio codec is not in the approved list of AAC, FLAC, MP3, Vorbis, and Opus. " \ f"Actual: {codec} [{stream['codec_long_name']}]" @@ -126,7 +142,7 @@ class VideoFileAnalyzer: try: validate_volume = int(seconds) > 0 except ValueError: - validate_volume = 0 + validate_volume = False if not validate_volume: return "" @@ -224,7 +240,8 @@ class VideoFileAnalyzer: async def _get_volume_filter(self): return self._conf.volume_filter if self._conf.volume_filter else "-af loudnorm" - def _get_best_container_extension(self, scan_data, video_encoder): + @staticmethod + def _get_best_container_extension(scan_data, video_encoder): # the container is chosen by the video format # if we are theora-encoded, we want ogg # if we are vp8/vp9/av1 we want webm @@ -235,9 +252,9 @@ class VideoFileAnalyzer: if stream["codec_type"] != "video": continue codec = stream["codec_name"].split(",") - if self._matches(codec, ["theora"]): + if "theora" in codec: return "ogg" - if self._matches(codec, ["vp8", "vp9", "av1"]): + if {"vp8", "vp9", "av1"}.intersection(codec): return "webm" if "theora" in video_encoder: @@ -260,7 +277,7 @@ class VideoFileAnalyzer: if "format" not in scan_data: if validate: - raise Exception(f'Unexpected video file contents in: {file_path}') + raise FileNotFoundError(f'Unexpected or absent video file contents at: {file_path}') log.info("Unable to optimize %s . FFmpeg output is missing the format section.", file_path) return diff --git a/scripts/check_video.py b/scripts/check_video.py index f3215060c..ee3a26899 100755 --- a/scripts/check_video.py +++ b/scripts/check_video.py @@ -2,6 +2,7 @@ import asyncio import logging +import platform import sys # noinspection PyUnresolvedReferences @@ -48,9 +49,11 @@ def main(): video_file = sys.argv[1] conf = TranscodeConfig() analyzer = VideoFileAnalyzer(conf) - loop = asyncio.ProactorEventLoop() if conf.needs_proactor else asyncio.get_event_loop() + if sys.version_info < (3, 8) and platform.system() == "Windows": + # TODO: remove after we move to requiring Python 3.8 + asyncio.set_event_loop(asyncio.ProactorEventLoop()) try: - loop.run_until_complete(process_video(analyzer, video_file)) + asyncio.run(process_video(analyzer, video_file)) except KeyboardInterrupt: pass diff --git a/tests/integration/blockchain/test_transcoding.py b/tests/integration/blockchain/test_transcoding.py index 12ed4c51d..3fec6cc33 100644 --- a/tests/integration/blockchain/test_transcoding.py +++ b/tests/integration/blockchain/test_transcoding.py @@ -34,6 +34,7 @@ class TranscodeValidation(ClaimTestCase): self.conf.volume_analysis_time = 0 # disable it as the test file isn't very good here self.analyzer = VideoFileAnalyzer(self.conf) file_ogg = self.make_name("ogg", ".ogg") + self.video_file_ogg = str(file_ogg) if not file_ogg.exists(): command = f'-i "{self.video_file_name}" -c:v libtheora -q:v 4 -c:a libvorbis -q:a 4 ' \ f'-c:s copy -c:d copy "{file_ogg}"' @@ -42,6 +43,7 @@ class TranscodeValidation(ClaimTestCase): self.assertEqual(code, 0, output) file_webm = self.make_name("webm", ".webm") + self.video_file_webm = str(file_webm) if not file_webm.exists(): command = f'-i "{self.video_file_name}" -c:v libvpx-vp9 -crf 36 -b:v 0 -cpu-used 2 ' \ f'-c:a libopus -b:a 128k -c:s copy -c:d copy "{file_webm}"' @@ -49,12 +51,13 @@ class TranscodeValidation(ClaimTestCase): output, code = await self.analyzer._execute("ffmpeg", command) self.assertEqual(code, 0, output) - self.should_work = [self.video_file_name, str(file_ogg), str(file_webm)] - async def test_should_work(self): - for should_work_file_name in self.should_work: - new_file_name = await self.analyzer.verify_or_repair(True, False, should_work_file_name) - self.assertEqual(should_work_file_name, new_file_name) + new_file_name = await self.analyzer.verify_or_repair(True, False, self.video_file_name) + self.assertEqual(self.video_file_name, new_file_name) + new_file_name = await self.analyzer.verify_or_repair(True, False, self.video_file_ogg) + self.assertEqual(self.video_file_ogg, new_file_name) + new_file_name = await self.analyzer.verify_or_repair(True, False, self.video_file_webm) + self.assertEqual(self.video_file_webm, new_file_name) async def test_volume(self): try: @@ -125,10 +128,17 @@ class TranscodeValidation(ClaimTestCase): async def test_extension_choice(self): - for file_name in self.should_work: - scan_data = await self.analyzer._get_scan_data(True, file_name) - extension = self.analyzer._get_best_container_extension(scan_data, "") - self.assertEqual(extension, pathlib.Path(file_name).suffix[1:]) + scan_data = await self.analyzer._get_scan_data(True, self.video_file_name) + extension = self.analyzer._get_best_container_extension(scan_data, "") + self.assertEqual(extension, pathlib.Path(self.video_file_name).suffix[1:]) + + scan_data = await self.analyzer._get_scan_data(True, self.video_file_ogg) + extension = self.analyzer._get_best_container_extension(scan_data, "") + self.assertEqual(extension, "ogg") + + scan_data = await self.analyzer._get_scan_data(True, self.video_file_webm) + extension = self.analyzer._get_best_container_extension(scan_data, "") + self.assertEqual(extension, "webm") extension = self.analyzer._get_best_container_extension("", "libx264 -crf 23") self.assertEqual("mp4", extension)