Madiator2011/lbry-sdk
1
0
Fork 0
forked from LBRYCommunity/lbry-sdk
Code Pull requests Activity

support search path for ffmpeg

Browse source
This commit is contained in:
Brannon King 2020-03-16 17:48:45 -06:00
parent bb1978d976
commit 5ab634e375
5 changed files with 60 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
docs/api.json
View file

@ -3077,7 +3077,7 @@
"curl": "curl -d'{\"method\": \"settings_get\", \"params\": {}}' http://localhost:5279/", "curl": "curl -d'{\"method\": \"settings_get\", \"params\": {}}' http://localhost:5279/",
"lbrynet": "lbrynet settings get", "lbrynet": "lbrynet settings get",
"python": "requests.post(\"http://localhost:5279\", json={\"method\": \"settings_get\", \"params\": {}}).json()", "python": "requests.post(\"http://localhost:5279\", json={\"method\": \"settings_get\", \"params\": {}}).json()",
"output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"announce_head_and_sd_only\": true,\n \"api\": \"localhost:5279\",\n \"audio_encoder\": \"aac -b:a 192k\",\n \"blob_download_timeout\": 30.0,\n \"blob_lru_cache_size\": 0,\n \"blockchain_name\": \"lbrycrd_regtest\",\n \"cache_time\": 150,\n \"coin_selection_strategy\": \"standard\",\n \"comment_server\": \"https://comments.lbry.com/api\",\n \"components_to_skip\": [\n \"dht\",\n \"upnp\",\n \"hash_announcer\",\n \"peer_protocol_server\"\n ],\n \"concurrent_blob_announcers\": 10,\n \"concurrent_reflector_uploads\": 10,\n \"config\": \"/home/lex/.local/share/lbry/lbrynet/daemon_settings.yml\",\n \"data_dir\": \"/tmp/tmpf0g7xmd6\",\n \"download_dir\": \"/tmp/tmpf0g7xmd6\",\n \"download_timeout\": 30.0,\n \"ffmpeg_folder\": \"\",\n \"fixed_peer_delay\": 2.0,\n \"known_dht_nodes\": [],\n \"lbryum_servers\": [\n [\n \"127.0.0.1\",\n 50001\n ]\n ],\n \"max_connections_per_download\": 4,\n \"max_key_fee\": {\n \"amount\": 50.0,\n \"currency\": \"USD\"\n },\n \"network_interface\": \"0.0.0.0\",\n \"node_rpc_timeout\": 5.0,\n \"peer_connect_timeout\": 3.0,\n \"prometheus_port\": 0,\n \"reflect_streams\": true,\n \"reflector_servers\": [\n [\n \"127.0.0.1\",\n 5566\n ]\n ],\n \"s3_headers_depth\": 960,\n \"save_blobs\": true,\n \"save_files\": true,\n \"share_usage_data\": false,\n \"split_buckets_under_index\": 1,\n \"streaming_get\": true,\n \"streaming_server\": \"localhost:5280\",\n \"tcp_port\": 3333,\n \"track_bandwidth\": true,\n \"udp_port\": 4444,\n \"use_upnp\": false,\n \"video_encoder\": \"libx264 -crf 18 -vf \\\"format=yuv420p\\\"\",\n \"volume_analysis_time\": \"240\",\n \"volume_filter\": \"-af loudnorm\",\n \"wallet_dir\": \"/tmp/tmpf0g7xmd6\",\n \"wallets\": [\n \"default_wallet\"\n ]\n }\n}" "output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"announce_head_and_sd_only\": true,\n \"api\": \"localhost:5279\",\n \"audio_encoder\": \"aac -b:a 192k\",\n \"blob_download_timeout\": 30.0,\n \"blob_lru_cache_size\": 0,\n \"blockchain_name\": \"lbrycrd_regtest\",\n \"cache_time\": 150,\n \"coin_selection_strategy\": \"standard\",\n \"comment_server\": \"https://comments.lbry.com/api\",\n \"components_to_skip\": [\n \"dht\",\n \"upnp\",\n \"hash_announcer\",\n \"peer_protocol_server\"\n ],\n \"concurrent_blob_announcers\": 10,\n \"concurrent_reflector_uploads\": 10,\n \"config\": \"/home/lex/.local/share/lbry/lbrynet/daemon_settings.yml\",\n \"data_dir\": \"/tmp/tmpf0g7xmd6\",\n \"download_dir\": \"/tmp/tmpf0g7xmd6\",\n \"download_timeout\": 30.0,\n \"ffmpeg_path\": \"\",\n \"fixed_peer_delay\": 2.0,\n \"known_dht_nodes\": [],\n \"lbryum_servers\": [\n [\n \"127.0.0.1\",\n 50001\n ]\n ],\n \"max_connections_per_download\": 4,\n \"max_key_fee\": {\n \"amount\": 50.0,\n \"currency\": \"USD\"\n },\n \"network_interface\": \"0.0.0.0\",\n \"node_rpc_timeout\": 5.0,\n \"peer_connect_timeout\": 3.0,\n \"prometheus_port\": 0,\n \"reflect_streams\": true,\n \"reflector_servers\": [\n [\n \"127.0.0.1\",\n 5566\n ]\n ],\n \"s3_headers_depth\": 960,\n \"save_blobs\": true,\n \"save_files\": true,\n \"share_usage_data\": false,\n \"split_buckets_under_index\": 1,\n \"streaming_get\": true,\n \"streaming_server\": \"localhost:5280\",\n \"tcp_port\": 3333,\n \"track_bandwidth\": true,\n \"udp_port\": 4444,\n \"use_upnp\": false,\n \"video_encoder\": \"libx264 -crf 18 -vf \\\"format=yuv420p\\\"\",\n \"volume_analysis_time\": \"240\",\n \"volume_filter\": \"-af loudnorm\",\n \"wallet_dir\": \"/tmp/tmpf0g7xmd6\",\n \"wallets\": [\n \"default_wallet\"\n ]\n }\n}"
} }
] ]
}, },

4
lbry/conf.py
View file

@ -464,7 +464,9 @@ class BaseConfig:
class TranscodeConfig(BaseConfig): class TranscodeConfig(BaseConfig):
ffmpeg_folder = String('The path to ffmpeg and ffprobe', '') ffmpeg_path = String('A list of places to check for ffmpeg and ffprobe. '
f'$data_dir/ffmpeg/bin and $PATH are checked afterward. Separator: {os.pathsep}',
'', previous_names=['ffmpeg_folder'])
video_encoder = String('FFmpeg codec and parameters for the video encoding. ' video_encoder = String('FFmpeg codec and parameters for the video encoding. '
'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental', 'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental',
'libx264 -crf 21 -preset faster -pix_fmt yuv420p') 'libx264 -crf 21 -preset faster -pix_fmt yuv420p')

2
lbry/extras/daemon/daemon.py
View file

@ -786,7 +786,7 @@ class Daemon(metaclass=JSONRPCServerType):
'analyze_audio_volume': (bool) should ffmpeg analyze audio 'analyze_audio_volume': (bool) should ffmpeg analyze audio
} }
""" """
return await self._video_file_analyzer.status(recheck=True) return await self._video_file_analyzer.status(reset=True)
async def jsonrpc_status(self): async def jsonrpc_status(self):
""" """

82
lbry/file_analysis.py
View file

@ -26,9 +26,9 @@ class VideoFileAnalyzer:
def __init__(self, conf: TranscodeConfig): def __init__(self, conf: TranscodeConfig):
self._conf = conf self._conf = conf
self._available_encoders = "" self._available_encoders = ""
self._ffmpeg_installed = False self._ffmpeg_installed = None
self._which = None self._whichFFmpeg = None
self._checked_ffmpeg = False self._whichFFprobe = None
self._env_copy = dict(os.environ) self._env_copy = dict(os.environ)
if lbry.utils.is_running_from_bundle(): if lbry.utils.is_running_from_bundle():
# handle the situation where PyInstaller overrides our runtime environment: # handle the situation where PyInstaller overrides our runtime environment:
@ -38,51 +38,69 @@ class VideoFileAnalyzer:
if DISABLED: if DISABLED:
return "Disabled on Windows", -1 return "Disabled on Windows", -1
args = shlex.split(arguments) args = shlex.split(arguments)
process = await asyncio.create_subprocess_exec( process = await asyncio.create_subprocess_exec(
os.path.join(self._conf.ffmpeg_folder, command), *args, command, *args, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, env=self._env_copy
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, env=self._env_copy
) )
stdout, stderr = await process.communicate() # returns when the streams are closed stdout, stderr = await process.communicate() # returns when the streams are closed
return stdout.decode(errors='replace') + stderr.decode(errors='replace'), process.returncode return stdout.decode(errors='replace') + stderr.decode(errors='replace'), process.returncode
async def _execute_ffmpeg(self, arguments):
assert self._ffmpeg_installed
return await self._execute(self._whichFFmpeg, arguments)
async def _execute_ffprobe(self, arguments):
assert self._ffmpeg_installed
return await self._execute(self._whichFFprobe, arguments)
async def _verify_executable(self, name): async def _verify_executable(self, name):
try: try:
version, code = await self._execute(name, "-version") version, code = await self._execute(name, "-version")
except Exception as e: except Exception as e:
code = -1 code = -1
version = str(e) version = str(e)
if code != 0 or not version.startswith(name): if code != 0 or not version.startswith(str(pathlib.Path(name).stem)):
log.warning("Unable to run %s, but it was requested. Code: %d; Message: %s", name, code, version) log.warning("Unable to run %s, but it was requested. Code: %d; Message: %s", name, code, version)
raise FileNotFoundError(f"Unable to locate or run {name}. Please install FFmpeg " raise FileNotFoundError(f"Unable to locate or run {name}. Please install FFmpeg "
f"and ensure that it is callable via PATH or conf.ffmpeg_folder") f"and ensure that it is callable via PATH or conf.ffmpeg_path")
return version return version
async def _verify_ffmpeg_installed(self): async def _verify_ffmpeg_installed(self):
if self._ffmpeg_installed: if self._ffmpeg_installed:
return return
await self._verify_executable("ffprobe") self._ffmpeg_installed = False
version = await self._verify_executable("ffmpeg") path = self._conf.ffmpeg_path
self._which = shutil.which(os.path.join(self._conf.ffmpeg_folder, "ffmpeg")) if hasattr(self._conf, "data_dir"):
self._ffmpeg_installed = True path += os.path.pathsep + os.path.join(getattr(self._conf, "data_dir"), "ffmpeg", "bin")
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which) path += os.path.pathsep + self._env_copy.get("PATH", "")
async def status(self, reset=False, recheck=False): self._whichFFmpeg = shutil.which("ffmpeg", path=path)
if not self._whichFFmpeg:
log.warning("Unable to locate ffmpeg executable. Path: %s", path)
raise FileNotFoundError(f"Unable to locate ffmpeg executable. Path: {path}")
self._whichFFprobe = shutil.which("ffprobe", path=path)
if not self._whichFFprobe:
log.warning("Unable to locate ffprobe executable. Path: %s", path)
raise FileNotFoundError(f"Unable to locate ffprobe executable. Path: {path}")
if os.path.dirname(self._whichFFmpeg) != os.path.dirname(self._whichFFprobe):
log.warning("ffmpeg and ffprobe are in different folders!")
await self._verify_executable(self._whichFFprobe)
version = await self._verify_executable(self._whichFFmpeg)
self._ffmpeg_installed = True
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._whichFFmpeg)
async def status(self, reset=False):
if reset: if reset:
self._available_encoders = "" self._available_encoders = ""
self._ffmpeg_installed = False self._ffmpeg_installed = None
self._which = None if self._ffmpeg_installed is None:
if self._checked_ffmpeg and not recheck:
installed = self._ffmpeg_installed
else:
installed = True
try: try:
await self._verify_ffmpeg_installed() await self._verify_ffmpeg_installed()
except FileNotFoundError: except FileNotFoundError:
installed = False pass
self._checked_ffmpeg = True
return { return {
"available": installed, "available": self._ffmpeg_installed,
"which": self._which, "which": self._whichFFmpeg,
"analyze_audio_volume": int(self._conf.volume_analysis_time) > 0 "analyze_audio_volume": int(self._conf.volume_analysis_time) > 0
} }
@ -135,7 +153,7 @@ class VideoFileAnalyzer:
if {"webm", "ogg"}.intersection(container.split(",")): if {"webm", "ogg"}.intersection(container.split(",")):
return "" return ""
result, _ = await self._execute("ffprobe", f'-v debug "{video_file}"') result, _ = await self._execute_ffprobe(f'-v debug "{video_file}"')
match = re.search(r"Before avformat_find_stream_info.+?\s+seeks:(\d+)\s+", result) match = re.search(r"Before avformat_find_stream_info.+?\s+seeks:(\d+)\s+", result)
if match and int(match.group(1)) != 0: if match and int(match.group(1)) != 0:
return "Video stream descriptors are not at the start of the file (the faststart flag was not used)." return "Video stream descriptors are not at the start of the file (the faststart flag was not used)."
@ -163,7 +181,7 @@ class VideoFileAnalyzer:
if not validate_volume: if not validate_volume:
return "" return ""
result, _ = await self._execute("ffmpeg", f'-i "{video_file}" -t {seconds} ' result, _ = await self._execute_ffmpeg(f'-i "{video_file}" -t {seconds} '
f'-af volumedetect -vn -sn -dn -f null "{os.devnull}"') f'-af volumedetect -vn -sn -dn -f null "{os.devnull}"')
try: try:
mean_volume = float(re.search(r"mean_volume:\s+([-+]?\d*\.\d+|\d+)", result).group(1)) mean_volume = float(re.search(r"mean_volume:\s+([-+]?\d*\.\d+|\d+)", result).group(1))
@ -199,7 +217,7 @@ class VideoFileAnalyzer:
# if we don't have h264 use vp9; it's fairly compatible even though it's slow # if we don't have h264 use vp9; it's fairly compatible even though it's slow
if not self._available_encoders: if not self._available_encoders:
self._available_encoders, _ = await self._execute("ffmpeg", "-encoders -v quiet") self._available_encoders, _ = await self._execute_ffmpeg("-encoders -v quiet")
encoder = self._conf.video_encoder.split(" ", 1)[0] encoder = self._conf.video_encoder.split(" ", 1)[0]
if re.search(fr"^\s*V..... {encoder} ", self._available_encoders, re.MULTILINE): if re.search(fr"^\s*V..... {encoder} ", self._available_encoders, re.MULTILINE):
@ -233,7 +251,7 @@ class VideoFileAnalyzer:
wants_opus = extension != "mp4" wants_opus = extension != "mp4"
if not self._available_encoders: if not self._available_encoders:
self._available_encoders, _ = await self._execute("ffmpeg", "-encoders -v quiet") self._available_encoders, _ = await self._execute_ffmpeg("-encoders -v quiet")
encoder = self._conf.audio_encoder.split(" ", 1)[0] encoder = self._conf.audio_encoder.split(" ", 1)[0]
if wants_opus and 'opus' in encoder: if wants_opus and 'opus' in encoder:
@ -283,8 +301,8 @@ class VideoFileAnalyzer:
return "mp4" return "mp4"
async def _get_scan_data(self, validate, file_path): async def _get_scan_data(self, validate, file_path):
result, _ = await self._execute("ffprobe", arguments = f'-v quiet -print_format json -show_format -show_streams "{file_path}"'
f'-v quiet -print_format json -show_format -show_streams "{file_path}"') result, _ = await self._execute_ffprobe(arguments)
try: try:
scan_data = json.loads(result) scan_data = json.loads(result)
except Exception as e: except Exception as e:
@ -293,7 +311,7 @@ class VideoFileAnalyzer:
if "format" not in scan_data or "duration" not in scan_data["format"]: if "format" not in scan_data or "duration" not in scan_data["format"]:
log.debug("Format data is missing from ffprobe results for: %s", file_path) log.debug("Format data is missing from ffprobe results for: %s", file_path)
raise ValueError(f'Media file does not appear to contain video content at: {file_path}') raise ValueError(f'Media file does not appear to contain video content: {file_path}')
if float(scan_data["format"]["duration"]) < 0.1: if float(scan_data["format"]["duration"]) < 0.1:
log.debug("Media file appears to be an image: %s", file_path) log.debug("Media file appears to be an image: %s", file_path)
@ -352,7 +370,6 @@ class VideoFileAnalyzer:
transcode_command.append("copy") transcode_command.append("copy")
transcode_command.append("-movflags +faststart -c:a") transcode_command.append("-movflags +faststart -c:a")
path = pathlib.Path(file_path)
extension = self._get_best_container_extension(scan_data, video_encoder) extension = self._get_best_container_extension(scan_data, video_encoder)
if audio_msg or volume_msg: if audio_msg or volume_msg:
@ -365,12 +382,13 @@ class VideoFileAnalyzer:
transcode_command.append("copy") transcode_command.append("copy")
# TODO: put it in a temp folder and delete it after we upload? # TODO: put it in a temp folder and delete it after we upload?
path = pathlib.Path(file_path)
output = path.parent / f"{path.stem}_fixed.{extension}" output = path.parent / f"{path.stem}_fixed.{extension}"
transcode_command.append(f'"{output}"') transcode_command.append(f'"{output}"')
ffmpeg_command = " ".join(transcode_command) ffmpeg_command = " ".join(transcode_command)
log.info("Proceeding on transcode via: ffmpeg %s", ffmpeg_command) log.info("Proceeding on transcode via: ffmpeg %s", ffmpeg_command)
result, code = await self._execute("ffmpeg", ffmpeg_command) result, code = await self._execute_ffmpeg(ffmpeg_command)
if code != 0: if code != 0:
raise Exception(f"Failure to complete the transcode command. Output: {result}") raise Exception(f"Failure to complete the transcode command. Output: {result}")
except Exception as e: except Exception as e:

5
tests/integration/other/test_transcoding.py
View file

@ -129,6 +129,8 @@ class TranscodeValidation(ClaimTestCase):
async def test_extension_choice(self): async def test_extension_choice(self):
self.assertTrue((await self.analyzer.status())["available"])
scan_data = await self.analyzer._get_scan_data(True, self.video_file_name) scan_data = await self.analyzer._get_scan_data(True, self.video_file_name)
extension = self.analyzer._get_best_container_extension(scan_data, "") extension = self.analyzer._get_best_container_extension(scan_data, "")
self.assertEqual(extension, pathlib.Path(self.video_file_name).suffix[1:]) self.assertEqual(extension, pathlib.Path(self.video_file_name).suffix[1:])
@ -151,6 +153,7 @@ class TranscodeValidation(ClaimTestCase):
self.assertEqual("ogv", extension) self.assertEqual("ogv", extension)
async def test_no_ffmpeg(self): async def test_no_ffmpeg(self):
self.conf.ffmpeg_folder = "I don't really exist/" self.conf.ffmpeg_path = "I don't really exist/"
self.analyzer._env_copy.pop("PATH", None)
with self.assertRaisesRegex(Exception, "Unable to locate"): with self.assertRaisesRegex(Exception, "Unable to locate"):
await self.analyzer.verify_or_repair(True, False, self.video_file_name) await self.analyzer.verify_or_repair(True, False, self.video_file_name)