mirror of
https://github.com/yt-dlp/yt-dlp.git
synced 2026-07-22 17:37:16 +00:00
[fd/external] aria2c: Remove support for m3u8/dash protocols
See https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-vx4q-3cr2-7cg2 Authored by: bashonly
This commit is contained in:
bashonly
+11
-131
@@ -1,17 +1,14 @@
|
||||
import enum
|
||||
import functools
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from .fragment import FragmentFD
|
||||
from ..networking import Request
|
||||
from ..postprocessor.ffmpeg import EXT_TO_OUT_FORMATS, FFmpegPostProcessor
|
||||
from ..utils import (
|
||||
Popen,
|
||||
@@ -25,7 +22,6 @@ from ..utils import (
|
||||
cli_valueless_option,
|
||||
determine_ext,
|
||||
encodeArgument,
|
||||
find_available_port,
|
||||
remove_end,
|
||||
traverse_obj,
|
||||
version_tuple,
|
||||
@@ -309,38 +305,17 @@ class WgetFD(ExternalFD):
|
||||
|
||||
class Aria2cFD(ExternalFD):
|
||||
AVAILABLE_OPT = '-v'
|
||||
SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps', 'dash_frag_urls', 'm3u8_frag_urls')
|
||||
|
||||
@staticmethod
|
||||
def supports_manifest(manifest):
|
||||
UNSUPPORTED_FEATURES = [
|
||||
r'#EXT-X-BYTERANGE', # playlists composed of byte ranges of media files [1]
|
||||
# 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.2
|
||||
]
|
||||
check_results = (not re.search(feature, manifest) for feature in UNSUPPORTED_FEATURES)
|
||||
return all(check_results)
|
||||
SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps')
|
||||
|
||||
@staticmethod
|
||||
def _aria2c_filename(fn):
|
||||
return fn if os.path.isabs(fn) else f'.{os.path.sep}{fn}'
|
||||
|
||||
def _call_downloader(self, tmpfilename, info_dict):
|
||||
# FIXME: Disabled due to https://github.com/yt-dlp/yt-dlp/issues/5931
|
||||
if False and 'no-external-downloader-progress' not in self.params.get('compat_opts', []):
|
||||
info_dict['__rpc'] = {
|
||||
'port': find_available_port() or 19190,
|
||||
'secret': str(uuid.uuid4()),
|
||||
}
|
||||
return super()._call_downloader(tmpfilename, info_dict)
|
||||
|
||||
def _make_cmd(self, tmpfilename, info_dict):
|
||||
cmd = [self.exe, '-c', '--no-conf',
|
||||
'--console-log-level=warn', '--summary-interval=0', '--download-result=hide',
|
||||
'--http-accept-gzip=true', '--file-allocation=none', '-x16', '-j16', '-s16']
|
||||
if 'fragments' in info_dict:
|
||||
cmd += ['--allow-overwrite=true', '--allow-piece-length-change=true']
|
||||
else:
|
||||
cmd += ['--min-split-size', '1M']
|
||||
'--http-accept-gzip=true', '--file-allocation=none', '-x16', '-j16', '-s16',
|
||||
'--min-split-size', '1M']
|
||||
|
||||
cmd += [f'--load-cookies={self._write_cookies()}']
|
||||
if info_dict.get('http_headers') is not None:
|
||||
@@ -354,12 +329,6 @@ class Aria2cFD(ExternalFD):
|
||||
cmd += self._bool_option('--show-console-readout', 'noprogress', 'false', 'true', '=')
|
||||
cmd += self._configuration_args()
|
||||
|
||||
if '__rpc' in info_dict:
|
||||
cmd += [
|
||||
'--enable-rpc',
|
||||
f'--rpc-listen-port={info_dict["__rpc"]["port"]}',
|
||||
f'--rpc-secret={info_dict["__rpc"]["secret"]}']
|
||||
|
||||
# aria2c strips out spaces from the beginning/end of filenames and paths.
|
||||
# We work around this issue by adding a "./" to the beginning of the
|
||||
# filename and relative path, and adding a "/" at the end of the path.
|
||||
@@ -369,106 +338,17 @@ class Aria2cFD(ExternalFD):
|
||||
dn = os.path.dirname(tmpfilename)
|
||||
if dn:
|
||||
cmd += ['--dir', self._aria2c_filename(dn) + os.path.sep]
|
||||
if 'fragments' not in info_dict:
|
||||
cmd += ['--out', self._aria2c_filename(os.path.basename(tmpfilename))]
|
||||
cmd += ['--auto-file-renaming=false']
|
||||
|
||||
if 'fragments' in info_dict:
|
||||
cmd += ['--uri-selector=inorder']
|
||||
url_list_file = f'{tmpfilename}.frag.urls'
|
||||
url_list = []
|
||||
for frag_index, fragment in enumerate(info_dict['fragments']):
|
||||
fragment_filename = f'{os.path.basename(tmpfilename)}-Frag{frag_index}'
|
||||
url_list.append('{}\n\tout={}'.format(fragment['url'], self._aria2c_filename(fragment_filename)))
|
||||
stream, _ = self.sanitize_open(url_list_file, 'wb')
|
||||
stream.write('\n'.join(url_list).encode())
|
||||
stream.close()
|
||||
cmd += ['-i', self._aria2c_filename(url_list_file)]
|
||||
else:
|
||||
cmd += ['--', info_dict['url']]
|
||||
cmd += [
|
||||
'--out',
|
||||
self._aria2c_filename(os.path.basename(tmpfilename)),
|
||||
'--auto-file-renaming=false',
|
||||
'--',
|
||||
info_dict['url'],
|
||||
]
|
||||
|
||||
return cmd
|
||||
|
||||
def aria2c_rpc(self, rpc_port, rpc_secret, method, params=()):
|
||||
# Does not actually need to be UUID, just unique
|
||||
sanitycheck = str(uuid.uuid4())
|
||||
d = json.dumps({
|
||||
'jsonrpc': '2.0',
|
||||
'id': sanitycheck,
|
||||
'method': method,
|
||||
'params': [f'token:{rpc_secret}', *params],
|
||||
}).encode()
|
||||
request = Request(
|
||||
f'http://localhost:{rpc_port}/jsonrpc',
|
||||
data=d, headers={
|
||||
'Content-Type': 'application/json',
|
||||
'Content-Length': f'{len(d)}',
|
||||
}, proxies={'all': None})
|
||||
with self.ydl.urlopen(request) as r:
|
||||
resp = json.load(r)
|
||||
assert resp.get('id') == sanitycheck, 'Something went wrong with RPC server'
|
||||
return resp['result']
|
||||
|
||||
def _call_process(self, cmd, info_dict):
|
||||
if '__rpc' not in info_dict:
|
||||
return super()._call_process(cmd, info_dict)
|
||||
|
||||
send_rpc = functools.partial(self.aria2c_rpc, info_dict['__rpc']['port'], info_dict['__rpc']['secret'])
|
||||
started = time.time()
|
||||
|
||||
fragmented = 'fragments' in info_dict
|
||||
frag_count = len(info_dict['fragments']) if fragmented else 1
|
||||
status = {
|
||||
'filename': info_dict.get('_filename'),
|
||||
'status': 'downloading',
|
||||
'elapsed': 0,
|
||||
'downloaded_bytes': 0,
|
||||
'fragment_count': frag_count if fragmented else None,
|
||||
'fragment_index': 0 if fragmented else None,
|
||||
}
|
||||
self._hook_progress(status, info_dict)
|
||||
|
||||
def get_stat(key, *obj, average=False):
|
||||
val = tuple(filter(None, map(float, traverse_obj(obj, (..., ..., key))))) or [0]
|
||||
return sum(val) / (len(val) if average else 1)
|
||||
|
||||
with Popen(cmd, text=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) as p:
|
||||
# Add a small sleep so that RPC client can receive response,
|
||||
# or the connection stalls infinitely
|
||||
time.sleep(0.2)
|
||||
retval = p.poll()
|
||||
while retval is None:
|
||||
# We don't use tellStatus as we won't know the GID without reading stdout
|
||||
# Ref: https://aria2.github.io/manual/en/html/aria2c.html#aria2.tellActive
|
||||
active = send_rpc('aria2.tellActive')
|
||||
completed = send_rpc('aria2.tellStopped', [0, frag_count])
|
||||
|
||||
downloaded = get_stat('totalLength', completed) + get_stat('completedLength', active)
|
||||
speed = get_stat('downloadSpeed', active)
|
||||
total = frag_count * get_stat('totalLength', active, completed, average=True)
|
||||
if total < downloaded:
|
||||
total = None
|
||||
|
||||
status.update({
|
||||
'downloaded_bytes': int(downloaded),
|
||||
'speed': speed,
|
||||
'total_bytes': None if fragmented else total,
|
||||
'total_bytes_estimate': total,
|
||||
'eta': (total - downloaded) / (speed or 1),
|
||||
'fragment_index': min(frag_count, len(completed) + 1) if fragmented else None,
|
||||
'elapsed': time.time() - started,
|
||||
})
|
||||
self._hook_progress(status, info_dict)
|
||||
|
||||
if not active and len(completed) >= frag_count:
|
||||
send_rpc('aria2.shutdown')
|
||||
retval = p.wait()
|
||||
break
|
||||
|
||||
time.sleep(0.1)
|
||||
retval = p.poll()
|
||||
|
||||
return '', p.stderr.read(), retval
|
||||
|
||||
|
||||
class HttpieFD(ExternalFD):
|
||||
AVAILABLE_OPT = '--version'
|
||||
|
||||
Reference in New Issue
Block a user