[ie/youtube] Implement external n/sig solver (#14157)
Closes #14404, Closes #14431, Closes #14680, Closes #14707 Authored by: bashonly, coletdjnz, seproDev, Grub4K Co-authored-by: coletdjnz <coletdjnz@protonmail.com> Co-authored-by: bashonly <bashonly@protonmail.com> Co-authored-by: sepro <sepro@sepr0.com>
This commit is contained in:
parent
d6ee677253
commit
6224a38988
45 changed files with 3400 additions and 1140 deletions
0
yt_dlp/extractor/youtube/jsc/_builtin/__init__.py
Normal file
0
yt_dlp/extractor/youtube/jsc/_builtin/__init__.py
Normal file
146
yt_dlp/extractor/youtube/jsc/_builtin/bun.py
Normal file
146
yt_dlp/extractor/youtube/jsc/_builtin/bun.py
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
import urllib.parse
|
||||
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.ejs import (
|
||||
_EJS_WIKI_URL,
|
||||
EJSBaseJCP,
|
||||
Script,
|
||||
ScriptSource,
|
||||
ScriptType,
|
||||
ScriptVariant,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.vendor import load_script
|
||||
from yt_dlp.extractor.youtube.jsc.provider import (
|
||||
JsChallengeProvider,
|
||||
JsChallengeProviderError,
|
||||
JsChallengeRequest,
|
||||
register_preference,
|
||||
register_provider,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.pot._provider import BuiltinIEContentProvider
|
||||
from yt_dlp.extractor.youtube.pot.provider import provider_bug_report_message
|
||||
from yt_dlp.utils import Popen
|
||||
from yt_dlp.utils.networking import HTTPHeaderDict, clean_proxies
|
||||
|
||||
# KNOWN ISSUES:
|
||||
# - If node_modules is present and includes a requested lib, the version we request is ignored
|
||||
# and whatever installed in node_modules is used.
|
||||
# - No way to ignore existing node_modules, lock files, etc.
|
||||
# - No sandboxing options available
|
||||
# - Cannot detect if npm packages are cached without potentially downloading them.
|
||||
# `--no-install` appears to disable the cache.
|
||||
# - npm auto-install may fail with an integrity error when using HTTP proxies
|
||||
# - npm auto-install HTTP proxy support may be limited on older Bun versions
|
||||
|
||||
|
||||
@register_provider
|
||||
class BunJCP(EJSBaseJCP, BuiltinIEContentProvider):
|
||||
PROVIDER_NAME = 'bun'
|
||||
JS_RUNTIME_NAME = 'bun'
|
||||
BUN_NPM_LIB_FILENAME = 'yt.solver.bun.lib.js'
|
||||
SUPPORTED_PROXY_SCHEMES = ['http', 'https']
|
||||
|
||||
def _iter_script_sources(self):
|
||||
yield from super()._iter_script_sources()
|
||||
yield ScriptSource.BUILTIN, self._bun_npm_source
|
||||
|
||||
def _bun_npm_source(self, script_type: ScriptType, /):
|
||||
if script_type != ScriptType.LIB:
|
||||
return None
|
||||
if 'ejs:npm' not in self.ie.get_param('remote_components', []):
|
||||
return self._skip_component('ejs:npm')
|
||||
|
||||
# Check to see if the environment proxies are compatible with Bun npm source
|
||||
if unsupported_scheme := self._check_env_proxies(self._get_env_options()):
|
||||
self.logger.warning(
|
||||
f'Bun NPM package downloads only support HTTP/HTTPS proxies; skipping remote NPM package downloads. '
|
||||
f'Provide another distribution of the challenge solver script or use '
|
||||
f'another JS runtime that supports "{unsupported_scheme}" proxies. '
|
||||
f'For more information and alternatives, refer to {_EJS_WIKI_URL}')
|
||||
return None
|
||||
|
||||
# Bun-specific lib scripts that uses Bun autoimport
|
||||
# https://bun.com/docs/runtime/autoimport
|
||||
error_hook = lambda e: self.logger.warning(
|
||||
f'Failed to read bun challenge solver lib script: {e}{provider_bug_report_message(self)}')
|
||||
code = load_script(
|
||||
self.BUN_NPM_LIB_FILENAME, error_hook=error_hook)
|
||||
if code:
|
||||
return Script(script_type, ScriptVariant.BUN_NPM, ScriptSource.BUILTIN, self._SCRIPT_VERSION, code)
|
||||
return None
|
||||
|
||||
def _check_env_proxies(self, env):
|
||||
# check that the schemes of both HTTP_PROXY and HTTPS_PROXY are supported
|
||||
for key in ('HTTP_PROXY', 'HTTPS_PROXY'):
|
||||
proxy = env.get(key)
|
||||
if not proxy:
|
||||
continue
|
||||
scheme = urllib.parse.urlparse(proxy).scheme.lower()
|
||||
if scheme not in self.SUPPORTED_PROXY_SCHEMES:
|
||||
return scheme
|
||||
return None
|
||||
|
||||
def _get_env_options(self) -> dict[str, str]:
|
||||
options = os.environ.copy() # pass through existing bun env vars
|
||||
request_proxies = self.ie._downloader.proxies.copy()
|
||||
clean_proxies(request_proxies, HTTPHeaderDict())
|
||||
|
||||
# Apply 'all' proxy first, then allow per-scheme overrides
|
||||
if request_proxies.get('all') is not None:
|
||||
options['HTTP_PROXY'] = options['HTTPS_PROXY'] = request_proxies['all']
|
||||
for key, env in (('http', 'HTTP_PROXY'), ('https', 'HTTPS_PROXY')):
|
||||
val = request_proxies.get(key)
|
||||
if val is not None:
|
||||
options[env] = val
|
||||
if self.ie.get_param('nocheckcertificate'):
|
||||
options['NODE_TLS_REJECT_UNAUTHORIZED'] = '0'
|
||||
|
||||
# Prevent segfault: <https://github.com/oven-sh/bun/issues/22901>
|
||||
options.pop('JSC_useJIT', None)
|
||||
if self.ejs_setting('jitless', ['false']) != ['false']:
|
||||
options['BUN_JSC_useJIT'] = '0'
|
||||
|
||||
return options
|
||||
|
||||
def _run_js_runtime(self, stdin: str, /) -> str:
|
||||
# https://bun.com/docs/cli/run
|
||||
options = ['--no-addons', '--prefer-offline']
|
||||
if self._lib_script.variant == ScriptVariant.BUN_NPM:
|
||||
# Enable auto-install even if node_modules is present
|
||||
options.append('--install=fallback')
|
||||
else:
|
||||
options.append('--no-install')
|
||||
cmd = [self.runtime_info.path, '--bun', 'run', *options, '-']
|
||||
self.logger.debug(f'Running bun: {shlex.join(cmd)}')
|
||||
|
||||
with Popen(
|
||||
cmd,
|
||||
text=True,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
env=self._get_env_options(),
|
||||
) as proc:
|
||||
stdout, stderr = proc.communicate_or_kill(stdin)
|
||||
stderr = self._clean_stderr(stderr)
|
||||
if proc.returncode or stderr:
|
||||
msg = f'Error running bun process (returncode: {proc.returncode})'
|
||||
if stderr:
|
||||
msg = f'{msg}: {stderr.strip()}'
|
||||
raise JsChallengeProviderError(msg)
|
||||
return stdout
|
||||
|
||||
def _clean_stderr(self, stderr):
|
||||
return '\n'.join(
|
||||
line for line in stderr.splitlines()
|
||||
if not re.match(r'^Bun v\d+\.\d+\.\d+ \([\w\s]+\)$', line))
|
||||
|
||||
|
||||
@register_preference(BunJCP)
|
||||
def preference(provider: JsChallengeProvider, requests: list[JsChallengeRequest]) -> int:
|
||||
return 800
|
||||
125
yt_dlp/extractor/youtube/jsc/_builtin/deno.py
Normal file
125
yt_dlp/extractor/youtube/jsc/_builtin/deno.py
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.ejs import (
|
||||
EJSBaseJCP,
|
||||
Script,
|
||||
ScriptSource,
|
||||
ScriptType,
|
||||
ScriptVariant,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.vendor import load_script
|
||||
from yt_dlp.extractor.youtube.jsc.provider import (
|
||||
JsChallengeProvider,
|
||||
JsChallengeProviderError,
|
||||
JsChallengeRequest,
|
||||
register_preference,
|
||||
register_provider,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.pot._provider import BuiltinIEContentProvider
|
||||
from yt_dlp.extractor.youtube.pot.provider import provider_bug_report_message
|
||||
from yt_dlp.utils import Popen, remove_terminal_sequences
|
||||
from yt_dlp.utils.networking import HTTPHeaderDict, clean_proxies
|
||||
|
||||
|
||||
@register_provider
|
||||
class DenoJCP(EJSBaseJCP, BuiltinIEContentProvider):
|
||||
PROVIDER_NAME = 'deno'
|
||||
JS_RUNTIME_NAME = 'deno'
|
||||
|
||||
_DENO_BASE_OPTIONS = ['--no-prompt', '--no-remote', '--no-lock', '--node-modules-dir=none', '--no-config']
|
||||
DENO_NPM_LIB_FILENAME = 'yt.solver.deno.lib.js'
|
||||
_NPM_PACKAGES_CACHED = False
|
||||
|
||||
def _iter_script_sources(self):
|
||||
yield from super()._iter_script_sources()
|
||||
yield ScriptSource.BUILTIN, self._deno_npm_source
|
||||
|
||||
def _deno_npm_source(self, script_type: ScriptType, /):
|
||||
if script_type != ScriptType.LIB:
|
||||
return None
|
||||
# Deno-specific lib scripts that use Deno NPM imports
|
||||
error_hook = lambda e: self.logger.warning(
|
||||
f'Failed to read deno challenge solver lib script: {e}{provider_bug_report_message(self)}')
|
||||
code = load_script(
|
||||
self.DENO_NPM_LIB_FILENAME, error_hook=error_hook)
|
||||
if not code:
|
||||
return None
|
||||
if 'ejs:npm' not in self.ie.get_param('remote_components', []):
|
||||
# We may still be able to continue if the npm packages are available/cached
|
||||
self._NPM_PACKAGES_CACHED = self._npm_packages_cached(code)
|
||||
if not self._NPM_PACKAGES_CACHED:
|
||||
return self._skip_component('ejs:npm')
|
||||
return Script(script_type, ScriptVariant.DENO_NPM, ScriptSource.BUILTIN, self._SCRIPT_VERSION, code)
|
||||
|
||||
def _npm_packages_cached(self, stdin: str) -> bool:
|
||||
# Check if npm packages are cached, so we can run without --remote-components ejs:npm
|
||||
self.logger.debug('Checking if npm packages are cached')
|
||||
try:
|
||||
self._run_deno(stdin, [*self._DENO_BASE_OPTIONS, '--cached-only'])
|
||||
except JsChallengeProviderError as e:
|
||||
self.logger.trace(f'Deno npm packages not cached: {e}')
|
||||
return False
|
||||
return True
|
||||
|
||||
def _run_js_runtime(self, stdin: str, /) -> str:
|
||||
options = [*self._DENO_BASE_OPTIONS]
|
||||
if self._lib_script.variant == ScriptVariant.DENO_NPM and self._NPM_PACKAGES_CACHED:
|
||||
options.append('--cached-only')
|
||||
elif self._lib_script.variant != ScriptVariant.DENO_NPM:
|
||||
options.append('--no-npm')
|
||||
options.append('--cached-only')
|
||||
if self.ie.get_param('nocheckcertificate'):
|
||||
options.append('--unsafely-ignore-certificate-errors')
|
||||
# XXX: Convert this extractor-arg into a general option if/when a JSI framework is implemented
|
||||
if self.ejs_setting('jitless', ['false']) != ['false']:
|
||||
options.append('--v8-flags=--jitless')
|
||||
return self._run_deno(stdin, options)
|
||||
|
||||
def _get_env_options(self) -> dict[str, str]:
|
||||
options = os.environ.copy() # pass through existing deno env vars
|
||||
request_proxies = self.ie._downloader.proxies.copy()
|
||||
clean_proxies(request_proxies, HTTPHeaderDict())
|
||||
# Apply 'all' proxy first, then allow per-scheme overrides
|
||||
if 'all' in request_proxies and request_proxies['all'] is not None:
|
||||
options['HTTP_PROXY'] = options['HTTPS_PROXY'] = request_proxies['all']
|
||||
for key, env in (('http', 'HTTP_PROXY'), ('https', 'HTTPS_PROXY'), ('no', 'NO_PROXY')):
|
||||
if key in request_proxies and request_proxies[key] is not None:
|
||||
options[env] = request_proxies[key]
|
||||
return options
|
||||
|
||||
def _run_deno(self, stdin, options) -> str:
|
||||
cmd = [self.runtime_info.path, 'run', *options, '-']
|
||||
self.logger.debug(f'Running deno: {shlex.join(cmd)}')
|
||||
with Popen(
|
||||
cmd,
|
||||
text=True,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
env=self._get_env_options(),
|
||||
) as proc:
|
||||
stdout, stderr = proc.communicate_or_kill(stdin)
|
||||
stderr = self._clean_stderr(stderr)
|
||||
if proc.returncode or stderr:
|
||||
msg = f'Error running deno process (returncode: {proc.returncode})'
|
||||
if stderr:
|
||||
msg = f'{msg}: {stderr.strip()}'
|
||||
raise JsChallengeProviderError(msg)
|
||||
return stdout
|
||||
|
||||
def _clean_stderr(self, stderr):
|
||||
return '\n'.join(
|
||||
line for line in stderr.splitlines()
|
||||
if not (
|
||||
re.match(r'^Download\s+https\S+$', remove_terminal_sequences(line))
|
||||
or re.match(r'DANGER: TLS certificate validation is disabled for all hostnames', remove_terminal_sequences(line))))
|
||||
|
||||
|
||||
@register_preference(DenoJCP)
|
||||
def preference(provider: JsChallengeProvider, requests: list[JsChallengeRequest]) -> int:
|
||||
return 1000
|
||||
326
yt_dlp/extractor/youtube/jsc/_builtin/ejs.py
Normal file
326
yt_dlp/extractor/youtube/jsc/_builtin/ejs.py
Normal file
|
|
@ -0,0 +1,326 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import collections
|
||||
import dataclasses
|
||||
import enum
|
||||
import functools
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
from yt_dlp.dependencies import yt_dlp_ejs as _has_ejs
|
||||
from yt_dlp.extractor.youtube.jsc._builtin import vendor
|
||||
from yt_dlp.extractor.youtube.jsc.provider import (
|
||||
JsChallengeProvider,
|
||||
JsChallengeProviderError,
|
||||
JsChallengeProviderRejectedRequest,
|
||||
JsChallengeProviderResponse,
|
||||
JsChallengeResponse,
|
||||
JsChallengeType,
|
||||
NChallengeOutput,
|
||||
SigChallengeOutput,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.pot._provider import configuration_arg
|
||||
from yt_dlp.extractor.youtube.pot.provider import provider_bug_report_message
|
||||
from yt_dlp.utils._jsruntime import JsRuntimeInfo
|
||||
|
||||
if _has_ejs:
|
||||
import yt_dlp_ejs.yt.solver
|
||||
|
||||
TYPE_CHECKING = False
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable, Generator
|
||||
|
||||
from yt_dlp.extractor.youtube.jsc.provider import JsChallengeRequest
|
||||
|
||||
_EJS_WIKI_URL = 'https://github.com/yt-dlp/yt-dlp/wiki/EJS'
|
||||
|
||||
|
||||
class ScriptType(enum.Enum):
|
||||
LIB = 'lib'
|
||||
CORE = 'core'
|
||||
|
||||
|
||||
class ScriptVariant(enum.Enum):
|
||||
UNKNOWN = 'unknown'
|
||||
MINIFIED = 'minified'
|
||||
UNMINIFIED = 'unminified'
|
||||
DENO_NPM = 'deno_npm'
|
||||
BUN_NPM = 'bun_npm'
|
||||
|
||||
|
||||
class ScriptSource(enum.Enum):
|
||||
PYPACKAGE = 'python package' # PyPI, PyInstaller exe, zipimport binary, etc
|
||||
CACHE = 'cache' # GitHub release assets (cached)
|
||||
WEB = 'web' # GitHub release assets (downloaded)
|
||||
BUILTIN = 'builtin' # vendored (full core script; import-only lib script + NPM cache)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Script:
|
||||
type: ScriptType
|
||||
variant: ScriptVariant
|
||||
source: ScriptSource
|
||||
version: str
|
||||
code: str
|
||||
|
||||
@functools.cached_property
|
||||
def hash(self, /) -> str:
|
||||
return hashlib.sha3_512(self.code.encode()).hexdigest()
|
||||
|
||||
def __str__(self, /):
|
||||
return f'<Script {self.type.value!r} v{self.version} (source: {self.source.value}) variant={self.variant.value!r} size={len(self.code)} hash={self.hash[:7]}...>'
|
||||
|
||||
|
||||
class EJSBaseJCP(JsChallengeProvider):
|
||||
JS_RUNTIME_NAME: str
|
||||
_CACHE_SECTION = 'challenge-solver'
|
||||
|
||||
_REPOSITORY = 'yt-dlp/ejs'
|
||||
_SUPPORTED_TYPES = [JsChallengeType.N, JsChallengeType.SIG]
|
||||
_SCRIPT_VERSION = vendor.VERSION
|
||||
# TODO: Integration tests for each kind of scripts source
|
||||
_ALLOWED_HASHES = {
|
||||
ScriptType.LIB: {
|
||||
ScriptVariant.UNMINIFIED: vendor.HASHES['yt.solver.lib.js'],
|
||||
ScriptVariant.MINIFIED: vendor.HASHES['yt.solver.lib.min.js'],
|
||||
ScriptVariant.DENO_NPM: vendor.HASHES['yt.solver.deno.lib.js'],
|
||||
ScriptVariant.BUN_NPM: vendor.HASHES['yt.solver.bun.lib.js'],
|
||||
},
|
||||
ScriptType.CORE: {
|
||||
ScriptVariant.MINIFIED: vendor.HASHES['yt.solver.core.min.js'],
|
||||
ScriptVariant.UNMINIFIED: vendor.HASHES['yt.solver.core.js'],
|
||||
},
|
||||
}
|
||||
|
||||
_SCRIPT_FILENAMES = {
|
||||
ScriptType.LIB: 'yt.solver.lib.js',
|
||||
ScriptType.CORE: 'yt.solver.core.js',
|
||||
}
|
||||
|
||||
_MIN_SCRIPT_FILENAMES = {
|
||||
ScriptType.LIB: 'yt.solver.lib.min.js',
|
||||
ScriptType.CORE: 'yt.solver.core.min.js',
|
||||
}
|
||||
|
||||
# currently disabled as files are large and we do not support rotation
|
||||
_ENABLE_PREPROCESSED_PLAYER_CACHE = False
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self._available = True
|
||||
self.ejs_settings = self.ie.get_param('extractor_args', {}).get('youtube-ejs', {})
|
||||
|
||||
# Note: The following 3 args are for developer use only & intentionally not documented.
|
||||
# - dev: bypasses verification of script hashes and versions.
|
||||
# - repo: use a custom GitHub repository to fetch web script from.
|
||||
# - script_version: use a custom script version.
|
||||
# E.g. --extractor-args "youtube-ejs:dev=true;script_version=0.1.4"
|
||||
|
||||
self.is_dev = self.ejs_setting('dev', ['false'])[0] == 'true'
|
||||
if self.is_dev:
|
||||
self.report_dev_option('You have enabled dev mode for EJS JCP Providers.')
|
||||
|
||||
custom_repo = self.ejs_setting('repo', [None])[0]
|
||||
if custom_repo:
|
||||
self.report_dev_option(f'You have set a custom GitHub repository for EJS JCP Providers ({custom_repo}).')
|
||||
self._REPOSITORY = custom_repo
|
||||
|
||||
custom_version = self.ejs_setting('script_version', [None])[0]
|
||||
if custom_version:
|
||||
self.report_dev_option(f'You have set a custom EJS script version for EJS JCP Providers ({custom_version}).')
|
||||
self._SCRIPT_VERSION = custom_version
|
||||
|
||||
def ejs_setting(self, key, *args, **kwargs):
|
||||
return configuration_arg(self.ejs_settings, key, *args, **kwargs)
|
||||
|
||||
def report_dev_option(self, message: str):
|
||||
self.ie.report_warning(
|
||||
f'{message} '
|
||||
f'This is a developer option intended for debugging. \n'
|
||||
' If you experience any issues while using this option, '
|
||||
f'{self.ie._downloader._format_err("DO NOT", self.ie._downloader.Styles.ERROR)} open a bug report', only_once=True)
|
||||
|
||||
def _run_js_runtime(self, stdin: str, /) -> str:
|
||||
"""To be implemented by subclasses"""
|
||||
raise NotImplementedError
|
||||
|
||||
def _real_bulk_solve(self, /, requests: list[JsChallengeRequest]):
|
||||
grouped: dict[str, list[JsChallengeRequest]] = collections.defaultdict(list)
|
||||
for request in requests:
|
||||
grouped[request.input.player_url].append(request)
|
||||
|
||||
for player_url, grouped_requests in grouped.items():
|
||||
player = None
|
||||
if self._ENABLE_PREPROCESSED_PLAYER_CACHE:
|
||||
player = self.ie.cache.load(self._CACHE_SECTION, f'player:{player_url}')
|
||||
|
||||
if player:
|
||||
cached = True
|
||||
else:
|
||||
cached = False
|
||||
video_id = next((request.video_id for request in grouped_requests), None)
|
||||
player = self._get_player(video_id, player_url)
|
||||
|
||||
# NB: This output belongs after the player request
|
||||
self.logger.info(f'Solving JS challenges using {self.JS_RUNTIME_NAME}')
|
||||
|
||||
stdin = self._construct_stdin(player, cached, grouped_requests)
|
||||
stdout = self._run_js_runtime(stdin)
|
||||
output = json.loads(stdout)
|
||||
if output['type'] == 'error':
|
||||
raise JsChallengeProviderError(output['error'])
|
||||
|
||||
if self._ENABLE_PREPROCESSED_PLAYER_CACHE and (preprocessed := output.get('preprocessed_player')):
|
||||
self.ie.cache.store(self._CACHE_SECTION, f'player:{player_url}', preprocessed)
|
||||
|
||||
for request, response_data in zip(grouped_requests, output['responses'], strict=True):
|
||||
if response_data['type'] == 'error':
|
||||
yield JsChallengeProviderResponse(request, None, response_data['error'])
|
||||
else:
|
||||
yield JsChallengeProviderResponse(request, JsChallengeResponse(request.type, (
|
||||
NChallengeOutput(response_data['data']) if request.type is JsChallengeType.N
|
||||
else SigChallengeOutput(response_data['data']))))
|
||||
|
||||
def _construct_stdin(self, player: str, preprocessed: bool, requests: list[JsChallengeRequest], /) -> str:
|
||||
json_requests = [{
|
||||
'type': request.type.value,
|
||||
'challenges': request.input.challenges,
|
||||
} for request in requests]
|
||||
data = {
|
||||
'type': 'preprocessed',
|
||||
'preprocessed_player': player,
|
||||
'requests': json_requests,
|
||||
} if preprocessed else {
|
||||
'type': 'player',
|
||||
'player': player,
|
||||
'requests': json_requests,
|
||||
'output_preprocessed': True,
|
||||
}
|
||||
return f'''\
|
||||
{self._lib_script.code}
|
||||
Object.assign(globalThis, lib);
|
||||
{self._core_script.code}
|
||||
console.log(JSON.stringify(jsc({json.dumps(data)})));
|
||||
'''
|
||||
|
||||
# region: challenge solver script
|
||||
|
||||
@functools.cached_property
|
||||
def _lib_script(self, /):
|
||||
return self._get_script(ScriptType.LIB)
|
||||
|
||||
@functools.cached_property
|
||||
def _core_script(self, /):
|
||||
return self._get_script(ScriptType.CORE)
|
||||
|
||||
def _get_script(self, script_type: ScriptType, /) -> Script:
|
||||
skipped_components: list[_SkippedComponent] = []
|
||||
for _, from_source in self._iter_script_sources():
|
||||
script = from_source(script_type)
|
||||
if not script:
|
||||
continue
|
||||
if isinstance(script, _SkippedComponent):
|
||||
skipped_components.append(script)
|
||||
continue
|
||||
if not self.is_dev:
|
||||
if script.version != self._SCRIPT_VERSION:
|
||||
self.logger.warning(
|
||||
f'Challenge solver {script_type.value} script version {script.version} '
|
||||
f'is not supported (source: {script.source.value}, variant: {script.variant}, supported version: {self._SCRIPT_VERSION})')
|
||||
if script.source is ScriptSource.CACHE:
|
||||
self.logger.debug('Clearing outdated cached script')
|
||||
self.ie.cache.store(self._CACHE_SECTION, script_type.value, None)
|
||||
continue
|
||||
script_hashes = self._ALLOWED_HASHES[script.type].get(script.variant, [])
|
||||
if script_hashes and script.hash not in script_hashes:
|
||||
self.logger.warning(
|
||||
f'Hash mismatch on challenge solver {script.type.value} script '
|
||||
f'(source: {script.source.value}, variant: {script.variant}, hash: {script.hash})!{provider_bug_report_message(self)}')
|
||||
if script.source is ScriptSource.CACHE:
|
||||
self.logger.debug('Clearing invalid cached script')
|
||||
self.ie.cache.store(self._CACHE_SECTION, script_type.value, None)
|
||||
continue
|
||||
self.logger.debug(
|
||||
f'Using challenge solver {script.type.value} script v{script.version} '
|
||||
f'(source: {script.source.value}, variant: {script.variant.value})')
|
||||
break
|
||||
|
||||
else:
|
||||
self._available = False
|
||||
raise JsChallengeProviderRejectedRequest(
|
||||
f'No usable challenge solver {script_type.value} script available',
|
||||
_skipped_components=skipped_components or None,
|
||||
)
|
||||
|
||||
return script
|
||||
|
||||
def _iter_script_sources(self) -> Generator[tuple[ScriptSource, Callable[[ScriptType], Script | None]]]:
|
||||
yield from [
|
||||
(ScriptSource.PYPACKAGE, self._pypackage_source),
|
||||
(ScriptSource.CACHE, self._cached_source),
|
||||
(ScriptSource.BUILTIN, self._builtin_source),
|
||||
(ScriptSource.WEB, self._web_release_source)]
|
||||
|
||||
def _pypackage_source(self, script_type: ScriptType, /) -> Script | None:
|
||||
if not _has_ejs:
|
||||
return None
|
||||
try:
|
||||
code = yt_dlp_ejs.yt.solver.core() if script_type is ScriptType.CORE else yt_dlp_ejs.yt.solver.lib()
|
||||
except Exception as e:
|
||||
self.logger.warning(
|
||||
f'Failed to load challenge solver {script_type.value} script from python package: {e}{provider_bug_report_message(self)}')
|
||||
return None
|
||||
return Script(script_type, ScriptVariant.MINIFIED, ScriptSource.PYPACKAGE, yt_dlp_ejs.version, code)
|
||||
|
||||
def _cached_source(self, script_type: ScriptType, /) -> Script | None:
|
||||
if data := self.ie.cache.load(self._CACHE_SECTION, script_type.value):
|
||||
return Script(script_type, ScriptVariant(data['variant']), ScriptSource.CACHE, data['version'], data['code'])
|
||||
return None
|
||||
|
||||
def _builtin_source(self, script_type: ScriptType, /) -> Script | None:
|
||||
error_hook = lambda _: self.logger.warning(
|
||||
f'Failed to read builtin challenge solver {script_type.value} script{provider_bug_report_message(self)}')
|
||||
code = vendor.load_script(
|
||||
self._SCRIPT_FILENAMES[script_type], error_hook=error_hook)
|
||||
if code:
|
||||
return Script(script_type, ScriptVariant.UNMINIFIED, ScriptSource.BUILTIN, self._SCRIPT_VERSION, code)
|
||||
return None
|
||||
|
||||
def _web_release_source(self, script_type: ScriptType, /):
|
||||
if 'ejs:github' not in (self.ie.get_param('remote_components') or ()):
|
||||
return self._skip_component('ejs:github')
|
||||
url = f'https://github.com/{self._REPOSITORY}/releases/download/{self._SCRIPT_VERSION}/{self._MIN_SCRIPT_FILENAMES[script_type]}'
|
||||
if code := self.ie._download_webpage_with_retries(
|
||||
url, None, f'[{self.logger.prefix}] Downloading challenge solver {script_type.value} script from {url}',
|
||||
f'[{self.logger.prefix}] Failed to download challenge solver {script_type.value} script', fatal=False,
|
||||
):
|
||||
self.ie.cache.store(self._CACHE_SECTION, script_type.value, {
|
||||
'version': self._SCRIPT_VERSION,
|
||||
'variant': ScriptVariant.MINIFIED.value,
|
||||
'code': code,
|
||||
})
|
||||
return Script(script_type, ScriptVariant.MINIFIED, ScriptSource.WEB, self._SCRIPT_VERSION, code)
|
||||
return None
|
||||
|
||||
# endregion: challenge solver script
|
||||
|
||||
@property
|
||||
def runtime_info(self) -> JsRuntimeInfo | None:
|
||||
runtime = self.ie._downloader._js_runtimes.get(self.JS_RUNTIME_NAME)
|
||||
if not runtime or not runtime.info or not runtime.info.supported:
|
||||
return None
|
||||
return runtime.info
|
||||
|
||||
def is_available(self, /) -> bool:
|
||||
if not self.runtime_info:
|
||||
return False
|
||||
return self._available
|
||||
|
||||
def _skip_component(self, component: str, /):
|
||||
return _SkippedComponent(component, self.JS_RUNTIME_NAME)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class _SkippedComponent:
|
||||
component: str
|
||||
runtime: str
|
||||
70
yt_dlp/extractor/youtube/jsc/_builtin/node.py
Normal file
70
yt_dlp/extractor/youtube/jsc/_builtin/node.py
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.ejs import EJSBaseJCP
|
||||
from yt_dlp.extractor.youtube.jsc.provider import (
|
||||
JsChallengeProvider,
|
||||
JsChallengeProviderError,
|
||||
JsChallengeRequest,
|
||||
register_preference,
|
||||
register_provider,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.pot._provider import BuiltinIEContentProvider
|
||||
from yt_dlp.utils import Popen
|
||||
|
||||
|
||||
@register_provider
|
||||
class NodeJCP(EJSBaseJCP, BuiltinIEContentProvider):
|
||||
PROVIDER_NAME = 'node'
|
||||
JS_RUNTIME_NAME = 'node'
|
||||
|
||||
_ARGS = ['-']
|
||||
|
||||
def _run_js_runtime(self, stdin: str, /) -> str:
|
||||
args = []
|
||||
|
||||
if self.ejs_setting('jitless', ['false']) != ['false']:
|
||||
args.append('--v8-flags=--jitless')
|
||||
|
||||
# Node permission flag changed from experimental to stable in v23.5.0
|
||||
if self.runtime_info.version_tuple < (23, 5, 0):
|
||||
args.append('--experimental-permission')
|
||||
args.append('--no-warnings=ExperimentalWarning')
|
||||
else:
|
||||
args.append('--permission')
|
||||
|
||||
cmd = [self.runtime_info.path, *args, *self._ARGS]
|
||||
self.logger.debug(f'Running node: {shlex.join(cmd)}')
|
||||
with Popen(
|
||||
cmd,
|
||||
text=True,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
) as proc:
|
||||
stdout, stderr = proc.communicate_or_kill(stdin)
|
||||
stderr = self._clean_stderr(stderr)
|
||||
if proc.returncode or stderr:
|
||||
msg = f'Error running node process (returncode: {proc.returncode})'
|
||||
if stderr:
|
||||
msg = f'{msg}: {stderr.strip()}'
|
||||
raise JsChallengeProviderError(msg)
|
||||
|
||||
return stdout
|
||||
|
||||
def _clean_stderr(self, stderr):
|
||||
return '\n'.join(
|
||||
line for line in stderr.splitlines()
|
||||
if not (
|
||||
re.match(r'^\[stdin\]:', line)
|
||||
or re.match(r'^var jsc', line)
|
||||
or '(Use `node --trace-uncaught ...` to show where the exception was thrown)' == line
|
||||
or re.match(r'^Node\.js v\d+\.\d+\.\d+$', line)))
|
||||
|
||||
|
||||
@register_preference(NodeJCP)
|
||||
def preference(provider: JsChallengeProvider, requests: list[JsChallengeRequest]) -> int:
|
||||
return 900
|
||||
59
yt_dlp/extractor/youtube/jsc/_builtin/quickjs.py
Normal file
59
yt_dlp/extractor/youtube/jsc/_builtin/quickjs.py
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import pathlib
|
||||
import shlex
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.ejs import EJSBaseJCP
|
||||
from yt_dlp.extractor.youtube.jsc.provider import (
|
||||
JsChallengeProvider,
|
||||
JsChallengeProviderError,
|
||||
JsChallengeRequest,
|
||||
register_preference,
|
||||
register_provider,
|
||||
)
|
||||
from yt_dlp.extractor.youtube.pot._provider import BuiltinIEContentProvider
|
||||
from yt_dlp.utils import Popen
|
||||
|
||||
|
||||
@register_provider
|
||||
class QuickJSJCP(EJSBaseJCP, BuiltinIEContentProvider):
|
||||
PROVIDER_NAME = 'quickjs'
|
||||
JS_RUNTIME_NAME = 'quickjs'
|
||||
|
||||
def _run_js_runtime(self, stdin: str, /) -> str:
|
||||
if self.runtime_info.name == 'quickjs-ng':
|
||||
self.logger.warning('QuickJS-NG is missing some optimizations making this very slow. Consider using upstream QuickJS instead.')
|
||||
elif self.runtime_info.version_tuple < (2025, 4, 26):
|
||||
self.logger.warning('Older QuickJS versions are missing optimizations making this very slow. Consider upgrading.')
|
||||
|
||||
# QuickJS does not support reading from stdin, so we have to use a temp file
|
||||
temp_file = tempfile.NamedTemporaryFile(mode='w', suffix='.js', delete=False, encoding='utf-8')
|
||||
try:
|
||||
temp_file.write(stdin)
|
||||
temp_file.close()
|
||||
cmd = [self.runtime_info.path, '--script', temp_file.name]
|
||||
self.logger.debug(f'Running QuickJS: {shlex.join(cmd)}')
|
||||
with Popen(
|
||||
cmd,
|
||||
text=True,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
) as proc:
|
||||
stdout, stderr = proc.communicate_or_kill()
|
||||
if proc.returncode or stderr:
|
||||
msg = f'Error running QuickJS process (returncode: {proc.returncode})'
|
||||
if stderr:
|
||||
msg = f'{msg}: {stderr.strip()}'
|
||||
raise JsChallengeProviderError(msg)
|
||||
finally:
|
||||
pathlib.Path(temp_file.name).unlink(missing_ok=True)
|
||||
|
||||
return stdout
|
||||
|
||||
|
||||
@register_preference(QuickJSJCP)
|
||||
def preference(provider: JsChallengeProvider, requests: list[JsChallengeRequest]) -> int:
|
||||
return 850
|
||||
17
yt_dlp/extractor/youtube/jsc/_builtin/vendor/__init__.py
vendored
Normal file
17
yt_dlp/extractor/youtube/jsc/_builtin/vendor/__init__.py
vendored
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
import importlib.resources
|
||||
|
||||
from yt_dlp.extractor.youtube.jsc._builtin.vendor._info import HASHES, VERSION
|
||||
|
||||
__all__ = ['HASHES', 'VERSION', 'load_script']
|
||||
|
||||
|
||||
def load_script(filename, error_hook=None):
|
||||
file = importlib.resources.files(__package__) / filename
|
||||
if file.is_file():
|
||||
try:
|
||||
return file.read_text(encoding='utf-8')
|
||||
except (OSError, FileNotFoundError, ModuleNotFoundError) as e:
|
||||
if error_hook:
|
||||
error_hook(e)
|
||||
return None
|
||||
return None
|
||||
11
yt_dlp/extractor/youtube/jsc/_builtin/vendor/_info.py
vendored
Normal file
11
yt_dlp/extractor/youtube/jsc/_builtin/vendor/_info.py
vendored
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
# This file is generated by devscripts/update_ejs.py. DO NOT MODIFY!
|
||||
|
||||
VERSION = '0.3.0'
|
||||
HASHES = {
|
||||
'yt.solver.bun.lib.js': '6ff45e94de9f0ea936a183c48173cfa9ce526ee4b7544cd556428427c1dd53c8073ef0174e79b320252bf0e7c64b0032cc1cf9c4358f3fda59033b7caa01c241',
|
||||
'yt.solver.core.js': '0cd96b2d3f319dfa62cae689efa7d930ef1706e95f5921794db5089b2262957ec0a17d73938d8975ea35d0309cbfb4c8e4418d5e219837215eee242890c8b64d',
|
||||
'yt.solver.core.min.js': '370d627703002b4a73b10027702734a3de9484f6b56b739942be1dc2b60fee49dee2aa86ed117d1c8ae1ac55181d326481f1fe2e2e8d5211154d48e2a55dac51',
|
||||
'yt.solver.deno.lib.js': '9c8ee3ab6c23e443a5a951e3ac73c6b8c1c8fb34335e7058a07bf99d349be5573611de00536dcd03ecd3cf34014c4e9b536081de37af3637c5390c6a6fd6a0f0',
|
||||
'yt.solver.lib.js': '1ee3753a8222fc855f5c39db30a9ccbb7967dbe1fb810e86dc9a89aa073a0907f294c720e9b65427d560a35aa1ce6af19ef854d9126a05ca00afe03f72047733',
|
||||
'yt.solver.lib.min.js': '8420c259ad16e99ce004e4651ac1bcabb53b4457bf5668a97a9359be9a998a789fee8ab124ee17f91a2ea8fd84e0f2b2fc8eabcaf0b16a186ba734cf422ad053',
|
||||
}
|
||||
9
yt_dlp/extractor/youtube/jsc/_builtin/vendor/yt.solver.bun.lib.js
vendored
Normal file
9
yt_dlp/extractor/youtube/jsc/_builtin/vendor/yt.solver.bun.lib.js
vendored
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
/*!
|
||||
* SPDX-License-Identifier: Unlicense
|
||||
* This file was automatically generated by https://github.com/yt-dlp/ejs
|
||||
*/
|
||||
const lib = {
|
||||
meriyah: await import('meriyah@6.1.4'),
|
||||
astring: await import('astring@1.9.0'),
|
||||
};
|
||||
export { lib };
|
||||
550
yt_dlp/extractor/youtube/jsc/_builtin/vendor/yt.solver.core.js
vendored
Normal file
550
yt_dlp/extractor/youtube/jsc/_builtin/vendor/yt.solver.core.js
vendored
Normal file
|
|
@ -0,0 +1,550 @@
|
|||
/*!
|
||||
* SPDX-License-Identifier: Unlicense
|
||||
* This file was automatically generated by https://github.com/yt-dlp/ejs
|
||||
*/
|
||||
var jsc = (function (meriyah, astring) {
|
||||
'use strict';
|
||||
function matchesStructure(obj, structure) {
|
||||
if (Array.isArray(structure)) {
|
||||
if (!Array.isArray(obj)) {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
structure.length === obj.length &&
|
||||
structure.every((value, index) => matchesStructure(obj[index], value))
|
||||
);
|
||||
}
|
||||
if (typeof structure === 'object') {
|
||||
if (!obj) {
|
||||
return !structure;
|
||||
}
|
||||
if ('or' in structure) {
|
||||
return structure.or.some((node) => matchesStructure(obj, node));
|
||||
}
|
||||
if ('anykey' in structure && Array.isArray(structure.anykey)) {
|
||||
const haystack = Array.isArray(obj) ? obj : Object.values(obj);
|
||||
return structure.anykey.every((value) =>
|
||||
haystack.some((el) => matchesStructure(el, value)),
|
||||
);
|
||||
}
|
||||
for (const [key, value] of Object.entries(structure)) {
|
||||
if (!matchesStructure(obj[key], value)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return structure === obj;
|
||||
}
|
||||
function isOneOf(value, ...of) {
|
||||
return of.includes(value);
|
||||
}
|
||||
function _optionalChain$2(ops) {
|
||||
let lastAccessLHS = undefined;
|
||||
let value = ops[0];
|
||||
let i = 1;
|
||||
while (i < ops.length) {
|
||||
const op = ops[i];
|
||||
const fn = ops[i + 1];
|
||||
i += 2;
|
||||
if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) {
|
||||
return undefined;
|
||||
}
|
||||
if (op === 'access' || op === 'optionalAccess') {
|
||||
lastAccessLHS = value;
|
||||
value = fn(value);
|
||||
} else if (op === 'call' || op === 'optionalCall') {
|
||||
value = fn((...args) => value.call(lastAccessLHS, ...args));
|
||||
lastAccessLHS = undefined;
|
||||
}
|
||||
}
|
||||
return value;
|
||||
}
|
||||
const logicalExpression = {
|
||||
type: 'ExpressionStatement',
|
||||
expression: {
|
||||
type: 'LogicalExpression',
|
||||
left: { type: 'Identifier' },
|
||||
right: {
|
||||
type: 'SequenceExpression',
|
||||
expressions: [
|
||||
{
|
||||
type: 'AssignmentExpression',
|
||||
left: { type: 'Identifier' },
|
||||
operator: '=',
|
||||
right: {
|
||||
type: 'CallExpression',
|
||||
callee: { type: 'Identifier' },
|
||||
arguments: {
|
||||
or: [
|
||||
[
|
||||
{ type: 'Literal' },
|
||||
{
|
||||
type: 'CallExpression',
|
||||
callee: {
|
||||
type: 'Identifier',
|
||||
name: 'decodeURIComponent',
|
||||
},
|
||||
arguments: [{ type: 'Identifier' }],
|
||||
optional: false,
|
||||
},
|
||||
],
|
||||
[
|
||||
{
|
||||
type: 'CallExpression',
|
||||
callee: {
|
||||
type: 'Identifier',
|
||||
name: 'decodeURIComponent',
|
||||
},
|
||||
arguments: [{ type: 'Identifier' }],
|
||||
optional: false,
|
||||
},
|
||||
],
|
||||
],
|
||||
},
|
||||
optional: false,
|
||||
},
|
||||
},
|
||||
{ type: 'CallExpression' },
|
||||
],
|
||||
},
|
||||
operator: '&&',
|
||||
},
|
||||
};
|
||||
const identifier$1 = {
|
||||
or: [
|
||||
{
|
||||
type: 'ExpressionStatement',
|
||||
expression: {
|
||||
type: 'AssignmentExpression',
|
||||
operator: '=',
|
||||
left: { type: 'Identifier' },
|
||||
right: { type: 'FunctionExpression', params: [{}, {}, {}] },
|
||||
},
|
||||
},
|
||||
{ type: 'FunctionDeclaration', params: [{}, {}, {}] },
|
||||
{
|
||||
type: 'VariableDeclaration',
|
||||
declarations: {
|
||||
anykey: [
|
||||
{
|
||||
type: 'VariableDeclarator',
|
||||
init: { type: 'FunctionExpression', params: [{}, {}, {}] },
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
function extract$1(node) {
|
||||
if (!matchesStructure(node, identifier$1)) {
|
||||
return null;
|
||||
}
|
||||
let block;
|
||||
if (
|
||||
node.type === 'ExpressionStatement' &&
|
||||
node.expression.type === 'AssignmentExpression' &&
|
||||
node.expression.right.type === 'FunctionExpression'
|
||||
) {
|
||||
block = node.expression.right.body;
|
||||
} else if (node.type === 'VariableDeclaration') {
|
||||
for (const decl of node.declarations) {
|
||||
if (
|
||||
decl.type === 'VariableDeclarator' &&
|
||||
_optionalChain$2([
|
||||
decl,
|
||||
'access',
|
||||
(_) => _.init,
|
||||
'optionalAccess',
|
||||
(_2) => _2.type,
|
||||
]) === 'FunctionExpression' &&
|
||||
_optionalChain$2([
|
||||
decl,
|
||||
'access',
|
||||
(_3) => _3.init,
|
||||
'optionalAccess',
|
||||
(_4) => _4.params,
|
||||
'access',
|
||||
(_5) => _5.length,
|
||||
]) === 3
|
||||
) {
|
||||
block = decl.init.body;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else if (node.type === 'FunctionDeclaration') {
|
||||
block = node.body;
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
const relevantExpression = _optionalChain$2([
|
||||
block,
|
||||
'optionalAccess',
|
||||
(_6) => _6.body,
|
||||
'access',
|
||||
(_7) => _7.at,
|
||||
'call',
|
||||
(_8) => _8(-2),
|
||||
]);
|
||||
if (!matchesStructure(relevantExpression, logicalExpression)) {
|
||||
return null;
|
||||
}
|
||||
if (
|
||||
_optionalChain$2([
|
||||
relevantExpression,
|
||||
'optionalAccess',
|
||||
(_9) => _9.type,
|
||||
]) !== 'ExpressionStatement' ||
|
||||
relevantExpression.expression.type !== 'LogicalExpression' ||
|
||||
relevantExpression.expression.right.type !== 'SequenceExpression' ||
|
||||
relevantExpression.expression.right.expressions[0].type !==
|
||||
'AssignmentExpression'
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const call = relevantExpression.expression.right.expressions[0].right;
|
||||
if (call.type !== 'CallExpression' || call.callee.type !== 'Identifier') {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
type: 'ArrowFunctionExpression',
|
||||
params: [{ type: 'Identifier', name: 'sig' }],
|
||||
body: {
|
||||
type: 'CallExpression',
|
||||
callee: { type: 'Identifier', name: call.callee.name },
|
||||
arguments:
|
||||
call.arguments.length === 1
|
||||
? [{ type: 'Identifier', name: 'sig' }]
|
||||
: [call.arguments[0], { type: 'Identifier', name: 'sig' }],
|
||||
optional: false,
|
||||
},
|
||||
async: false,
|
||||
expression: false,
|
||||
generator: false,
|
||||
};
|
||||
}
|
||||
function _optionalChain$1(ops) {
|
||||
let lastAccessLHS = undefined;
|
||||
let value = ops[0];
|
||||
let i = 1;
|
||||
while (i < ops.length) {
|
||||
const op = ops[i];
|
||||
const fn = ops[i + 1];
|
||||
i += 2;
|
||||
if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) {
|
||||
return undefined;
|
||||
}
|
||||
if (op === 'access' || op === 'optionalAccess') {
|
||||
lastAccessLHS = value;
|
||||
value = fn(value);
|
||||
} else if (op === 'call' || op === 'optionalCall') {
|
||||
value = fn((...args) => value.call(lastAccessLHS, ...args));
|
||||
lastAccessLHS = undefined;
|
||||
}
|
||||
}
|
||||
return value;
|
||||
}
|
||||
const identifier = {
|
||||
or: [
|
||||
{
|
||||
type: 'VariableDeclaration',
|
||||
kind: 'var',
|
||||
declarations: {
|
||||
anykey: [
|
||||
{
|
||||
type: 'VariableDeclarator',
|
||||
id: { type: 'Identifier' },
|
||||
init: {
|
||||
type: 'ArrayExpression',
|
||||
elements: [{ type: 'Identifier' }],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'ExpressionStatement',
|
||||
expression: {
|
||||
type: 'AssignmentExpression',
|
||||
left: { type: 'Identifier' },
|
||||
operator: '=',
|
||||
right: {
|
||||
type: 'ArrayExpression',
|
||||
elements: [{ type: 'Identifier' }],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const catchBlockBody = [
|
||||
{
|
||||
type: 'ReturnStatement',
|
||||
argument: {
|
||||
type: 'BinaryExpression',
|
||||
left: {
|
||||
type: 'MemberExpression',
|
||||
object: { type: 'Identifier' },
|
||||
computed: true,
|
||||
property: { type: 'Literal' },
|
||||
optional: false,
|
||||
},
|
||||
right: { type: 'Identifier' },
|
||||
operator: '+',
|
||||
},
|
||||
},
|
||||
];
|
||||
function extract(node) {
|
||||
if (!matchesStructure(node, identifier)) {
|
||||
let name = null;
|
||||
let block = null;
|
||||
switch (node.type) {
|
||||
case 'ExpressionStatement': {
|
||||
if (
|
||||
node.expression.type === 'AssignmentExpression' &&
|
||||
node.expression.left.type === 'Identifier' &&
|
||||
node.expression.right.type === 'FunctionExpression' &&
|
||||
node.expression.right.params.length === 1
|
||||
) {
|
||||
name = node.expression.left.name;
|
||||
block = node.expression.right.body;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'FunctionDeclaration': {
|
||||
if (node.params.length === 1) {
|
||||
name = _optionalChain$1([
|
||||
node,
|
||||
'access',
|
||||
(_) => _.id,
|
||||
'optionalAccess',
|
||||
(_2) => _2.name,
|
||||
]);
|
||||
block = node.body;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!block || !name) {
|
||||
return null;
|
||||
}
|
||||
const tryNode = block.body.at(-2);
|
||||
if (
|
||||
_optionalChain$1([tryNode, 'optionalAccess', (_3) => _3.type]) !==
|
||||
'TryStatement' ||
|
||||
_optionalChain$1([
|
||||
tryNode,
|
||||
'access',
|
||||
(_4) => _4.handler,
|
||||
'optionalAccess',
|
||||
(_5) => _5.type,
|
||||
]) !== 'CatchClause'
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const catchBody = tryNode.handler.body.body;
|
||||
if (matchesStructure(catchBody, catchBlockBody)) {
|
||||
return makeSolverFuncFromName(name);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
if (node.type === 'VariableDeclaration') {
|
||||
for (const declaration of node.declarations) {
|
||||
if (
|
||||
declaration.type !== 'VariableDeclarator' ||
|
||||
!declaration.init ||
|
||||
declaration.init.type !== 'ArrayExpression' ||
|
||||
declaration.init.elements.length !== 1
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const [firstElement] = declaration.init.elements;
|
||||
if (firstElement && firstElement.type === 'Identifier') {
|
||||
return makeSolverFuncFromName(firstElement.name);
|
||||
}
|
||||
}
|
||||
} else if (node.type === 'ExpressionStatement') {
|
||||
const expr = node.expression;
|
||||
if (
|
||||
expr.type === 'AssignmentExpression' &&
|
||||
expr.left.type === 'Identifier' &&
|
||||
expr.operator === '=' &&
|
||||
expr.right.type === 'ArrayExpression' &&
|
||||
expr.right.elements.length === 1
|
||||
) {
|
||||
const [firstElement] = expr.right.elements;
|
||||
if (firstElement && firstElement.type === 'Identifier') {
|
||||
return makeSolverFuncFromName(firstElement.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
function makeSolverFuncFromName(name) {
|
||||
return {
|
||||
type: 'ArrowFunctionExpression',
|
||||
params: [{ type: 'Identifier', name: 'n' }],
|
||||
body: {
|
||||
type: 'CallExpression',
|
||||
callee: { type: 'Identifier', name: name },
|
||||
arguments: [{ type: 'Identifier', name: 'n' }],
|
||||
optional: false,
|
||||
},
|
||||
async: false,
|
||||
expression: false,
|
||||
generator: false,
|
||||
};
|
||||
}
|
||||
const setupNodes = meriyah.parse(
|
||||
`\nif (typeof globalThis.XMLHttpRequest === "undefined") {\n globalThis.XMLHttpRequest = { prototype: {} };\n}\nconst window = Object.create(null);\nif (typeof URL === "undefined") {\n window.location = {\n hash: "",\n host: "www.youtube.com",\n hostname: "www.youtube.com",\n href: "https://www.youtube.com/watch?v=yt-dlp-wins",\n origin: "https://www.youtube.com",\n password: "",\n pathname: "/watch",\n port: "",\n protocol: "https:",\n search: "?v=yt-dlp-wins",\n username: "",\n };\n} else {\n window.location = new URL("https://www.youtube.com/watch?v=yt-dlp-wins");\n}\nif (typeof globalThis.document === "undefined") {\n globalThis.document = Object.create(null);\n}\nif (typeof globalThis.navigator === "undefined") {\n globalThis.navigator = Object.create(null);\n}\nif (typeof globalThis.self === "undefined") {\n globalThis.self = globalThis;\n}\n`,
|
||||
).body;
|
||||
function _optionalChain(ops) {
|
||||
let lastAccessLHS = undefined;
|
||||
let value = ops[0];
|
||||
let i = 1;
|
||||
while (i < ops.length) {
|
||||
const op = ops[i];
|
||||
const fn = ops[i + 1];
|
||||
i += 2;
|
||||
if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) {
|
||||
return undefined;
|
||||
}
|
||||
if (op === 'access' || op === 'optionalAccess') {
|
||||
lastAccessLHS = value;
|
||||
value = fn(value);
|
||||
} else if (op === 'call' || op === 'optionalCall') {
|
||||
value = fn((...args) => value.call(lastAccessLHS, ...args));
|
||||
lastAccessLHS = undefined;
|
||||
}
|
||||
}
|
||||
return value;
|
||||
}
|
||||
function preprocessPlayer(data) {
|
||||
const ast = meriyah.parse(data);
|
||||
const body = ast.body;
|
||||
const block = (() => {
|
||||
switch (body.length) {
|
||||
case 1: {
|
||||
const func = body[0];
|
||||
if (
|
||||
_optionalChain([func, 'optionalAccess', (_) => _.type]) ===
|
||||
'ExpressionStatement' &&
|
||||
func.expression.type === 'CallExpression' &&
|
||||
func.expression.callee.type === 'MemberExpression' &&
|
||||
func.expression.callee.object.type === 'FunctionExpression'
|
||||
) {
|
||||
return func.expression.callee.object.body;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
const func = body[1];
|
||||
if (
|
||||
_optionalChain([func, 'optionalAccess', (_2) => _2.type]) ===
|
||||
'ExpressionStatement' &&
|
||||
func.expression.type === 'CallExpression' &&
|
||||
func.expression.callee.type === 'FunctionExpression'
|
||||
) {
|
||||
const block = func.expression.callee.body;
|
||||
block.body.splice(0, 1);
|
||||
return block;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
throw 'unexpected structure';
|
||||
})();
|
||||
const found = { n: [], sig: [] };
|
||||
const plainExpressions = block.body.filter((node) => {
|
||||
const n = extract(node);
|
||||
if (n) {
|
||||
found.n.push(n);
|
||||
}
|
||||
const sig = extract$1(node);
|
||||
if (sig) {
|
||||
found.sig.push(sig);
|
||||
}
|
||||
if (node.type === 'ExpressionStatement') {
|
||||
if (node.expression.type === 'AssignmentExpression') {
|
||||
return true;
|
||||
}
|
||||
return node.expression.type === 'Literal';
|
||||
}
|
||||
return true;
|
||||
});
|
||||
block.body = plainExpressions;
|
||||
for (const [name, options] of Object.entries(found)) {
|
||||
const unique = new Set(options.map((x) => JSON.stringify(x)));
|
||||
if (unique.size !== 1) {
|
||||
const message = `found ${unique.size} ${name} function possibilities`;
|
||||
throw (
|
||||
message +
|
||||
(unique.size
|
||||
? `: ${options.map((x) => astring.generate(x)).join(', ')}`
|
||||
: '')
|
||||
);
|
||||
}
|
||||
plainExpressions.push({
|
||||
type: 'ExpressionStatement',
|
||||
expression: {
|
||||
type: 'AssignmentExpression',
|
||||
operator: '=',
|
||||
left: {
|
||||
type: 'MemberExpression',
|
||||
computed: false,
|
||||
object: { type: 'Identifier', name: '_result' },
|
||||
property: { type: 'Identifier', name: name },
|
||||
},
|
||||
right: options[0],
|
||||
},
|
||||
});
|
||||
}
|
||||
ast.body.splice(0, 0, ...setupNodes);
|
||||
return astring.generate(ast);
|
||||
}
|
||||
function getFromPrepared(code) {
|
||||
const resultObj = { n: null, sig: null };
|
||||
Function('_result', code)(resultObj);
|
||||
return resultObj;
|
||||
}
|
||||
function main(input) {
|
||||
const preprocessedPlayer =
|
||||
input.type === 'player'
|
||||
? preprocessPlayer(input.player)
|
||||
: input.preprocessed_player;
|
||||
const solvers = getFromPrepared(preprocessedPlayer);
|
||||
const responses = input.requests.map((input) => {
|
||||
if (!isOneOf(input.type, 'n', 'sig')) {
|
||||
return { type: 'error', error: `Unknown request type: ${input.type}` };
|
||||
}
|
||||
const solver = solvers[input.type];
|
||||
if (!solver) {
|
||||
return {
|
||||
type: 'error',
|
||||
error: `Failed to extract ${input.type} function`,
|
||||
};
|
||||
}
|
||||
try {
|
||||
return {
|
||||
type: 'result',
|
||||
data: Object.fromEntries(
|
||||
input.challenges.map((challenge) => [challenge, solver(challenge)]),
|
||||
),
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
type: 'error',
|
||||
error:
|
||||
error instanceof Error
|
||||
? `${error.message}\n${error.stack}`
|
||||
: `${error}`,
|
||||
};
|
||||
}
|
||||
});
|
||||
const output = { type: 'result', responses: responses };
|
||||
if (input.type === 'player' && input.output_preprocessed) {
|
||||
output.preprocessed_player = preprocessedPlayer;
|
||||
}
|
||||
return output;
|
||||
}
|
||||
return main;
|
||||
})(meriyah, astring);
|
||||
9
yt_dlp/extractor/youtube/jsc/_builtin/vendor/yt.solver.deno.lib.js
vendored
Normal file
9
yt_dlp/extractor/youtube/jsc/_builtin/vendor/yt.solver.deno.lib.js
vendored
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
/*!
|
||||
* SPDX-License-Identifier: Unlicense
|
||||
* This file was automatically generated by https://github.com/yt-dlp/ejs
|
||||
*/
|
||||
const lib = {
|
||||
meriyah: await import('npm:meriyah@6.1.4'),
|
||||
astring: await import('npm:astring@1.9.0'),
|
||||
};
|
||||
export { lib };
|
||||
Loading…
Add table
Add a link
Reference in a new issue