feat(audit): Batch 25 — project reorg (CC12), leading DI args (CC7), CC10 by-design
CC12: renderer views/ (5 screens + Onboarding + settings cards) + lib/ (theme/thumb/datetime/id/qualityOptions/useClipboardLink/queueStats); main core/ (buildArgs/validation/indexerCore/ytdlpPolicy). git mv preserved history; all src+test imports updated. Build emits view chunks from views/, 344 tests + typecheck green, live probe rendered all screens. CC7: wc/onProgress is the leading arg everywhere — downloadAppUpdate(wc,url), indexSource(onProgress,url,signal), indexSourceCancelable(onProgress,url). CC10: closed by-design — jsonStore backs all records; settings stay in electron-store for DPAPI secret encryption (a deliberate two-store split). Also closes the UI24/W4 context-menu cross-reference. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,486 @@
|
||||
/**
|
||||
* Pure yt-dlp argv construction for a download.
|
||||
*
|
||||
* This module is deliberately free of any electron / node-runtime dependency
|
||||
* (it only imports types + the BEST_FORMAT_ID const from the shared contract)
|
||||
* so the argv it produces can be unit-tested without spinning up Electron.
|
||||
* `buildArgs` takes the ffmpeg/yt-dlp `binDir` as a parameter rather than
|
||||
* resolving it itself; the caller in download.ts passes `getBinDir()`.
|
||||
*/
|
||||
|
||||
import { join } from 'path'
|
||||
import {
|
||||
BEST_FORMAT_ID,
|
||||
type StartDownloadOptions,
|
||||
type DownloadOptions,
|
||||
type CollectionContext,
|
||||
type CookieBrowser,
|
||||
type CommandTemplate
|
||||
} from '@shared/ipc'
|
||||
|
||||
// --- Quality → yt-dlp selector mapping --------------------------------------
|
||||
|
||||
function videoFormat(quality: string): string {
|
||||
switch (quality) {
|
||||
case '1080p':
|
||||
return 'bv*[height<=1080]+ba/b[height<=1080]'
|
||||
case '720p':
|
||||
return 'bv*[height<=720]+ba/b[height<=720]'
|
||||
case '480p':
|
||||
return 'bv*[height<=480]+ba/b[height<=480]'
|
||||
case '360p':
|
||||
return 'bv*[height<=360]+ba/b[height<=360]'
|
||||
default:
|
||||
return 'bv*+ba/b' // Best available
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The video format selector. A specific probed format_id wins; otherwise fall
|
||||
* back to the height-based preset. Video-only formats get +bestaudio so the
|
||||
* merge produces a file with sound (with a video-only fallback if no audio).
|
||||
*/
|
||||
function videoSelector(opts: StartDownloadOptions): string {
|
||||
if (opts.formatId && opts.formatId !== BEST_FORMAT_ID) {
|
||||
return opts.formatHasAudio ? opts.formatId : `${opts.formatId}+bestaudio/${opts.formatId}`
|
||||
}
|
||||
return videoFormat(opts.quality)
|
||||
}
|
||||
|
||||
function audioQuality(quality: string): string {
|
||||
switch (quality) {
|
||||
case '320 kbps':
|
||||
return '320K'
|
||||
case '192 kbps':
|
||||
return '192K'
|
||||
case '128 kbps':
|
||||
return '128K'
|
||||
case 'Best':
|
||||
return '0'
|
||||
default:
|
||||
// Unrecognised label — fall back to best (0) so the download still works.
|
||||
return '0'
|
||||
}
|
||||
}
|
||||
|
||||
// Stdout line markers, shared by the emit side (the --print templates here) and
|
||||
// the parse side (download.ts). Defined once so changing a marker can't silently
|
||||
// break the parser that splits on it (CL1).
|
||||
export const PROGRESS_MARKER = 'prog|'
|
||||
export const FILEPATH_MARKER = 'path|'
|
||||
// Emitted once at the before_dl stage carrying the resolved FINAL output path, so
|
||||
// the engine knows the destination stem while the download is still in flight —
|
||||
// used to delete orphaned .part / .fNNN intermediates when a download is canceled
|
||||
// (R4). `%(filepath)s` is still unresolved this early (prints 'NA'); `%(filename)s`
|
||||
// already resolves to the full output path.
|
||||
export const DEST_MARKER = 'dest|'
|
||||
|
||||
// The progress line yt-dlp emits (one per --newline tick). Note the leading
|
||||
// `download:` is the progress-template TYPE selector and is consumed by yt-dlp
|
||||
// (it does NOT appear in the output). The PROGRESS_MARKER that follows is our
|
||||
// own marker, so we can tell progress lines apart from the after-move filepath
|
||||
// print on the same stdout stream.
|
||||
const PROGRESS_TEMPLATE =
|
||||
`download:${PROGRESS_MARKER}%(progress.status)s|%(progress.downloaded_bytes)s|` +
|
||||
'%(progress.total_bytes)s|%(progress.total_bytes_estimate)s|' +
|
||||
'%(progress.speed)s|%(progress.eta)s'
|
||||
|
||||
// Crop embedded audio artwork to a centred square. The double-quote/single-quote
|
||||
// dance is deliberate: yt-dlp shlex-splits the `--ppa` value, consuming the outer
|
||||
// double quotes and leaving ffmpeg single-quoted crop expressions whose commas are
|
||||
// protected from ffmpeg's filtergraph separator. Side = min(width, height).
|
||||
export const CROP_SQUARE_PPA =
|
||||
'EmbedThumbnail+ffmpeg_o:-c:v mjpeg -vf ' + "crop=\"'if(gt(ih,iw),iw,ih)':'if(gt(iw,ih),ih,iw)'\""
|
||||
|
||||
/**
|
||||
* Phase B "Access & networking" settings — not per-download options, always
|
||||
* applied from the global settings (there's no per-download override for
|
||||
* these, unlike DownloadOptions).
|
||||
*/
|
||||
export interface AccessOptions {
|
||||
/** yt-dlp --proxy value; empty disables it */
|
||||
proxy: string
|
||||
/** yt-dlp --limit-rate value, e.g. '2M'; empty means unlimited */
|
||||
rateLimit: string
|
||||
/** absolute path to aria2c.exe; omit to use yt-dlp's built-in downloader */
|
||||
aria2cPath?: string
|
||||
/** aria2c parallel connections (-x/-s), clamped to 1–16; omitted = 16 (L64) */
|
||||
aria2cConnections?: number
|
||||
/** read auth cookies from an installed browser's own cookie store */
|
||||
cookiesFromBrowser?: CookieBrowser
|
||||
/** absolute path to a Netscape-format cookie file (the sign-in window's export) */
|
||||
cookiesFile?: string
|
||||
/** sanitize output filenames to a portable ASCII-only subset */
|
||||
restrictFilenames: boolean
|
||||
/** absolute path to a --download-archive file; omit to disable archive tracking */
|
||||
downloadArchivePath?: string
|
||||
/** YouTube extraction client override (e.g. 'web_safari'); empty/omitted = default */
|
||||
youtubePlayerClient?: string
|
||||
/** manually-supplied YouTube Proof-of-Origin token; empty/omitted = none */
|
||||
youtubePoToken?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* The yt-dlp `--downloader-args` value for aria2c: N connections split across N
|
||||
* segments, each at least 1MB so tiny files don't get split pointlessly. The
|
||||
* connection count is user-tunable (Settings → Network, L64) and clamped here to
|
||||
* aria2c's own -x/-s ceiling of 16 — settings validation clamps too, but this is
|
||||
* pure-module defence for any other caller. Non-finite input → the default 16.
|
||||
*/
|
||||
export function aria2cArgs(connections?: number): string {
|
||||
const n =
|
||||
connections !== undefined && Number.isFinite(connections)
|
||||
? Math.min(16, Math.max(1, Math.round(connections)))
|
||||
: 16
|
||||
return `aria2c:-x ${n} -s ${n} -k 1M`
|
||||
}
|
||||
|
||||
/** The default tuning (16 connections) — kept for tests/docs that reference it. */
|
||||
export const ARIA2C_ARGS = aria2cArgs()
|
||||
|
||||
/**
|
||||
* Shell-like split of a raw "extra yt-dlp args" string (Phase C custom-command
|
||||
* templates) into argv tokens. Supports single- and double-quoted spans so a
|
||||
* flag value containing spaces (e.g. a --ppa recipe) can be typed as one token.
|
||||
*
|
||||
* LIMITATIONS (audit M2): there is no escape-sequence support (`\"`, `\'`) and
|
||||
* quotes cannot nest — a literal quote inside a value can't be expressed, and an
|
||||
* unterminated quote falls through to the `\S+` branch and captures the quote
|
||||
* char itself. For the yt-dlp use cases this targets (proxy URLs, filename
|
||||
* patterns, ffmpeg recipes), that's sufficient. If a future template needs
|
||||
* nested or escaped quotes, redesign this to a proper shlex or JSON-based format.
|
||||
*/
|
||||
export function parseExtraArgs(raw: string): string[] {
|
||||
const args: string[] = []
|
||||
const re = /"([^"]*)"|'([^']*)'|(\S+)/g
|
||||
let m: RegExpExecArray | null
|
||||
while ((m = re.exec(raw)) !== null) {
|
||||
// Exactly one of the three alternation groups matches; the '' fallback only
|
||||
// satisfies the type checker (noUncheckedIndexedAccess) and never fires.
|
||||
args.push(m[1] ?? m[2] ?? m[3] ?? '')
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the extra yt-dlp args for a download, enforcing the custom-command
|
||||
* consent gate (audit F2).
|
||||
*
|
||||
* Extra args are powerful enough to run arbitrary code (e.g. `--exec`), so they
|
||||
* are honoured ONLY when custom commands are explicitly enabled in settings —
|
||||
* the same persisted flag the Settings UI and backup-import treat as consent. A
|
||||
* per-download override wins over the persisted default template, but NEITHER is
|
||||
* applied while the gate is off. This keeps a compromised renderer from smuggling
|
||||
* code-exec flags through a lone `startDownload({ extraArgs })` call: it would
|
||||
* first have to flip the visible `enableCustomCommands` setting, leaving a trace
|
||||
* (the same defence-in-depth posture as the main-side maxConcurrent cap).
|
||||
*
|
||||
* - enableCustomCommands off → [] (always)
|
||||
* - perDownloadExtraArgs defined (even '') → those args
|
||||
* - else a template whose urlPattern matches → that template's args (most specific)
|
||||
* - else a matching defaultTemplateId → that template's args
|
||||
* - else → []
|
||||
*/
|
||||
export function selectExtraArgs(params: {
|
||||
enableCustomCommands: boolean
|
||||
perDownloadExtraArgs: string | undefined
|
||||
defaultTemplateId: string | null
|
||||
templates: Pick<CommandTemplate, 'id' | 'args' | 'urlPattern'>[]
|
||||
/** the download URL, for urlPattern auto-matching */
|
||||
url?: string
|
||||
}): string[] {
|
||||
if (!params.enableCustomCommands) return []
|
||||
if (params.perDownloadExtraArgs !== undefined) return parseExtraArgs(params.perDownloadExtraArgs)
|
||||
// A template whose urlPattern matches this URL auto-applies, ahead of the global
|
||||
// default — it's the more specific choice.
|
||||
if (params.url) {
|
||||
const matched = params.templates.find(
|
||||
(t) => t.urlPattern && matchesUrl(t.urlPattern, params.url!)
|
||||
)
|
||||
if (matched) return parseExtraArgs(matched.args)
|
||||
}
|
||||
if (params.defaultTemplateId) {
|
||||
const tpl = params.templates.find((t) => t.id === params.defaultTemplateId)
|
||||
if (tpl) return parseExtraArgs(tpl.args)
|
||||
}
|
||||
return []
|
||||
}
|
||||
|
||||
/** Case-insensitive regex test of a template's urlPattern; a bad pattern never matches. */
|
||||
export function matchesUrl(pattern: string, url: string): boolean {
|
||||
try {
|
||||
return new RegExp(pattern, 'i').test(url)
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a raw trim string (one or more time ranges, comma- or newline-separated)
|
||||
* into yt-dlp `--download-sections` specs. Each range is normalised to the
|
||||
* `*START-END` time-range form yt-dlp expects (the leading `*` distinguishes a
|
||||
* timestamp range from a chapter-title regex). Tokens that don't look like a
|
||||
* numeric time range are dropped, so malformed input can't reach yt-dlp's argv.
|
||||
*
|
||||
* Accepts H:MM:SS / M:SS / SS with optional fractional seconds, e.g. '1:30-2:00',
|
||||
* '90-120', '0:00:10.5-0:00:20'. A token may already carry the leading `*`.
|
||||
*/
|
||||
export function parseTrimSections(raw: string | undefined): string[] {
|
||||
if (!raw) return []
|
||||
// A time is SS, M:SS, or H:MM:SS — at most two colon-separated groups. The old
|
||||
// `*` quantifier accepted nonsense like `1:2:3:4`, which then failed inside
|
||||
// yt-dlp's --download-sections rather than being rejected up front (L146).
|
||||
const TIME = String.raw`\d+(?::\d{1,2}){0,2}(?:\.\d+)?`
|
||||
const RANGE = new RegExp(`^${TIME}-${TIME}$`)
|
||||
const out: string[] = []
|
||||
for (const piece of raw.split(/[,\n]/)) {
|
||||
let t = piece.trim()
|
||||
if (!t) continue
|
||||
if (t.startsWith('*')) t = t.slice(1).trim()
|
||||
if (RANGE.test(t)) out.push(`*${t}`)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Wrap a single argv token for human-readable display only (Phase C command
|
||||
// preview) — never used to build the real argv that gets spawned.
|
||||
function quoteForDisplay(arg: string): string {
|
||||
if (arg === '') return '""'
|
||||
if (/[\s"]/.test(arg)) return `"${arg.replace(/"/g, '\\"')}"`
|
||||
return arg
|
||||
}
|
||||
|
||||
/** Render a binary + argv as a single copy-pasteable command line. */
|
||||
export function formatCommandLine(exe: string, args: string[]): string {
|
||||
return [exe, ...args].map(quoteForDisplay).join(' ')
|
||||
}
|
||||
|
||||
// --- Collection (media-manager) folder paths --------------------------------
|
||||
|
||||
/**
|
||||
* Sanitize one path segment (a channel or playlist name) into a Windows-safe
|
||||
* directory name. This matters because, unlike the filename yt-dlp itself writes
|
||||
* (which `--restrict-filenames` can clean), these directory segments are built by
|
||||
* AeroFetch from untrusted channel/playlist titles and joined onto the output
|
||||
* dir — so they must be neutered for both illegal characters AND path traversal.
|
||||
*
|
||||
* - illegal chars (`< > : " / \ | ? *`) and control chars → space
|
||||
* - leading/trailing dots and spaces stripped (illegal / invisible on Windows),
|
||||
* which also turns a bare `..` traversal segment into nothing
|
||||
* - reserved device names (CON, PRN, NUL, COM1…) get an underscore prefix
|
||||
* - length-capped so the full path stays well under MAX_PATH
|
||||
* - empty result falls back to 'Untitled'
|
||||
*/
|
||||
export function sanitizeDirSegment(name: string): string {
|
||||
// C0 control chars (charCode < 0x20) and Windows-illegal chars both become a
|
||||
// space. The control-char filter is done by char code so no literal control
|
||||
// byte ever appears in this source file.
|
||||
let s = Array.from(name ?? '', (ch) => (ch.charCodeAt(0) < 0x20 ? ' ' : ch)).join('')
|
||||
s = s.replace(/[<>:"/\\|?*]/g, ' ')
|
||||
s = s
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
.replace(/[. ]+$/, '')
|
||||
.replace(/^[. ]+/, '')
|
||||
// Reserved device names are reserved even WITH an extension ('CON.txt' is still
|
||||
// the CON device), so match an optional trailing '.<ext>' too (audit T3).
|
||||
if (/^(con|prn|aux|nul|com[1-9]|lpt[1-9])(\..*)?$/i.test(s)) s = `_${s}`
|
||||
s = s.slice(0, 80).trim()
|
||||
return s || 'Untitled'
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the `-o` output template for a collection download:
|
||||
* <outDir>/<Channel>/<Playlist>/<NNN> - <baseFilename>
|
||||
* where NNN is the 1-based playlist index zero-padded to three digits and
|
||||
* baseFilename keeps its yt-dlp field tokens (e.g. '%(title)s.%(ext)s') so they
|
||||
* still expand. Channel/playlist are sanitized via sanitizeDirSegment.
|
||||
*/
|
||||
export function collectionOutputTemplate(
|
||||
outDir: string,
|
||||
c: CollectionContext,
|
||||
baseFilename: string
|
||||
): string {
|
||||
const n = Number.isFinite(c.index) && c.index > 0 ? Math.floor(c.index) : 1
|
||||
const nnn = String(n).padStart(3, '0')
|
||||
const segments = [sanitizeDirSegment(c.channel), sanitizeDirSegment(c.playlist)]
|
||||
return join(outDir, ...segments, `${nnn} - ${baseFilename}`)
|
||||
}
|
||||
|
||||
function accessArgs(a: AccessOptions): string[] {
|
||||
const args: string[] = []
|
||||
if (a.proxy.trim()) args.push('--proxy', a.proxy.trim())
|
||||
if (a.rateLimit.trim()) args.push('--limit-rate', a.rateLimit.trim())
|
||||
if (a.aria2cPath) {
|
||||
args.push('--downloader', a.aria2cPath, '--downloader-args', aria2cArgs(a.aria2cConnections))
|
||||
}
|
||||
// cookiesFromBrowser and cookiesFile are mutually exclusive sources — the UI
|
||||
// only ever sets one (driven by Settings.cookieSource), but browser wins if
|
||||
// a caller somehow sets both.
|
||||
if (a.cookiesFromBrowser) args.push('--cookies-from-browser', a.cookiesFromBrowser)
|
||||
else if (a.cookiesFile) args.push('--cookies', a.cookiesFile)
|
||||
if (a.restrictFilenames) args.push('--restrict-filenames')
|
||||
if (a.downloadArchivePath) args.push('--download-archive', a.downloadArchivePath)
|
||||
// YouTube reliability: combine the player-client + PO-token overrides into one
|
||||
// `youtube:` extractor-args group (semicolon-separated, as yt-dlp expects).
|
||||
const yt: string[] = []
|
||||
if (a.youtubePlayerClient?.trim()) yt.push(`player_client=${a.youtubePlayerClient.trim()}`)
|
||||
if (a.youtubePoToken?.trim()) yt.push(`po_token=${a.youtubePoToken.trim()}`)
|
||||
if (yt.length > 0) args.push('--extractor-args', `youtube:${yt.join(';')}`)
|
||||
return args
|
||||
}
|
||||
|
||||
/** Subtitle / SponsorBlock / chapter / metadata flags shared by both kinds. */
|
||||
function postProcessArgs(opts: StartDownloadOptions, o: DownloadOptions): string[] {
|
||||
const args: string[] = []
|
||||
|
||||
// Subtitles — video only; audio extraction has nowhere to embed them.
|
||||
if (opts.kind === 'video' && o.embedSubtitles) {
|
||||
args.push('--write-subs', '--embed-subs', '--sub-langs', o.subtitleLanguages || 'en')
|
||||
if (o.autoSubtitles) args.push('--write-auto-subs')
|
||||
// Normalise to srt first so embedding works across containers (mp4 → mov_text).
|
||||
args.push('--convert-subs', 'srt')
|
||||
}
|
||||
|
||||
// SponsorBlock.
|
||||
let forcedKeyframes = false
|
||||
if (o.sponsorBlock && o.sponsorBlockCategories.length > 0) {
|
||||
const cats = o.sponsorBlockCategories.join(',')
|
||||
if (o.sponsorBlockMode === 'remove') {
|
||||
// Force keyframes at the cut points so segment boundaries are frame-accurate.
|
||||
args.push('--sponsorblock-remove', cats, '--force-keyframes-at-cuts')
|
||||
forcedKeyframes = true
|
||||
} else {
|
||||
args.push('--sponsorblock-mark', cats)
|
||||
}
|
||||
}
|
||||
|
||||
// Trim — download only the given time ranges (works for video + audio). Force
|
||||
// keyframes at the cut points for frame-accurate boundaries, unless
|
||||
// SponsorBlock-remove already requested it (the flag is idempotent, but we
|
||||
// avoid emitting it twice).
|
||||
const sections = parseTrimSections(opts.trim)
|
||||
for (const sec of sections) args.push('--download-sections', sec)
|
||||
if (sections.length > 0 && !forcedKeyframes) args.push('--force-keyframes-at-cuts')
|
||||
|
||||
if (o.embedChapters) args.push('--embed-chapters')
|
||||
// Split into one file per chapter. yt-dlp also keeps the full file; the
|
||||
// per-chapter files use yt-dlp's default `chapter:` output template.
|
||||
if (o.splitChapters) args.push('--split-chapters')
|
||||
|
||||
// Metadata embedding + optional per-field overrides.
|
||||
// --replace-in-metadata FIELD REGEX REPLACE is used instead of --parse-metadata
|
||||
// FROM:TO because the replacement arg is a separate element (no colon-splitting
|
||||
// on values like "Foo: Bar"). ^.*$ matches any string including empty. The only
|
||||
// escaping Python's re.sub needs in the replacement string is backslash.
|
||||
const metaOverrides: [string, string | undefined][] = [
|
||||
['title', o.metadataTitle],
|
||||
['artist', o.metadataArtist],
|
||||
['album', o.metadataAlbum]
|
||||
]
|
||||
const hasOverrides = metaOverrides.some(([, v]) => v?.trim())
|
||||
if (o.embedMetadata || hasOverrides) args.push('--embed-metadata')
|
||||
const escRepl = (s: string): string => s.replace(/\\/g, '\\\\')
|
||||
for (const [field, val] of metaOverrides) {
|
||||
if (val?.trim()) args.push('--replace-in-metadata', field, '^.*$', escRepl(val.trim()))
|
||||
}
|
||||
|
||||
// Media-server sidecar files (Phase K) — written next to the output file so
|
||||
// Jellyfin/Plex/Kodi can ingest metadata, poster art, and the description.
|
||||
if (o.writeInfoJson) args.push('--write-info-json')
|
||||
if (o.writeThumbnailFile) args.push('--write-thumbnail')
|
||||
if (o.writeDescription) args.push('--write-description')
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
/**
|
||||
* Everything buildArgs needs to construct a yt-dlp argv. A single options object
|
||||
* (rather than six positional params) so callers can't transpose `opts`/`options`
|
||||
* or the two path-like strings, and new inputs can be added without churning every
|
||||
* call site (CL2).
|
||||
*/
|
||||
export interface BuildArgsInput {
|
||||
/** The per-download request (url, kind, quality, chosen format, trim, …). */
|
||||
opts: StartDownloadOptions
|
||||
/** The resolved `-o` output template (flat filename or collection folder tree). */
|
||||
outputTemplate: string
|
||||
/** The post-processing options group (per-download override or the persisted default). */
|
||||
options: DownloadOptions
|
||||
/** ffmpeg/yt-dlp bin dir, passed in so this module stays free of path resolution. */
|
||||
binDir: string
|
||||
/** Global access/networking settings (proxy, cookies, rate limit, …). */
|
||||
access: AccessOptions
|
||||
/** Custom-command extra args, already consent-gated by the caller. */
|
||||
extraArgs?: string[]
|
||||
}
|
||||
|
||||
export function buildArgs(input: BuildArgsInput): string[] {
|
||||
const { opts, outputTemplate, options: o, binDir, access, extraArgs = [] } = input
|
||||
const args = [
|
||||
'--newline',
|
||||
'--no-color',
|
||||
'--no-playlist',
|
||||
// Bound a dead/hung connection so yt-dlp aborts (and retries, then exits) a
|
||||
// stalled socket instead of hanging forever and pinning a concurrency slot
|
||||
// with no recovery. The app-side idle watchdog in download.ts is the backstop
|
||||
// for a fully-wedged process. (B1)
|
||||
'--socket-timeout',
|
||||
'30',
|
||||
// --print (below) implies --quiet, which would suppress progress; --progress
|
||||
// forces the progress template to emit anyway.
|
||||
'--progress',
|
||||
'--ffmpeg-location',
|
||||
binDir,
|
||||
'-o',
|
||||
outputTemplate,
|
||||
'--progress-template',
|
||||
PROGRESS_TEMPLATE,
|
||||
// Print the final path after post-processing/move so we can open it later.
|
||||
'--print',
|
||||
`after_move:${FILEPATH_MARKER}%(filepath)s`,
|
||||
// Print the resolved destination up front (before_dl) so a cancel can find and
|
||||
// delete this download's orphaned partials by stem, mid-flight (R4).
|
||||
'--print',
|
||||
`before_dl:${DEST_MARKER}%(filename)s`,
|
||||
'--no-simulate'
|
||||
]
|
||||
|
||||
args.push(...accessArgs(access))
|
||||
args.push(...postProcessArgs(opts, o))
|
||||
|
||||
if (opts.kind === 'audio') {
|
||||
// --audio-quality is a bitrate selector meaningful only for lossy re-encodes.
|
||||
// For lossless formats (flac/wav) it is silently ignored by yt-dlp (M21).
|
||||
const lossless = o.audioFormat === 'flac' || o.audioFormat === 'wav'
|
||||
args.push('-x', '--audio-format', o.audioFormat)
|
||||
if (!lossless) args.push('--audio-quality', audioQuality(opts.quality))
|
||||
if (o.embedThumbnail) {
|
||||
args.push('--embed-thumbnail')
|
||||
if (o.cropThumbnail) args.push('--ppa', CROP_SQUARE_PPA)
|
||||
}
|
||||
} else {
|
||||
args.push('-f', videoSelector(opts), '--merge-output-format', o.videoContainer)
|
||||
// A raw format-sort string (advanced) wins outright; otherwise the codec
|
||||
// preference is a sort tiebreaker AFTER resolution/fps, nudging the pick toward
|
||||
// the chosen codec without overriding the requested quality.
|
||||
const sort = o.formatSort.trim()
|
||||
if (sort) {
|
||||
args.push('-S', sort)
|
||||
} else if (o.preferredVideoCodec !== 'any') {
|
||||
const token = o.preferredVideoCodec === 'av1' ? 'av01' : o.preferredVideoCodec
|
||||
args.push('-S', `res,fps,vcodec:${token}`)
|
||||
}
|
||||
// mp4/mkv carry a cover image; webm has no reliable way to embed one.
|
||||
if (o.embedThumbnail && o.videoContainer !== 'webm') args.push('--embed-thumbnail')
|
||||
}
|
||||
|
||||
// Custom-command extra args go last, immediately before the `--` URL
|
||||
// terminator, so a user-supplied flag can override anything chosen above
|
||||
// (yt-dlp takes the last occurrence of most options).
|
||||
if (extraArgs.length > 0) args.push(...extraArgs)
|
||||
|
||||
// `--` terminates option parsing so the URL can never be read as a flag.
|
||||
args.push('--', opts.url)
|
||||
return args
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
/**
|
||||
* Pure, dependency-free core for Source indexing (Pinchflat-style media manager;
|
||||
* see ROADMAP-PINCHFLAT.md). Like buildArgs.ts / validation.ts this module imports
|
||||
* nothing from electron or the node runtime, so its URL-classification and
|
||||
* item-merge logic can be unit-tested without spinning up Electron.
|
||||
*
|
||||
* The impure orchestration (spawning yt-dlp, persisting to disk) lives in
|
||||
* indexer.ts, which composes these helpers.
|
||||
*/
|
||||
|
||||
import type { MediaItem, SourceKind } from '@shared/ipc'
|
||||
|
||||
// --- yt-dlp --flat-playlist entry shape (shared with probe.ts) --------------
|
||||
|
||||
/** The slice of a flat-playlist entry we read (a video, or a nested playlist). */
|
||||
export interface RawEntry {
|
||||
id?: string
|
||||
title?: string
|
||||
url?: string
|
||||
webpage_url?: string
|
||||
duration?: number
|
||||
uploader?: string
|
||||
channel?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the canonical URL for a flat-playlist entry. Prefer an explicit http(s)
|
||||
* URL; otherwise build a YouTube watch URL from the id (the common case where flat
|
||||
* entries carry only an id). Returns null when nothing usable is present.
|
||||
*
|
||||
* Note: the returned value is always either an http(s) URL or null, so it can
|
||||
* never begin with '-' and be mis-read as a yt-dlp option (callers also pass `--`).
|
||||
*/
|
||||
export function entryUrl(e: RawEntry): string | null {
|
||||
const cand = e.url || e.webpage_url
|
||||
if (cand && /^https?:\/\//i.test(cand)) return cand
|
||||
// e.id is untrusted JSON from yt-dlp, so percent-encode it rather than splicing
|
||||
// it raw into the query string (audit T2). A normal 11-char id is unaffected.
|
||||
if (e.id) return `https://www.youtube.com/watch?v=${encodeURIComponent(e.id)}`
|
||||
return null
|
||||
}
|
||||
|
||||
/** Seconds → 'M:SS' or 'H:MM:SS'. Flat entries give duration as a number. */
|
||||
export function fmtDuration(sec?: number): string | undefined {
|
||||
if (sec == null || !Number.isFinite(sec)) return undefined
|
||||
const s = Math.max(0, Math.round(sec))
|
||||
const h = Math.floor(s / 3600)
|
||||
const m = Math.floor((s % 3600) / 60)
|
||||
const r = s % 60
|
||||
const mm = h ? String(m).padStart(2, '0') : String(m)
|
||||
return `${h ? `${h}:` : ''}${mm}:${String(r).padStart(2, '0')}`
|
||||
}
|
||||
|
||||
// --- Source-URL classification ----------------------------------------------
|
||||
|
||||
/** A classified Source URL: the kind, plus a normalised base for tab probing. */
|
||||
export interface SourceClass {
|
||||
kind: SourceKind
|
||||
/**
|
||||
* For 'channel': the channel root (e.g. https://www.youtube.com/@handle) onto
|
||||
* which '/videos' and '/playlists' tabs are appended. For 'playlist': the
|
||||
* canonical playlist URL.
|
||||
*/
|
||||
base: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify a pasted URL as a YouTube channel, a playlist, or neither.
|
||||
*
|
||||
* Channel forms: /@handle, /channel/<id>, /c/<name>, /user/<name> — any trailing
|
||||
* tab (/videos, /playlists, /streams, /shorts, /featured) is stripped to the root.
|
||||
* Playlist form: any URL carrying a `list=` query param. Returns null for a lone
|
||||
* video or a non-YouTube URL (Phase F scopes the channel walk to YouTube, whose
|
||||
* /videos + /playlists tab structure this relies on).
|
||||
*/
|
||||
export function classifySource(raw: string): SourceClass | null {
|
||||
let u: URL
|
||||
try {
|
||||
u = new URL((raw ?? '').trim())
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
if (u.protocol !== 'http:' && u.protocol !== 'https:') return null
|
||||
|
||||
const host = u.hostname.replace(/^www\./i, '').toLowerCase()
|
||||
const isYouTube = host === 'youtube.com' || host.endsWith('.youtube.com') || host === 'youtu.be'
|
||||
|
||||
// A playlist is identified purely by its list= param (works on any youtube host).
|
||||
const list = u.searchParams.get('list')
|
||||
if (isYouTube && list) {
|
||||
return {
|
||||
kind: 'playlist',
|
||||
base: `https://www.youtube.com/playlist?list=${encodeURIComponent(list)}`
|
||||
}
|
||||
}
|
||||
|
||||
if (!isYouTube) return null
|
||||
|
||||
// Channel roots. The handle/id segment is captured; later path segments
|
||||
// (the tab) are discarded so we always probe from the channel root.
|
||||
const m = u.pathname.match(/^\/(@[^/]+|channel\/[^/]+|c\/[^/]+|user\/[^/]+)/i)
|
||||
if (m) return { kind: 'channel', base: `https://www.youtube.com/${m[1]}` }
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Channel tab titles from yt-dlp often look like "<Name> - Videos" or
|
||||
* "<Name> - Playlists". Strip a trailing known-tab suffix to recover the bare
|
||||
* channel name. Leaves anything else untouched.
|
||||
*/
|
||||
export function stripTabSuffix(title: string | undefined): string | undefined {
|
||||
if (!title) return title
|
||||
return title.replace(/\s[-–—]\s(Videos|Playlists|Shorts|Live|Streams|Home|Featured)$/i, '').trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Deterministic 32-bit FNV-1a hash → 8 hex chars. Used to derive a stable Source
|
||||
* id from its normalised base URL so re-indexing the same channel updates the
|
||||
* existing record instead of creating a duplicate. Pure (no crypto import).
|
||||
*/
|
||||
export function stableSourceId(base: string): string {
|
||||
let h = 0x811c9dc5
|
||||
for (let i = 0; i < base.length; i++) {
|
||||
h ^= base.charCodeAt(i)
|
||||
h = Math.imul(h, 0x01000193)
|
||||
}
|
||||
return (h >>> 0).toString(16).padStart(8, '0')
|
||||
}
|
||||
|
||||
// --- RSS fast-check (Phase J: watched sources) ------------------------------
|
||||
|
||||
/**
|
||||
* Build a YouTube RSS feed URL for cheap "anything new?" polling of a watched
|
||||
* source. Channels feed by `channel_id` (UC…), playlists by `playlist_id` (PL…).
|
||||
* Returns undefined when the id is missing (the sync then falls back to a full
|
||||
* re-index). The feed only carries the latest ~15 uploads, so it's a freshness
|
||||
* check, not a substitute for the initial full index.
|
||||
*/
|
||||
export function buildFeedUrl(kind: SourceKind, ytId: string | undefined): string | undefined {
|
||||
if (!ytId) return undefined
|
||||
const param = kind === 'channel' ? 'channel_id' : 'playlist_id'
|
||||
return `https://www.youtube.com/feeds/videos.xml?${param}=${encodeURIComponent(ytId)}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Guard for the watched-source RSS pre-check (audit T7). The only feed AeroFetch
|
||||
* ever builds is a YouTube videos.xml feed (see buildFeedUrl), so the sync refuses
|
||||
* to fetch anything else — this keeps a hand-edited / corrupted sources.json from
|
||||
* pointing fetch() at an internal service, a cloud-metadata endpoint, or any other
|
||||
* arbitrary host (SSRF). Requires https, the youtube.com host (www optional), and
|
||||
* the exact feed path; the channel_id/playlist_id query is free.
|
||||
*/
|
||||
export function isYouTubeFeedUrl(url: string): boolean {
|
||||
try {
|
||||
const u = new URL(url)
|
||||
const host = u.hostname.replace(/^www\./i, '').toLowerCase()
|
||||
return u.protocol === 'https:' && host === 'youtube.com' && u.pathname === '/feeds/videos.xml'
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/** Extract the video ids from a YouTube RSS/Atom feed body (the <yt:videoId> tags). */
|
||||
export function parseRssVideoIds(xml: string): string[] {
|
||||
const ids: string[] = []
|
||||
const re = /<yt:videoId>\s*([\w-]+)\s*<\/yt:videoId>/g
|
||||
let m: RegExpExecArray | null
|
||||
while ((m = re.exec(xml)) !== null) if (m[1]) ids.push(m[1])
|
||||
return ids
|
||||
}
|
||||
|
||||
// --- Entry merge / dedup ----------------------------------------------------
|
||||
|
||||
/** A named playlist and its (flat) video entries, ready to merge. */
|
||||
export interface NamedPlaylist {
|
||||
title: string
|
||||
entries: RawEntry[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge a Source's playlists (and its catch-all uploads) into a deduped list of
|
||||
* MediaItem records. Dedup is by video id: the FIRST playlist a video appears in
|
||||
* wins its folder assignment, so videos grouped into a real playlist land there,
|
||||
* and only videos in no playlist fall through to the synthetic 'Uploads' folder.
|
||||
*
|
||||
* `playlistIndex` is the 1-based position WITHIN the winning playlist (not the
|
||||
* global upload order), so the on-disk "NNN - Title" numbering matches the
|
||||
* playlist the file is filed under.
|
||||
*/
|
||||
export function buildMediaItems(
|
||||
sourceId: string,
|
||||
playlists: NamedPlaylist[],
|
||||
uploads: RawEntry[]
|
||||
): MediaItem[] {
|
||||
const byVideo = new Map<string, MediaItem>()
|
||||
|
||||
const add = (e: RawEntry, playlistTitle: string, index: number): void => {
|
||||
const videoId = e.id
|
||||
if (!videoId) return
|
||||
if (byVideo.has(videoId)) return // first playlist wins
|
||||
const url = entryUrl(e)
|
||||
if (!url) return // not turnable into a downloadable URL
|
||||
byVideo.set(videoId, {
|
||||
id: `${sourceId}:${videoId}`,
|
||||
sourceId,
|
||||
videoId,
|
||||
title: e.title || `Video ${videoId}`,
|
||||
url,
|
||||
playlistTitle,
|
||||
playlistIndex: index,
|
||||
durationLabel: fmtDuration(e.duration),
|
||||
downloaded: false
|
||||
})
|
||||
}
|
||||
|
||||
for (const p of playlists) p.entries.forEach((e, i) => add(e, p.title, i + 1))
|
||||
uploads.forEach((e, i) => add(e, 'Uploads', i + 1))
|
||||
|
||||
return [...byVideo.values()]
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge a freshly-enumerated item list with the previously-persisted one for the
|
||||
* same source (incremental re-index; see ROADMAP-PINCHFLAT.md Phase I). The fresh
|
||||
* list defines the current membership/ordering, but any video already marked
|
||||
* downloaded keeps its `downloaded`/`downloadedAt`/`filePath` so a re-sync never
|
||||
* loses that state or re-downloads it. Videos that vanished from the source (now
|
||||
* private/deleted) are dropped. `newCount` is how many fresh videos weren't in
|
||||
* the previous set — the "X new since last sync" figure.
|
||||
*/
|
||||
export function mergeItemsPreservingState(
|
||||
existing: MediaItem[],
|
||||
fresh: MediaItem[]
|
||||
): { items: MediaItem[]; newCount: number } {
|
||||
const prevByVideo = new Map(existing.map((m) => [m.videoId, m]))
|
||||
let newCount = 0
|
||||
const items = fresh.map((f) => {
|
||||
const prev = prevByVideo.get(f.videoId)
|
||||
if (!prev) {
|
||||
newCount++
|
||||
return f
|
||||
}
|
||||
return prev.downloaded
|
||||
? { ...f, downloaded: true, downloadedAt: prev.downloadedAt, filePath: prev.filePath }
|
||||
: f
|
||||
})
|
||||
return { items, newCount }
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
/**
|
||||
* Pure, dependency-free validators for untrusted input that crosses a trust
|
||||
* boundary (renderer writes, backup files, hand-edited JSON on disk).
|
||||
*
|
||||
* Like buildArgs.ts, this module imports nothing from electron/node beyond the
|
||||
* `path` builtin, so it can be unit-tested without spinning up Electron.
|
||||
*/
|
||||
|
||||
import { isAbsolute } from 'path'
|
||||
import type { HistoryEntry, ErrorLogEntry, CommandTemplate, Source, MediaItem } from '@shared/ipc'
|
||||
|
||||
// --- Path-traversal sanitization (audit S4) ---------------------------------
|
||||
|
||||
/**
|
||||
* A filenameTemplate is joined onto the output directory and handed to yt-dlp's
|
||||
* `-o`. Reject anything that could write outside that directory so a malicious
|
||||
* backup/settings write can't traverse out of the chosen folder (e.g.
|
||||
* '%(title)s\..\..\win32.exe'). Legitimate templates still contain yt-dlp
|
||||
* `%(field)s` tokens and `/` or `\` for sub-folders, which are fine.
|
||||
*
|
||||
* Two Windows-specific bypasses are guarded beyond the obvious cases (audit T1):
|
||||
* - drive-relative prefixes like 'C:foo' — `path.isAbsolute` returns FALSE for
|
||||
* these, yet they escape the output dir, so a leading drive letter is rejected.
|
||||
* - a '..' segment dressed up with trailing dots/spaces ('.. ', '.. .') —
|
||||
* Windows silently trims those, so an exact `=== '..'` check would miss them.
|
||||
*/
|
||||
const TRAVERSAL_SEGMENT = /^[. ]*\.\.[. ]*$/
|
||||
export function isSafeFilenameTemplate(template: string): boolean {
|
||||
if (isAbsolute(template) || /^[a-zA-Z]:/.test(template)) return false
|
||||
return !template.split(/[\\/]/).some((segment) => TRAVERSAL_SEGMENT.test(segment))
|
||||
}
|
||||
|
||||
/** An output directory must be an absolute path ('' resolves to OS Downloads). */
|
||||
export function isSafeOutputDir(dir: string): boolean {
|
||||
return dir === '' || isAbsolute(dir)
|
||||
}
|
||||
|
||||
// --- Persisted-JSON entry validation (audit S5) -----------------------------
|
||||
|
||||
const isOptionalString = (v: unknown): boolean => v === undefined || typeof v === 'string'
|
||||
|
||||
/** A persisted history.json row must have the right shape or it's dropped on read. */
|
||||
export function isValidHistoryEntry(o: unknown): o is HistoryEntry {
|
||||
if (!o || typeof o !== 'object') return false
|
||||
const e = o as Record<string, unknown>
|
||||
return (
|
||||
typeof e.id === 'string' &&
|
||||
typeof e.title === 'string' &&
|
||||
typeof e.url === 'string' &&
|
||||
(e.kind === 'video' || e.kind === 'audio') &&
|
||||
typeof e.quality === 'string' &&
|
||||
typeof e.completedAt === 'number' &&
|
||||
isOptionalString(e.filePath) &&
|
||||
isOptionalString(e.channel) &&
|
||||
isOptionalString(e.sizeLabel) &&
|
||||
isOptionalString(e.thumbnail)
|
||||
)
|
||||
}
|
||||
|
||||
/** A persisted errorlog.json row must have the right shape or it's dropped on read. */
|
||||
export function isValidErrorLogEntry(o: unknown): o is ErrorLogEntry {
|
||||
if (!o || typeof o !== 'object') return false
|
||||
const e = o as Record<string, unknown>
|
||||
return (
|
||||
typeof e.id === 'string' &&
|
||||
typeof e.url === 'string' &&
|
||||
(e.kind === 'video' || e.kind === 'audio') &&
|
||||
typeof e.error === 'string' &&
|
||||
typeof e.occurredAt === 'number' &&
|
||||
isOptionalString(e.title)
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* A persisted templates.json row must at least be an object carrying an id we
|
||||
* can stringify; name/args are then coerced by templates.ts's sanitize(), so a
|
||||
* weak shape check here plus normalisation there is enough.
|
||||
*/
|
||||
export function isTemplateLike(o: unknown): o is CommandTemplate {
|
||||
if (!o || typeof o !== 'object') return false
|
||||
const t = o as Record<string, unknown>
|
||||
return typeof t.id === 'string' || typeof t.id === 'number'
|
||||
}
|
||||
|
||||
/** A persisted sources.json row must have the right shape or it's dropped on read. */
|
||||
export function isValidSource(o: unknown): o is Source {
|
||||
if (!o || typeof o !== 'object') return false
|
||||
const s = o as Record<string, unknown>
|
||||
return (
|
||||
typeof s.id === 'string' &&
|
||||
typeof s.url === 'string' &&
|
||||
(s.kind === 'channel' || s.kind === 'playlist') &&
|
||||
typeof s.title === 'string' &&
|
||||
typeof s.addedAt === 'number' &&
|
||||
typeof s.itemCount === 'number' &&
|
||||
isOptionalString(s.channel) &&
|
||||
// feedUrl drives a network fetch in the sync, so validate its shape here too
|
||||
// (audit T7); the host is additionally restricted at the fetch boundary.
|
||||
isOptionalString(s.feedUrl) &&
|
||||
(s.watched === undefined || typeof s.watched === 'boolean') &&
|
||||
(s.lastIndexedAt === undefined || typeof s.lastIndexedAt === 'number')
|
||||
)
|
||||
}
|
||||
|
||||
/** A persisted media-items.json row must have the right shape or it's dropped on read. */
|
||||
export function isValidMediaItem(o: unknown): o is MediaItem {
|
||||
if (!o || typeof o !== 'object') return false
|
||||
const m = o as Record<string, unknown>
|
||||
return (
|
||||
typeof m.id === 'string' &&
|
||||
typeof m.sourceId === 'string' &&
|
||||
typeof m.videoId === 'string' &&
|
||||
typeof m.title === 'string' &&
|
||||
typeof m.url === 'string' &&
|
||||
typeof m.playlistTitle === 'string' &&
|
||||
typeof m.playlistIndex === 'number' &&
|
||||
typeof m.downloaded === 'boolean' &&
|
||||
isOptionalString(m.durationLabel) &&
|
||||
isOptionalString(m.filePath) &&
|
||||
(m.downloadedAt === undefined || typeof m.downloadedAt === 'number')
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
/**
|
||||
* Pure auto-update scheduling policy for yt-dlp. Kept free of any electron /
|
||||
* node-runtime dependency (like buildArgs.ts) so the throttle can be unit-tested
|
||||
* without spinning up Electron. ytdlp.ts imports these to drive the real check.
|
||||
*/
|
||||
|
||||
/** Once-a-day throttle for the startup auto-update check. */
|
||||
export const AUTO_UPDATE_INTERVAL_MS = 24 * 60 * 60 * 1000
|
||||
|
||||
/**
|
||||
* Whether a background yt-dlp update check should run now: only when auto-update
|
||||
* is enabled AND at least one interval has elapsed since the last check. A
|
||||
* zero/absent lastCheck means "never checked" → run. `now` and the interval are
|
||||
* passed in so the decision is a pure function of its inputs.
|
||||
*/
|
||||
export function shouldAutoCheckYtdlp(
|
||||
enabled: boolean,
|
||||
lastCheck: number,
|
||||
now: number,
|
||||
intervalMs: number = AUTO_UPDATE_INTERVAL_MS
|
||||
): boolean {
|
||||
if (!enabled) return false
|
||||
if (!lastCheck) return true
|
||||
// A backward system-clock correction can leave `lastCheck` in the "future"
|
||||
// relative to `now`, making `now - lastCheck` negative so the daily check would
|
||||
// never come due again. Treat a future timestamp as skewed/bogus and run the
|
||||
// check rather than waiting out a negative interval (R9).
|
||||
if (lastCheck > now) return true
|
||||
return now - lastCheck >= intervalMs
|
||||
}
|
||||
Reference in New Issue
Block a user