feat(network): request-throttle controls to avoid YouTube rate-limiting
Re-implements the sleepRequests/sleepInterval/maxSleepInterval feature (--sleep-requests / --sleep-interval / --max-sleep-interval) from the stale feat/binary-management-and-library-scale branch, ported onto current main's structure (zod settingsSchema, decomposed NetworkCard). All default to 0/off, preserving current behaviour. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -102,6 +102,12 @@ export interface AccessOptions {
|
|||||||
proxy: string
|
proxy: string
|
||||||
/** yt-dlp --limit-rate value, e.g. '2M'; empty means unlimited */
|
/** yt-dlp --limit-rate value, e.g. '2M'; empty means unlimited */
|
||||||
rateLimit: string
|
rateLimit: string
|
||||||
|
/** `--sleep-requests` seconds (pause between extraction requests); 0/omitted = off */
|
||||||
|
sleepRequests?: number
|
||||||
|
/** `--sleep-interval` seconds (min pause before each download); 0/omitted = off */
|
||||||
|
sleepInterval?: number
|
||||||
|
/** `--max-sleep-interval` seconds (max of the random pre-download pause); applies only with a non-zero sleepInterval */
|
||||||
|
maxSleepInterval?: number
|
||||||
/** absolute path to aria2c.exe; omit to use yt-dlp's built-in downloader */
|
/** absolute path to aria2c.exe; omit to use yt-dlp's built-in downloader */
|
||||||
aria2cPath?: string
|
aria2cPath?: string
|
||||||
/** aria2c parallel connections (-x/-s), clamped to 1–16; omitted = 16 (L64) */
|
/** aria2c parallel connections (-x/-s), clamped to 1–16; omitted = 16 (L64) */
|
||||||
@@ -323,6 +329,19 @@ function accessArgs(a: AccessOptions): string[] {
|
|||||||
const args: string[] = []
|
const args: string[] = []
|
||||||
if (a.proxy.trim()) args.push('--proxy', a.proxy.trim())
|
if (a.proxy.trim()) args.push('--proxy', a.proxy.trim())
|
||||||
if (a.rateLimit.trim()) args.push('--limit-rate', a.rateLimit.trim())
|
if (a.rateLimit.trim()) args.push('--limit-rate', a.rateLimit.trim())
|
||||||
|
// Request throttle — the antidote to YouTube's rate-limit/403 wall on big
|
||||||
|
// channel pulls. `--sleep-requests` spaces out the extraction requests; the
|
||||||
|
// sleep-interval pair brackets a random pause before each download. yt-dlp
|
||||||
|
// requires --sleep-interval to be present for --max-sleep-interval to be
|
||||||
|
// meaningful, and a max below the min is nonsensical, so max is gated on both.
|
||||||
|
const sleepRequests = a.sleepRequests ?? 0
|
||||||
|
if (sleepRequests > 0) args.push('--sleep-requests', String(sleepRequests))
|
||||||
|
const sleepInterval = a.sleepInterval ?? 0
|
||||||
|
if (sleepInterval > 0) {
|
||||||
|
args.push('--sleep-interval', String(sleepInterval))
|
||||||
|
const maxSleep = a.maxSleepInterval ?? 0
|
||||||
|
if (maxSleep > sleepInterval) args.push('--max-sleep-interval', String(maxSleep))
|
||||||
|
}
|
||||||
if (a.aria2cPath) {
|
if (a.aria2cPath) {
|
||||||
args.push('--downloader', a.aria2cPath, '--downloader-args', aria2cArgs(a.aria2cConnections))
|
args.push('--downloader', a.aria2cPath, '--downloader-args', aria2cArgs(a.aria2cConnections))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -222,6 +222,9 @@ export function buildCommand(opts: StartDownloadOptions, cookiesFile?: string):
|
|||||||
const access = {
|
const access = {
|
||||||
proxy: settings.proxy,
|
proxy: settings.proxy,
|
||||||
rateLimit: settings.rateLimit,
|
rateLimit: settings.rateLimit,
|
||||||
|
sleepRequests: settings.sleepRequests,
|
||||||
|
sleepInterval: settings.sleepInterval,
|
||||||
|
maxSleepInterval: settings.maxSleepInterval,
|
||||||
aria2cPath,
|
aria2cPath,
|
||||||
aria2cConnections: settings.aria2cConnections,
|
aria2cConnections: settings.aria2cConnections,
|
||||||
// L136/M6: incognito attaches no cookies from the browser either.
|
// L136/M6: incognito attaches no cookies from the browser either.
|
||||||
|
|||||||
@@ -94,6 +94,10 @@ export const SETTINGS_FIELD_SCHEMAS: { [K in keyof Settings]: z.ZodType<Settings
|
|||||||
defaultVideoQuality: z.enum(VIDEO_QUALITY_OPTIONS),
|
defaultVideoQuality: z.enum(VIDEO_QUALITY_OPTIONS),
|
||||||
defaultAudioQuality: z.enum(AUDIO_QUALITY_OPTIONS),
|
defaultAudioQuality: z.enum(AUDIO_QUALITY_OPTIONS),
|
||||||
rateLimit: trimmedWhere((v) => RATE_LIMIT_RE.test(v)),
|
rateLimit: trimmedWhere((v) => RATE_LIMIT_RE.test(v)),
|
||||||
|
// Throttle seconds (0–300, 0 = off) to dodge YouTube's rate-limit/403 wall.
|
||||||
|
sleepRequests: clampedInt(0, 300),
|
||||||
|
sleepInterval: clampedInt(0, 300),
|
||||||
|
maxSleepInterval: clampedInt(0, 300),
|
||||||
// Power-user field; yt-dlp's client list evolves. Accept any trimmed string.
|
// Power-user field; yt-dlp's client list evolves. Accept any trimmed string.
|
||||||
youtubePlayerClient: z.string().transform((v) => v.trim()),
|
youtubePlayerClient: z.string().transform((v) => v.trim()),
|
||||||
// Credential-bearing fields — settings.ts encrypts these AFTER parsing.
|
// Credential-bearing fields — settings.ts encrypts these AFTER parsing.
|
||||||
|
|||||||
@@ -17,6 +17,9 @@ export function NetworkCard(): React.JSX.Element {
|
|||||||
const styles = useSettingsStyles()
|
const styles = useSettingsStyles()
|
||||||
const proxy = useSettings((s) => s.proxy)
|
const proxy = useSettings((s) => s.proxy)
|
||||||
const rateLimit = useSettings((s) => s.rateLimit)
|
const rateLimit = useSettings((s) => s.rateLimit)
|
||||||
|
const sleepRequests = useSettings((s) => s.sleepRequests)
|
||||||
|
const sleepInterval = useSettings((s) => s.sleepInterval)
|
||||||
|
const maxSleepInterval = useSettings((s) => s.maxSleepInterval)
|
||||||
const useAria2c = useSettings((s) => s.useAria2c)
|
const useAria2c = useSettings((s) => s.useAria2c)
|
||||||
const aria2cConnections = useSettings((s) => s.aria2cConnections)
|
const aria2cConnections = useSettings((s) => s.aria2cConnections)
|
||||||
const youtubePlayerClient = useSettings((s) => s.youtubePlayerClient)
|
const youtubePlayerClient = useSettings((s) => s.youtubePlayerClient)
|
||||||
@@ -56,6 +59,58 @@ export function NetworkCard(): React.JSX.Element {
|
|||||||
/>
|
/>
|
||||||
</Field>
|
</Field>
|
||||||
|
|
||||||
|
<Field
|
||||||
|
label="Pause between requests"
|
||||||
|
hint="Seconds yt-dlp waits between requests (--sleep-requests). The key setting for downloading a whole channel without YouTube rate-limiting you (the "unavailable, try again later" / 403 errors). Try 2-5s for large channels. 0 = off."
|
||||||
|
>
|
||||||
|
<SpinButton
|
||||||
|
value={sleepRequests}
|
||||||
|
min={0}
|
||||||
|
max={300}
|
||||||
|
onChange={(_, d) => {
|
||||||
|
const v = d.value ?? Number(d.displayValue)
|
||||||
|
if (typeof v === 'number' && Number.isFinite(v)) {
|
||||||
|
update({ sleepRequests: Math.min(300, Math.max(0, Math.round(v))) })
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</Field>
|
||||||
|
|
||||||
|
<Field
|
||||||
|
label="Pause before each download (min)"
|
||||||
|
hint="Minimum seconds to wait before starting each download (--sleep-interval). Combined with the max below, yt-dlp waits a random amount in that range. 0 = off."
|
||||||
|
>
|
||||||
|
<SpinButton
|
||||||
|
value={sleepInterval}
|
||||||
|
min={0}
|
||||||
|
max={300}
|
||||||
|
onChange={(_, d) => {
|
||||||
|
const v = d.value ?? Number(d.displayValue)
|
||||||
|
if (typeof v === 'number' && Number.isFinite(v)) {
|
||||||
|
update({ sleepInterval: Math.min(300, Math.max(0, Math.round(v))) })
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</Field>
|
||||||
|
|
||||||
|
<Field
|
||||||
|
label="Pause before each download (max)"
|
||||||
|
hint="Maximum seconds for the random pre-download pause (--max-sleep-interval). Only used when it's set above the minimum above; otherwise the minimum is used as a fixed pause."
|
||||||
|
>
|
||||||
|
<SpinButton
|
||||||
|
value={maxSleepInterval}
|
||||||
|
min={0}
|
||||||
|
max={300}
|
||||||
|
disabled={sleepInterval <= 0}
|
||||||
|
onChange={(_, d) => {
|
||||||
|
const v = d.value ?? Number(d.displayValue)
|
||||||
|
if (typeof v === 'number' && Number.isFinite(v)) {
|
||||||
|
update({ maxSleepInterval: Math.min(300, Math.max(0, Math.round(v))) })
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</Field>
|
||||||
|
|
||||||
<Field
|
<Field
|
||||||
label="Use aria2c downloader"
|
label="Use aria2c downloader"
|
||||||
hint="Multi-connection downloads for faster speeds on supported sites. Falls back to the standard downloader if the accelerator isn't available."
|
hint="Multi-connection downloads for faster speeds on supported sites. Falls back to the standard downloader if the accelerator isn't available."
|
||||||
|
|||||||
@@ -615,6 +615,20 @@ export interface Settings {
|
|||||||
proxy: string
|
proxy: string
|
||||||
/** yt-dlp --limit-rate value, e.g. '2M' or '500K'. Empty means unlimited. */
|
/** yt-dlp --limit-rate value, e.g. '2M' or '500K'. Empty means unlimited. */
|
||||||
rateLimit: string
|
rateLimit: string
|
||||||
|
/**
|
||||||
|
* Request throttle (seconds), to avoid YouTube's "rate-limited for up to an
|
||||||
|
* hour" / HTTP 403 wall when pulling a whole channel. All default to 0 = off
|
||||||
|
* (yt-dlp's as-fast-as-possible behaviour):
|
||||||
|
* - sleepRequests: `--sleep-requests`, a pause between the metadata/extraction
|
||||||
|
* HTTP requests — the single most effective knob for large channel pulls.
|
||||||
|
* - sleepInterval / maxSleepInterval: `--sleep-interval` (min) and
|
||||||
|
* `--max-sleep-interval` (max) bracket a random pause before each download.
|
||||||
|
* max only applies when it's set above a non-zero min (yt-dlp requires the
|
||||||
|
* min to be present, and a max below it is meaningless).
|
||||||
|
*/
|
||||||
|
sleepRequests: number
|
||||||
|
sleepInterval: number
|
||||||
|
maxSleepInterval: number
|
||||||
/** use bundled aria2c (resources/bin/aria2c.exe) as the external downloader */
|
/** use bundled aria2c (resources/bin/aria2c.exe) as the external downloader */
|
||||||
useAria2c: boolean
|
useAria2c: boolean
|
||||||
/** aria2c parallel connections per download (-x/-s), 1–16; only used when useAria2c is on (L64) */
|
/** aria2c parallel connections per download (-x/-s), 1–16; only used when useAria2c is on (L64) */
|
||||||
@@ -709,6 +723,9 @@ export const DEFAULT_SETTINGS: Settings = {
|
|||||||
downloadOptions: DEFAULT_DOWNLOAD_OPTIONS,
|
downloadOptions: DEFAULT_DOWNLOAD_OPTIONS,
|
||||||
proxy: '',
|
proxy: '',
|
||||||
rateLimit: '',
|
rateLimit: '',
|
||||||
|
sleepRequests: 0,
|
||||||
|
sleepInterval: 0,
|
||||||
|
maxSleepInterval: 0,
|
||||||
useAria2c: false,
|
useAria2c: false,
|
||||||
aria2cConnections: 16,
|
aria2cConnections: 16,
|
||||||
cookieSource: 'none',
|
cookieSource: 'none',
|
||||||
|
|||||||
@@ -127,6 +127,46 @@ describe('network options', () => {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// --- Request throttle: sleep-requests / sleep-interval ----------------------
|
||||||
|
|
||||||
|
describe('throttle options', () => {
|
||||||
|
it('emits no sleep flags when all throttle values are 0/unset', () => {
|
||||||
|
const argv = build(opts(), dlo(), NO_ACCESS)
|
||||||
|
expect(argv).not.toContain('--sleep-requests')
|
||||||
|
expect(argv).not.toContain('--sleep-interval')
|
||||||
|
expect(argv).not.toContain('--max-sleep-interval')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('emits --sleep-requests when sleepRequests > 0', () => {
|
||||||
|
const argv = build(opts(), dlo(), { ...NO_ACCESS, sleepRequests: 3 })
|
||||||
|
expect(hasSeq(argv, '--sleep-requests', '3')).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('emits --sleep-interval when sleepInterval > 0', () => {
|
||||||
|
const argv = build(opts(), dlo(), { ...NO_ACCESS, sleepInterval: 5 })
|
||||||
|
expect(hasSeq(argv, '--sleep-interval', '5')).toBe(true)
|
||||||
|
expect(argv).not.toContain('--max-sleep-interval')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('emits --max-sleep-interval only when it exceeds a non-zero sleepInterval', () => {
|
||||||
|
const argv = build(opts(), dlo(), { ...NO_ACCESS, sleepInterval: 5, maxSleepInterval: 10 })
|
||||||
|
expect(hasSeq(argv, '--sleep-interval', '5')).toBe(true)
|
||||||
|
expect(hasSeq(argv, '--max-sleep-interval', '10')).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('drops --max-sleep-interval when sleepInterval is 0 (yt-dlp needs the min)', () => {
|
||||||
|
const argv = build(opts(), dlo(), { ...NO_ACCESS, sleepInterval: 0, maxSleepInterval: 10 })
|
||||||
|
expect(argv).not.toContain('--sleep-interval')
|
||||||
|
expect(argv).not.toContain('--max-sleep-interval')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('drops --max-sleep-interval when it is not above the min', () => {
|
||||||
|
const argv = build(opts(), dlo(), { ...NO_ACCESS, sleepInterval: 8, maxSleepInterval: 5 })
|
||||||
|
expect(hasSeq(argv, '--sleep-interval', '8')).toBe(true)
|
||||||
|
expect(argv).not.toContain('--max-sleep-interval')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
// --- Cookies: browser store vs. sign-in window's exported file --------------
|
// --- Cookies: browser store vs. sign-in window's exported file --------------
|
||||||
|
|
||||||
describe('cookies', () => {
|
describe('cookies', () => {
|
||||||
|
|||||||
@@ -106,6 +106,14 @@ describe('parseSettingsField (CC9 schema parity)', () => {
|
|||||||
expect(parseSettingsField('rateLimit', 'fast')).toEqual(fail)
|
expect(parseSettingsField('rateLimit', 'fast')).toEqual(fail)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('sleepRequests/sleepInterval/maxSleepInterval: coerce, round, clamp to [0,300]', () => {
|
||||||
|
expect(parseSettingsField('sleepRequests', 5)).toEqual(ok(5))
|
||||||
|
expect(parseSettingsField('sleepRequests', -3)).toEqual(ok(0))
|
||||||
|
expect(parseSettingsField('sleepRequests', 999)).toEqual(ok(300))
|
||||||
|
expect(parseSettingsField('sleepInterval', '4')).toEqual(ok(4))
|
||||||
|
expect(parseSettingsField('maxSleepInterval', 7.6)).toEqual(ok(8))
|
||||||
|
})
|
||||||
|
|
||||||
it('quality presets: allowlisted', () => {
|
it('quality presets: allowlisted', () => {
|
||||||
expect(parseSettingsField('defaultVideoQuality', '720p')).toEqual(ok('720p'))
|
expect(parseSettingsField('defaultVideoQuality', '720p')).toEqual(ok('720p'))
|
||||||
expect(parseSettingsField('defaultVideoQuality', '4K')).toEqual(fail)
|
expect(parseSettingsField('defaultVideoQuality', '4K')).toEqual(fail)
|
||||||
|
|||||||
Reference in New Issue
Block a user