feat(network): request-throttle controls to avoid YouTube rate-limiting
Re-implements the sleepRequests/sleepInterval/maxSleepInterval feature (--sleep-requests / --sleep-interval / --max-sleep-interval) from the stale feat/binary-management-and-library-scale branch, ported onto current main's structure (zod settingsSchema, decomposed NetworkCard). All default to 0/off, preserving current behaviour. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -102,6 +102,12 @@ export interface AccessOptions {
|
||||
proxy: string
|
||||
/** yt-dlp --limit-rate value, e.g. '2M'; empty means unlimited */
|
||||
rateLimit: string
|
||||
/** `--sleep-requests` seconds (pause between extraction requests); 0/omitted = off */
|
||||
sleepRequests?: number
|
||||
/** `--sleep-interval` seconds (min pause before each download); 0/omitted = off */
|
||||
sleepInterval?: number
|
||||
/** `--max-sleep-interval` seconds (max of the random pre-download pause); applies only with a non-zero sleepInterval */
|
||||
maxSleepInterval?: number
|
||||
/** absolute path to aria2c.exe; omit to use yt-dlp's built-in downloader */
|
||||
aria2cPath?: string
|
||||
/** aria2c parallel connections (-x/-s), clamped to 1–16; omitted = 16 (L64) */
|
||||
@@ -323,6 +329,19 @@ function accessArgs(a: AccessOptions): string[] {
|
||||
const args: string[] = []
|
||||
if (a.proxy.trim()) args.push('--proxy', a.proxy.trim())
|
||||
if (a.rateLimit.trim()) args.push('--limit-rate', a.rateLimit.trim())
|
||||
// Request throttle — the antidote to YouTube's rate-limit/403 wall on big
|
||||
// channel pulls. `--sleep-requests` spaces out the extraction requests; the
|
||||
// sleep-interval pair brackets a random pause before each download. yt-dlp
|
||||
// requires --sleep-interval to be present for --max-sleep-interval to be
|
||||
// meaningful, and a max below the min is nonsensical, so max is gated on both.
|
||||
const sleepRequests = a.sleepRequests ?? 0
|
||||
if (sleepRequests > 0) args.push('--sleep-requests', String(sleepRequests))
|
||||
const sleepInterval = a.sleepInterval ?? 0
|
||||
if (sleepInterval > 0) {
|
||||
args.push('--sleep-interval', String(sleepInterval))
|
||||
const maxSleep = a.maxSleepInterval ?? 0
|
||||
if (maxSleep > sleepInterval) args.push('--max-sleep-interval', String(maxSleep))
|
||||
}
|
||||
if (a.aria2cPath) {
|
||||
args.push('--downloader', a.aria2cPath, '--downloader-args', aria2cArgs(a.aria2cConnections))
|
||||
}
|
||||
|
||||
@@ -222,6 +222,9 @@ export function buildCommand(opts: StartDownloadOptions, cookiesFile?: string):
|
||||
const access = {
|
||||
proxy: settings.proxy,
|
||||
rateLimit: settings.rateLimit,
|
||||
sleepRequests: settings.sleepRequests,
|
||||
sleepInterval: settings.sleepInterval,
|
||||
maxSleepInterval: settings.maxSleepInterval,
|
||||
aria2cPath,
|
||||
aria2cConnections: settings.aria2cConnections,
|
||||
// L136/M6: incognito attaches no cookies from the browser either.
|
||||
|
||||
@@ -94,6 +94,10 @@ export const SETTINGS_FIELD_SCHEMAS: { [K in keyof Settings]: z.ZodType<Settings
|
||||
defaultVideoQuality: z.enum(VIDEO_QUALITY_OPTIONS),
|
||||
defaultAudioQuality: z.enum(AUDIO_QUALITY_OPTIONS),
|
||||
rateLimit: trimmedWhere((v) => RATE_LIMIT_RE.test(v)),
|
||||
// Throttle seconds (0–300, 0 = off) to dodge YouTube's rate-limit/403 wall.
|
||||
sleepRequests: clampedInt(0, 300),
|
||||
sleepInterval: clampedInt(0, 300),
|
||||
maxSleepInterval: clampedInt(0, 300),
|
||||
// Power-user field; yt-dlp's client list evolves. Accept any trimmed string.
|
||||
youtubePlayerClient: z.string().transform((v) => v.trim()),
|
||||
// Credential-bearing fields — settings.ts encrypts these AFTER parsing.
|
||||
|
||||
@@ -17,6 +17,9 @@ export function NetworkCard(): React.JSX.Element {
|
||||
const styles = useSettingsStyles()
|
||||
const proxy = useSettings((s) => s.proxy)
|
||||
const rateLimit = useSettings((s) => s.rateLimit)
|
||||
const sleepRequests = useSettings((s) => s.sleepRequests)
|
||||
const sleepInterval = useSettings((s) => s.sleepInterval)
|
||||
const maxSleepInterval = useSettings((s) => s.maxSleepInterval)
|
||||
const useAria2c = useSettings((s) => s.useAria2c)
|
||||
const aria2cConnections = useSettings((s) => s.aria2cConnections)
|
||||
const youtubePlayerClient = useSettings((s) => s.youtubePlayerClient)
|
||||
@@ -56,6 +59,58 @@ export function NetworkCard(): React.JSX.Element {
|
||||
/>
|
||||
</Field>
|
||||
|
||||
<Field
|
||||
label="Pause between requests"
|
||||
hint="Seconds yt-dlp waits between requests (--sleep-requests). The key setting for downloading a whole channel without YouTube rate-limiting you (the "unavailable, try again later" / 403 errors). Try 2-5s for large channels. 0 = off."
|
||||
>
|
||||
<SpinButton
|
||||
value={sleepRequests}
|
||||
min={0}
|
||||
max={300}
|
||||
onChange={(_, d) => {
|
||||
const v = d.value ?? Number(d.displayValue)
|
||||
if (typeof v === 'number' && Number.isFinite(v)) {
|
||||
update({ sleepRequests: Math.min(300, Math.max(0, Math.round(v))) })
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</Field>
|
||||
|
||||
<Field
|
||||
label="Pause before each download (min)"
|
||||
hint="Minimum seconds to wait before starting each download (--sleep-interval). Combined with the max below, yt-dlp waits a random amount in that range. 0 = off."
|
||||
>
|
||||
<SpinButton
|
||||
value={sleepInterval}
|
||||
min={0}
|
||||
max={300}
|
||||
onChange={(_, d) => {
|
||||
const v = d.value ?? Number(d.displayValue)
|
||||
if (typeof v === 'number' && Number.isFinite(v)) {
|
||||
update({ sleepInterval: Math.min(300, Math.max(0, Math.round(v))) })
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</Field>
|
||||
|
||||
<Field
|
||||
label="Pause before each download (max)"
|
||||
hint="Maximum seconds for the random pre-download pause (--max-sleep-interval). Only used when it's set above the minimum above; otherwise the minimum is used as a fixed pause."
|
||||
>
|
||||
<SpinButton
|
||||
value={maxSleepInterval}
|
||||
min={0}
|
||||
max={300}
|
||||
disabled={sleepInterval <= 0}
|
||||
onChange={(_, d) => {
|
||||
const v = d.value ?? Number(d.displayValue)
|
||||
if (typeof v === 'number' && Number.isFinite(v)) {
|
||||
update({ maxSleepInterval: Math.min(300, Math.max(0, Math.round(v))) })
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</Field>
|
||||
|
||||
<Field
|
||||
label="Use aria2c downloader"
|
||||
hint="Multi-connection downloads for faster speeds on supported sites. Falls back to the standard downloader if the accelerator isn't available."
|
||||
|
||||
@@ -615,6 +615,20 @@ export interface Settings {
|
||||
proxy: string
|
||||
/** yt-dlp --limit-rate value, e.g. '2M' or '500K'. Empty means unlimited. */
|
||||
rateLimit: string
|
||||
/**
|
||||
* Request throttle (seconds), to avoid YouTube's "rate-limited for up to an
|
||||
* hour" / HTTP 403 wall when pulling a whole channel. All default to 0 = off
|
||||
* (yt-dlp's as-fast-as-possible behaviour):
|
||||
* - sleepRequests: `--sleep-requests`, a pause between the metadata/extraction
|
||||
* HTTP requests — the single most effective knob for large channel pulls.
|
||||
* - sleepInterval / maxSleepInterval: `--sleep-interval` (min) and
|
||||
* `--max-sleep-interval` (max) bracket a random pause before each download.
|
||||
* max only applies when it's set above a non-zero min (yt-dlp requires the
|
||||
* min to be present, and a max below it is meaningless).
|
||||
*/
|
||||
sleepRequests: number
|
||||
sleepInterval: number
|
||||
maxSleepInterval: number
|
||||
/** use bundled aria2c (resources/bin/aria2c.exe) as the external downloader */
|
||||
useAria2c: boolean
|
||||
/** aria2c parallel connections per download (-x/-s), 1–16; only used when useAria2c is on (L64) */
|
||||
@@ -709,6 +723,9 @@ export const DEFAULT_SETTINGS: Settings = {
|
||||
downloadOptions: DEFAULT_DOWNLOAD_OPTIONS,
|
||||
proxy: '',
|
||||
rateLimit: '',
|
||||
sleepRequests: 0,
|
||||
sleepInterval: 0,
|
||||
maxSleepInterval: 0,
|
||||
useAria2c: false,
|
||||
aria2cConnections: 16,
|
||||
cookieSource: 'none',
|
||||
|
||||
@@ -127,6 +127,46 @@ describe('network options', () => {
|
||||
})
|
||||
})
|
||||
|
||||
// --- Request throttle: sleep-requests / sleep-interval ----------------------
|
||||
|
||||
describe('throttle options', () => {
|
||||
it('emits no sleep flags when all throttle values are 0/unset', () => {
|
||||
const argv = build(opts(), dlo(), NO_ACCESS)
|
||||
expect(argv).not.toContain('--sleep-requests')
|
||||
expect(argv).not.toContain('--sleep-interval')
|
||||
expect(argv).not.toContain('--max-sleep-interval')
|
||||
})
|
||||
|
||||
it('emits --sleep-requests when sleepRequests > 0', () => {
|
||||
const argv = build(opts(), dlo(), { ...NO_ACCESS, sleepRequests: 3 })
|
||||
expect(hasSeq(argv, '--sleep-requests', '3')).toBe(true)
|
||||
})
|
||||
|
||||
it('emits --sleep-interval when sleepInterval > 0', () => {
|
||||
const argv = build(opts(), dlo(), { ...NO_ACCESS, sleepInterval: 5 })
|
||||
expect(hasSeq(argv, '--sleep-interval', '5')).toBe(true)
|
||||
expect(argv).not.toContain('--max-sleep-interval')
|
||||
})
|
||||
|
||||
it('emits --max-sleep-interval only when it exceeds a non-zero sleepInterval', () => {
|
||||
const argv = build(opts(), dlo(), { ...NO_ACCESS, sleepInterval: 5, maxSleepInterval: 10 })
|
||||
expect(hasSeq(argv, '--sleep-interval', '5')).toBe(true)
|
||||
expect(hasSeq(argv, '--max-sleep-interval', '10')).toBe(true)
|
||||
})
|
||||
|
||||
it('drops --max-sleep-interval when sleepInterval is 0 (yt-dlp needs the min)', () => {
|
||||
const argv = build(opts(), dlo(), { ...NO_ACCESS, sleepInterval: 0, maxSleepInterval: 10 })
|
||||
expect(argv).not.toContain('--sleep-interval')
|
||||
expect(argv).not.toContain('--max-sleep-interval')
|
||||
})
|
||||
|
||||
it('drops --max-sleep-interval when it is not above the min', () => {
|
||||
const argv = build(opts(), dlo(), { ...NO_ACCESS, sleepInterval: 8, maxSleepInterval: 5 })
|
||||
expect(hasSeq(argv, '--sleep-interval', '8')).toBe(true)
|
||||
expect(argv).not.toContain('--max-sleep-interval')
|
||||
})
|
||||
})
|
||||
|
||||
// --- Cookies: browser store vs. sign-in window's exported file --------------
|
||||
|
||||
describe('cookies', () => {
|
||||
|
||||
@@ -106,6 +106,14 @@ describe('parseSettingsField (CC9 schema parity)', () => {
|
||||
expect(parseSettingsField('rateLimit', 'fast')).toEqual(fail)
|
||||
})
|
||||
|
||||
it('sleepRequests/sleepInterval/maxSleepInterval: coerce, round, clamp to [0,300]', () => {
|
||||
expect(parseSettingsField('sleepRequests', 5)).toEqual(ok(5))
|
||||
expect(parseSettingsField('sleepRequests', -3)).toEqual(ok(0))
|
||||
expect(parseSettingsField('sleepRequests', 999)).toEqual(ok(300))
|
||||
expect(parseSettingsField('sleepInterval', '4')).toEqual(ok(4))
|
||||
expect(parseSettingsField('maxSleepInterval', 7.6)).toEqual(ok(8))
|
||||
})
|
||||
|
||||
it('quality presets: allowlisted', () => {
|
||||
expect(parseSettingsField('defaultVideoQuality', '720p')).toEqual(ok('720p'))
|
||||
expect(parseSettingsField('defaultVideoQuality', '4K')).toEqual(fail)
|
||||
|
||||
Reference in New Issue
Block a user