feat(m4): persist the download queue across restarts
The queue/scheduler lived only in renderer memory, so saved/scheduled and other pending downloads were silently lost on quit (audit M4). Persist them: - main: queue.ts store (proven cached-atomic createJsonStore, queue.json) + queue:list / queue:save IPC; QUEUE_MAX cap. - shared: PersistedQueueItem = the durable subset of DownloadItem (runtime-only progress/speed/eta/sizeLabel/finishing dropped). - renderer: mirror the persistable items on every queue change (a signature over the subset means progress ticks don't re-save; main's jsonStore coalesces the writes) and rehydrate on launch via an App useEffect. Pure mappers extracted to store/queuePersist.ts and unit-tested. Restore semantics: downloading -> queued (yt-dlp continues the .part), saved + scheduledFor survives so the promoter fires when due, paused/error return actionable; completed/canceled are never persisted. A queueHydrated gate stops a launch-time store change from wiping queue.json before it's read. Reconciles ROADMAP.md. typecheck + 268 tests + eslint + prettier green. (OS-level quit/relaunch cycle is worth a manual smoke test.) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -63,3 +63,5 @@ export const ERRORLOG_MAX_ENTRIES = 200
|
||||
export const TEMPLATES_MAX = 100
|
||||
/** Indexed media items across all sources (media-items.json). */
|
||||
export const MEDIA_ITEMS_MAX = 20_000
|
||||
/** Persisted download-queue rows (queue.json, M4) — generous; a whole channel can be queued. */
|
||||
export const QUEUE_MAX = 10_000
|
||||
|
||||
+8
-1
@@ -21,7 +21,8 @@ import {
|
||||
type CommandTemplate,
|
||||
type YtdlpUpdateChannel,
|
||||
type SystemThemeInfo,
|
||||
type TaskbarProgress
|
||||
type TaskbarProgress,
|
||||
type PersistedQueueItem
|
||||
} from '@shared/ipc'
|
||||
import { PAGE_BACKGROUND } from '@shared/theme'
|
||||
import { getYtdlpVersion, updateYtdlp } from './ytdlp'
|
||||
@@ -36,6 +37,7 @@ import { listTemplates, saveTemplate, removeTemplate } from './templates'
|
||||
import { safeOpenPath, safeShowInFolder } from './reveal'
|
||||
import { openCookieLoginWindow, getCookiesStatus, clearCookies } from './cookies'
|
||||
import { listErrorLog, addErrorLog, clearErrorLog } from './errorlog'
|
||||
import { listQueue, saveQueue } from './queue'
|
||||
import { exportBackup, importBackup } from './backup'
|
||||
import {
|
||||
listSources,
|
||||
@@ -208,6 +210,11 @@ export function registerIpcHandlers(getMainWindow: () => BrowserWindow | null):
|
||||
ipcMain.handle(IpcChannels.errorLogList, () => listErrorLog())
|
||||
ipcMain.handle(IpcChannels.errorLogClear, () => clearErrorLog())
|
||||
|
||||
// Persisted download queue (M4): the renderer mirrors its durable items here and
|
||||
// rehydrates them on launch, so saved/scheduled/pending downloads survive a quit.
|
||||
ipcMain.handle(IpcChannels.queueList, () => listQueue())
|
||||
ipcMain.handle(IpcChannels.queueSave, (_e, items: PersistedQueueItem[]) => saveQueue(items))
|
||||
|
||||
ipcMain.handle(IpcChannels.backupExport, (e) =>
|
||||
exportBackup(BrowserWindow.fromWebContents(e.sender) ?? undefined)
|
||||
)
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import { createJsonStore } from './jsonStore'
|
||||
import { QUEUE_MAX } from './constants'
|
||||
import type { PersistedQueueItem, PersistedQueueStatus } from '@shared/ipc'
|
||||
|
||||
/**
|
||||
* Persistence for the download queue (M4). The queue lived only in renderer memory,
|
||||
* so `saved` / scheduled and other pending downloads were silently lost on quit.
|
||||
* The renderer now mirrors its durable items here (via `queueSave`) and rehydrates
|
||||
* them on launch (via `queueList`), so a parked/scheduled download returns.
|
||||
*
|
||||
* Reuses the shared cached-atomic `createJsonStore` (R1/R2/R3), so writes are atomic
|
||||
* and coalesced — the renderer can call `queueSave` freely on queue changes.
|
||||
*/
|
||||
|
||||
const PERSISTED_STATUSES: readonly PersistedQueueStatus[] = [
|
||||
'queued',
|
||||
'downloading',
|
||||
'paused',
|
||||
'saved',
|
||||
'error'
|
||||
]
|
||||
|
||||
/** Defensive row validator (a hand-edited or partially-written file can't crash a read). */
|
||||
function isValidQueueItem(o: unknown): o is PersistedQueueItem {
|
||||
if (typeof o !== 'object' || o === null) return false
|
||||
const it = o as Record<string, unknown>
|
||||
return (
|
||||
typeof it.id === 'string' &&
|
||||
typeof it.url === 'string' &&
|
||||
typeof it.title === 'string' &&
|
||||
(it.kind === 'video' || it.kind === 'audio') &&
|
||||
typeof it.quality === 'string' &&
|
||||
typeof it.status === 'string' &&
|
||||
(PERSISTED_STATUSES as readonly string[]).includes(it.status)
|
||||
)
|
||||
}
|
||||
|
||||
const store = createJsonStore('queue.json', isValidQueueItem, QUEUE_MAX)
|
||||
|
||||
export function listQueue(): PersistedQueueItem[] {
|
||||
return store.read()
|
||||
}
|
||||
|
||||
export function saveQueue(items: PersistedQueueItem[]): void {
|
||||
store.write(items)
|
||||
}
|
||||
@@ -31,7 +31,8 @@ import {
|
||||
type ScheduledSyncStatus,
|
||||
type TerminalEvent,
|
||||
type TerminalRunResult,
|
||||
type TaskbarProgress
|
||||
type TaskbarProgress,
|
||||
type PersistedQueueItem
|
||||
} from '@shared/ipc'
|
||||
|
||||
// The surface exposed to the renderer. Keep this thin: it only forwards to IPC.
|
||||
@@ -139,6 +140,13 @@ const api = {
|
||||
|
||||
clearErrorLog: (): Promise<ErrorLogEntry[]> => ipcRenderer.invoke(IpcChannels.errorLogClear),
|
||||
|
||||
/** Load the persisted download queue on launch (M4). */
|
||||
listQueue: (): Promise<PersistedQueueItem[]> => ipcRenderer.invoke(IpcChannels.queueList),
|
||||
|
||||
/** Mirror the renderer's durable queue items to disk so they survive a quit (M4). */
|
||||
saveQueue: (items: PersistedQueueItem[]): Promise<void> =>
|
||||
ipcRenderer.invoke(IpcChannels.queueSave, items),
|
||||
|
||||
/** Opens a save dialog and writes settings + templates to the chosen JSON file. */
|
||||
exportBackup: (): Promise<BackupExportResult> => ipcRenderer.invoke(IpcChannels.backupExport),
|
||||
|
||||
|
||||
@@ -98,11 +98,17 @@ function App(): React.JSX.Element {
|
||||
return useDownloads.subscribe((st) => push(st.items))
|
||||
}, [])
|
||||
|
||||
// Rehydrate the persisted download queue on launch (M4) so saved/scheduled and
|
||||
// other pending items return from the previous session.
|
||||
useEffect(() => {
|
||||
useDownloads.getState().hydrate()
|
||||
}, [])
|
||||
|
||||
// Promote scheduled ('saved' + a due scheduledFor) items when their time arrives.
|
||||
// Session-only: the queue isn't persisted, so a schedule only fires while the app
|
||||
// runs (noted in ROADMAP.md). A 15s tick is plenty for minute-granularity times.
|
||||
// Lives here (not at the downloads store's module load) so importing the store in
|
||||
// tests/preview doesn't start a stray timer, and the interval is cleared cleanly
|
||||
// The queue is persisted (M4), so a scheduled item survives a quit and fires on the
|
||||
// next launch once its time has passed. A 15s tick is plenty for minute-granularity
|
||||
// times. Lives here (not at the downloads store's module load) so importing the store
|
||||
// in tests/preview doesn't start a stray timer, and the interval is cleared cleanly
|
||||
// on unmount (L1).
|
||||
useEffect(() => {
|
||||
const tick = setInterval(() => {
|
||||
|
||||
@@ -184,6 +184,8 @@ export const mockApi: Api = {
|
||||
onYtdlpAutoUpdateStatus: () => () => {},
|
||||
listErrorLog: async () => [],
|
||||
clearErrorLog: async () => [],
|
||||
listQueue: async () => [],
|
||||
saveQueue: async () => {},
|
||||
exportBackup: async () => ({
|
||||
ok: true,
|
||||
path: 'C:\\Users\\you\\Downloads\\aerofetch-backup.json'
|
||||
|
||||
@@ -124,4 +124,6 @@ export interface DownloadState {
|
||||
pump: () => void
|
||||
/** internal: apply a main-process download event */
|
||||
applyEvent: (ev: DownloadEvent) => void
|
||||
/** Load the persisted queue from main on launch (M4); no-op in preview. */
|
||||
hydrate: () => void
|
||||
}
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
import { create } from 'zustand'
|
||||
import { type MediaKind } from '@shared/ipc'
|
||||
import { isPreview as PREVIEW } from '../isPreview'
|
||||
import { logError } from '../reportError'
|
||||
import { useSettings } from './settings'
|
||||
import { useHistory } from './history'
|
||||
import { emit, on } from './coordinator'
|
||||
import { RESOLVING, buildItem } from './downloadItem'
|
||||
import { fromPersisted, persistableItems } from './queuePersist'
|
||||
import { seed } from './downloadSeed'
|
||||
import {
|
||||
type ChosenFormat,
|
||||
@@ -19,6 +21,11 @@ import {
|
||||
// `import { … } from '../store/downloads'` sites keep working.
|
||||
export type { MediaKind, ChosenFormat, DownloadItem, DownloadStatus, AddOptions, AddEntry }
|
||||
|
||||
// M4: gate the persist subscription until the initial hydrate has read the saved
|
||||
// queue, so a launch-time store mutation can't overwrite queue.json with [] before
|
||||
// we've loaded it back.
|
||||
let queueHydrated = false
|
||||
|
||||
// Side-effects to run exactly once when a download completes: record it to
|
||||
// history (unless private) and mark its source MediaItem downloaded. Shared by
|
||||
// the real `applyEvent('done')` path and the preview ticker so the two can't
|
||||
@@ -154,6 +161,29 @@ export const useDownloads = create<DownloadState>((set, get) => {
|
||||
items: seed,
|
||||
pump,
|
||||
|
||||
hydrate: () => {
|
||||
if (PREVIEW) return
|
||||
window.api
|
||||
.listQueue()
|
||||
.then((persisted) => {
|
||||
if (persisted.length === 0) return
|
||||
// Merge ahead of anything already present (normally nothing at launch),
|
||||
// de-duping by id so a double-hydrate can't double-insert.
|
||||
set((s) => {
|
||||
const have = new Set(s.items.map((i) => i.id))
|
||||
const add = persisted.filter((p) => !have.has(p.id)).map(fromPersisted)
|
||||
return add.length ? { items: [...s.items, ...add] } : {}
|
||||
})
|
||||
pump()
|
||||
})
|
||||
.catch(logError('listQueue'))
|
||||
.finally(() => {
|
||||
// Enable persistence once the read has completed (success or failure), so
|
||||
// subsequent queue changes are saved but the pre-hydrate window can't wipe.
|
||||
queueHydrated = true
|
||||
})
|
||||
},
|
||||
|
||||
addFromUrl: (url, kind, quality, opts) => {
|
||||
set((s) => ({ items: [buildItem(url, kind, quality, opts), ...s.items] }))
|
||||
pump()
|
||||
@@ -409,6 +439,24 @@ export const useDownloads = create<DownloadState>((set, get) => {
|
||||
// no direct import from sources, so the two stores stay acyclic.
|
||||
on('enqueueDownloads', (entries) => useDownloads.getState().addMany(entries))
|
||||
|
||||
// M4: mirror the durable queue to main whenever it changes, so saved/scheduled and
|
||||
// other pending items survive a quit. Skipped in preview (no IPC bridge). A
|
||||
// progress-only update doesn't churn this — toPersisted drops progress/speed/eta, so
|
||||
// the persistable snapshot is unchanged by a tick and no redundant save fires. Main's
|
||||
// jsonStore coalesces the actual disk writes.
|
||||
if (!PREVIEW) {
|
||||
let lastSig = ''
|
||||
useDownloads.subscribe((state) => {
|
||||
// Don't persist until the initial hydrate has read the saved file (above).
|
||||
if (!queueHydrated) return
|
||||
const persistable = persistableItems(state.items)
|
||||
const sig = JSON.stringify(persistable)
|
||||
if (sig === lastSig) return
|
||||
lastSig = sig
|
||||
window.api.saveQueue(persistable).catch(logError('saveQueue'))
|
||||
})
|
||||
}
|
||||
|
||||
// The scheduled-download promoter tick lives in App (a useEffect), not here, so
|
||||
// importing this store in tests/preview doesn't start a stray never-cleared timer
|
||||
// on module load (L1). See SCHEDULE_TICK_MS / App.tsx.
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
import { type PersistedQueueItem, type PersistedQueueStatus } from '@shared/ipc'
|
||||
import { type DownloadItem, type DownloadStatus } from './downloadTypes'
|
||||
|
||||
/**
|
||||
* Pure mapping between the live `DownloadItem` and the durable `PersistedQueueItem`
|
||||
* main stores across restarts (M4). Kept free of any store/zustand import so it can
|
||||
* be unit-tested in isolation (mirrors queueStats / formatters). Only stable fields
|
||||
* persist — runtime-only progress/speed/eta/sizeLabel/finishing are dropped and
|
||||
* re-derived on restore.
|
||||
*/
|
||||
|
||||
// Statuses worth surviving a quit. 'completed'/'canceled' are transient results
|
||||
// (completions already live in history) and are never persisted.
|
||||
export const PERSISTED_STATUSES: readonly DownloadStatus[] = [
|
||||
'queued',
|
||||
'downloading',
|
||||
'paused',
|
||||
'saved',
|
||||
'error'
|
||||
]
|
||||
|
||||
export function toPersisted(i: DownloadItem): PersistedQueueItem {
|
||||
return {
|
||||
id: i.id,
|
||||
url: i.url,
|
||||
title: i.title,
|
||||
channel: i.channel,
|
||||
durationLabel: i.durationLabel,
|
||||
thumbnail: i.thumbnail,
|
||||
kind: i.kind,
|
||||
quality: i.quality,
|
||||
status: i.status as PersistedQueueStatus,
|
||||
filePath: i.filePath,
|
||||
error: i.error,
|
||||
formatId: i.formatId,
|
||||
formatHasAudio: i.formatHasAudio,
|
||||
options: i.options,
|
||||
extraArgs: i.extraArgs,
|
||||
trim: i.trim,
|
||||
scheduledFor: i.scheduledFor,
|
||||
incognito: i.incognito,
|
||||
probedMeta: i.probedMeta,
|
||||
collection: i.collection,
|
||||
mediaItemId: i.mediaItemId
|
||||
}
|
||||
}
|
||||
|
||||
export function fromPersisted(p: PersistedQueueItem): DownloadItem {
|
||||
return {
|
||||
...p,
|
||||
// A download in flight when we quit lost its process; re-queue it (yt-dlp
|
||||
// continues the .part on relaunch). Progress restarts from 0.
|
||||
status: p.status === 'downloading' ? 'queued' : p.status,
|
||||
progress: 0
|
||||
}
|
||||
}
|
||||
|
||||
/** The persistable subset of the live queue, in durable form. */
|
||||
export function persistableItems(items: DownloadItem[]): PersistedQueueItem[] {
|
||||
return items
|
||||
.filter((i) => (PERSISTED_STATUSES as readonly string[]).includes(i.status))
|
||||
.map(toPersisted)
|
||||
}
|
||||
@@ -46,6 +46,9 @@ export const IpcChannels = {
|
||||
ytdlpAutoUpdateStatus: 'ytdlp:auto-update-status',
|
||||
errorLogList: 'errorlog:list',
|
||||
errorLogClear: 'errorlog:clear',
|
||||
/** Persisted download queue (M4): saved/scheduled/pending items survive a quit. */
|
||||
queueList: 'queue:list',
|
||||
queueSave: 'queue:save',
|
||||
backupExport: 'backup:export',
|
||||
backupImport: 'backup:import',
|
||||
systemThemeGet: 'system-theme:get',
|
||||
@@ -501,6 +504,41 @@ export type DownloadEvent =
|
||||
| { type: 'done'; id: string; filePath?: string }
|
||||
| { type: 'error'; id: string; error: string }
|
||||
|
||||
/** The subset of a persisted queue item's status worth surviving a quit. */
|
||||
export type PersistedQueueStatus = 'queued' | 'downloading' | 'paused' | 'saved' | 'error'
|
||||
|
||||
/**
|
||||
* A queue item persisted across restarts (M4) so saved/scheduled and other pending
|
||||
* downloads return after a quit. This is the *durable* subset of the renderer's
|
||||
* `DownloadItem`: runtime-only fields (progress/speed/eta/sizeLabel/finishing) are
|
||||
* dropped since a restored item re-derives them. Transient results ('completed' /
|
||||
* 'canceled') are never persisted; on load a 'downloading' item is normalized back
|
||||
* to 'queued' because its process died with the previous session.
|
||||
*/
|
||||
export interface PersistedQueueItem {
|
||||
id: string
|
||||
url: string
|
||||
title: string
|
||||
channel?: string
|
||||
durationLabel?: string
|
||||
thumbnail?: string
|
||||
kind: MediaKind
|
||||
quality: string
|
||||
status: PersistedQueueStatus
|
||||
filePath?: string
|
||||
error?: string
|
||||
formatId?: string
|
||||
formatHasAudio?: boolean
|
||||
options?: DownloadOptions
|
||||
extraArgs?: string
|
||||
trim?: string
|
||||
scheduledFor?: number
|
||||
incognito?: boolean
|
||||
probedMeta?: DownloadMeta
|
||||
collection?: CollectionContext
|
||||
mediaItemId?: string
|
||||
}
|
||||
|
||||
/** Persisted user settings (electron-store). */
|
||||
export interface Settings {
|
||||
/** where video downloads are saved; empty string = the default Documents\Video folder */
|
||||
|
||||
Reference in New Issue
Block a user