From 8e161fe9cee70bdbc8bd3816350f53b1271ce1a0 Mon Sep 17 00:00:00 2001 From: debont80 Date: Fri, 26 Jun 2026 07:27:52 -0400 Subject: [PATCH] feat: Library bulk download + list virtualization for large channels Add a source-wide "Select all" across all playlist groups, and make one click queue every indexed video (the previous 100-per-click cap is gone). Bulk download skips items that are already downloaded or in flight, so it can't enqueue duplicates, and it includes failed/canceled ones so it also retries them. Make the UI hold up at channel scale (1000s of videos): virtualize both the Downloads queue and the Library item list with @tanstack/react-virtual so only on-screen rows hit the DOM, memoize the queue row so they don't all reconcile on every progress tick, and batch the enqueue so queuing a whole channel stays O(n) instead of O(n^2). - VirtualList: reusable windowed list that owns its own scroll container - DownloadsView: virtualized, flex-filled queue - LibraryView: groups flattened to rows; virtualized panel above 100 rows, inline below so small sources are unchanged - QueueItem: React.memo - downloads store: addMany (one state update + one pump); sources: enqueue the whole selection via addMany, no cap Co-Authored-By: Claude Opus 4.8 --- package-lock.json | 32 ++- package.json | 1 + src/renderer/src/components/DownloadsView.tsx | 30 ++- src/renderer/src/components/LibraryView.tsx | 193 ++++++++++++------ src/renderer/src/components/QueueItem.tsx | 9 +- src/renderer/src/components/VirtualList.tsx | 70 +++++++ src/renderer/src/store/downloads.ts | 115 ++++++----- src/renderer/src/store/sources.ts | 47 ++--- 8 files changed, 354 insertions(+), 143 deletions(-) create mode 100644 src/renderer/src/components/VirtualList.tsx diff --git a/package-lock.json b/package-lock.json index a0fcf77..904574b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,16 +1,17 @@ { "name": "aerofetch", - "version": "0.1.0", + "version": "0.4.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "aerofetch", - "version": "0.1.0", + "version": "0.4.1", "dependencies": { "@electron-toolkit/utils": "^4.0.0", "@fluentui/react-components": "^9.74.1", "@fluentui/react-icons": "^2.0.330", + "@tanstack/react-virtual": "^3.14.3", "electron-store": "^11.0.2", "react": "^19.2.7", "react-dom": "^19.2.7", @@ -3373,6 +3374,33 @@ "node": ">=10" } }, + "node_modules/@tanstack/react-virtual": { + "version": "3.14.3", + "resolved": "https://registry.npmjs.org/@tanstack/react-virtual/-/react-virtual-3.14.3.tgz", + "integrity": "sha512-k/cnHPVaOfn46hSbiY6n4Dzf4QjCGWSF40zR5QIIYUqPAjpA6TN7InfYmcMiDVQGP2iUn9xsRbAl8u1v3UmeVQ==", + "license": "MIT", + "dependencies": { + "@tanstack/virtual-core": "3.17.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/@tanstack/virtual-core": { + "version": "3.17.1", + "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.17.1.tgz", + "integrity": "sha512-VZyW2Uiml5tmBZwPGrSD3Sz73OxzljQMCmzYHsUTPEuTsERf5xwa+uWb01xEzkz3ZSYTjj8NEb/mKHvgKxyZdA==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + } + }, "node_modules/@types/babel__core": { "version": "7.20.5", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", diff --git a/package.json b/package.json index 9424335..d2a9d8d 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "@electron-toolkit/utils": "^4.0.0", "@fluentui/react-components": "^9.74.1", "@fluentui/react-icons": "^2.0.330", + "@tanstack/react-virtual": "^3.14.3", "electron-store": "^11.0.2", "react": "^19.2.7", "react-dom": "^19.2.7", diff --git a/src/renderer/src/components/DownloadsView.tsx b/src/renderer/src/components/DownloadsView.tsx index efd0e55..cb88ba4 100644 --- a/src/renderer/src/components/DownloadsView.tsx +++ b/src/renderer/src/components/DownloadsView.tsx @@ -9,22 +9,28 @@ import { ArrowDownloadRegular } from '@fluentui/react-icons' import { useDownloads } from '../store/downloads' import { DownloadBar } from './DownloadBar' import { QueueItem } from './QueueItem' +import { VirtualList } from './VirtualList' const useStyles = makeStyles({ root: { + // Fill the scroll area so the queue list below can flex to the remaining + // height and scroll internally (it's virtualized), instead of the whole page + // growing to thousands of rows. display: 'flex', flexDirection: 'column', - gap: '20px' + gap: '20px', + height: '100%' }, queueHeader: { display: 'flex', alignItems: 'center', justifyContent: 'space-between' }, - list: { - display: 'flex', - flexDirection: 'column', - gap: '10px' + listScroll: { + flexGrow: 1, + // min-height:0 lets this flex child shrink below its content height so it, + // not the page, owns the scrolling. + minHeight: 0 }, empty: { display: 'flex', @@ -65,11 +71,15 @@ export function DownloadsView(): React.JSX.Element { Nothing queued yet. Paste a URL above to get started. ) : ( -
- {items.map((item) => ( - - ))} -
+ 100} + getKey={(item) => item.id} + renderItem={(item) => } + /> )} ) diff --git a/src/renderer/src/components/LibraryView.tsx b/src/renderer/src/components/LibraryView.tsx index 0fffe6a..5cca864 100644 --- a/src/renderer/src/components/LibraryView.tsx +++ b/src/renderer/src/components/LibraryView.tsx @@ -28,11 +28,12 @@ import { LibraryRegular } from '@fluentui/react-icons' import type { MediaItem, Source } from '@shared/ipc' -import { useSources, MAX_ENQUEUE_BATCH } from '../store/sources' +import { useSources } from '../store/sources' import { useSettings } from '../store/settings' import { useDownloads, type DownloadStatus } from '../store/downloads' import { thumbUrl } from '../thumb' import { MediaThumb } from './MediaThumb' +import { VirtualList } from './VirtualList' // True in the standalone browser preview (no Electron preload). const PREVIEW = typeof window === 'undefined' || !window.electron @@ -40,6 +41,18 @@ const PREVIEW = typeof window === 'undefined' || !window.electron /** Per-item status shown in the library: a live queue status, or pending/downloaded. */ type ItemStatus = DownloadStatus | 'pending' +/** + * A flattened row for the virtualized item list: either a playlist header or a + * single video. Flattening the groups into one list lets a 1000s-video source + * window cleanly (one virtualizer over a flat array, headers included). + */ +type LibRow = + | { kind: 'header'; title: string; items: MediaItem[] } + | { kind: 'item'; item: MediaItem } + +/** Above this many flattened rows, the item list switches to a virtualized panel. */ +const VIRTUALIZE_AT = 100 + const STATUS_LABEL: Record = { pending: 'Pending', queued: 'Queued', @@ -134,7 +147,6 @@ const useStyles = makeStyles({ }, actionRow: { display: 'flex', alignItems: 'center', gap: '8px', flexWrap: 'wrap' }, actionSpacer: { flexGrow: 1 }, - group: { display: 'flex', flexDirection: 'column' }, groupHead: { display: 'flex', alignItems: 'center', @@ -143,13 +155,13 @@ const useStyles = makeStyles({ color: tokens.colorNeutralForeground2 }, groupTitle: { fontWeight: tokens.fontWeightSemibold, flexGrow: 1, minWidth: 0 }, - rows: { display: 'flex', flexDirection: 'column' }, row: { display: 'flex', alignItems: 'center', gap: '10px', padding: '5px 4px 5px 18px' }, + plainList: { display: 'flex', flexDirection: 'column' }, rowThumb: { width: '60px', height: '34px', @@ -236,6 +248,7 @@ export function LibraryView(): React.JSX.Element { const [url, setUrl] = useState('') const [error, setError] = useState(null) const [selected, setSelected] = useState>(new Set()) + const [batchNote, setBatchNote] = useState(null) const [syncNote, setSyncNote] = useState(null) const [scheduled, setScheduled] = useState(false) @@ -261,8 +274,11 @@ export function LibraryView(): React.JSX.Element { if (res.error) setError(res.error) } - // Reset the selection whenever the expanded source changes. - useEffect(() => setSelected(new Set()), [selectedSourceId]) + // Reset the selection (and any batch note) whenever the expanded source changes. + useEffect(() => { + setSelected(new Set()) + setBatchNote(null) + }, [selectedSourceId]) // Live per-URL queue status so a video row reflects its real download state. const statusByUrl = useMemo(() => { @@ -273,12 +289,34 @@ export function LibraryView(): React.JSX.Element { const items = selectedSourceId ? (itemsBySource[selectedSourceId] ?? []) : [] const groups = useMemo(() => groupByPlaylist(items), [items]) + // Flatten groups → [header, ...its items, header, ...] for the virtualized list. + const flatRows = useMemo(() => { + const rows: LibRow[] = [] + for (const g of groups) { + rows.push({ kind: 'header', title: g.title, items: g.items }) + for (const it of g.items) rows.push({ kind: 'item', item: it }) + } + return rows + }, [groups]) const effStatus = (it: MediaItem): ItemStatus => statusByUrl.get(it.url) ?? (it.downloaded ? 'completed' : 'pending') const pendingItems = items.filter((it) => effStatus(it) === 'pending') + // An item is worth (re)queuing when it isn't already downloaded or in flight: + // pending, or a previous failure/cancel to retry. Skipping completed/queued/ + // downloading items keeps "Select all → Download" from enqueuing duplicates, + // since addFromUrl doesn't dedupe by URL. + const actionable = (it: MediaItem): boolean => { + const st = effStatus(it) + return st === 'pending' || st === 'error' || st === 'canceled' + } + const actionableItems = items.filter(actionable) + const selectedActionable = actionableItems.filter((it) => selected.has(it.id)) + const allActionableSelected = + actionableItems.length > 0 && actionableItems.every((it) => selected.has(it.id)) + async function onIndex(): Promise { setError(null) const u = url.trim() @@ -308,16 +346,24 @@ export function LibraryView(): React.JSX.Element { }) } + // Select / clear every downloadable item across all groups in this source. + function toggleAll(on: boolean): void { + setSelected(on ? new Set(actionableItems.map((it) => it.id)) : new Set()) + } + + // One click queues every chosen item — maxConcurrent gates how many actually + // run, the rest wait in the queue. Selection clears since nothing is held back. function downloadSelected(): void { - if (!selectedSourceId) return - const chosen = items.filter((it) => selected.has(it.id)) - enqueueItems(selectedSourceId, chosen) + if (!selectedSourceId || selectedActionable.length === 0) return + const n = enqueueItems(selectedSourceId, selectedActionable) setSelected(new Set()) + setBatchNote(`Queued ${n} download${n === 1 ? '' : 's'}.`) } function downloadPending(): void { - if (!selectedSourceId) return - enqueueItems(selectedSourceId, pendingItems) + if (!selectedSourceId || pendingItems.length === 0) return + const n = enqueueItems(selectedSourceId, pendingItems) + setBatchNote(`Queued ${n} download${n === 1 ? '' : 's'}.`) } function pillClass(status: ItemStatus): string { @@ -327,6 +373,52 @@ export function LibraryView(): React.JSX.Element { return styles.pill } + // One row of the item list — a playlist header or a video — shared by the + // inline (small source) and virtualized (large source) render paths. + function rowKey(row: LibRow): string { + return row.kind === 'header' ? `h:${row.title}` : row.item.id + } + + function renderRow(row: LibRow): React.JSX.Element { + if (row.kind === 'header') { + const allOn = row.items.every((it) => selected.has(it.id)) + return ( +
+ + {row.title} + {row.items.length} + +
+ ) + } + const it = row.item + const status = effStatus(it) + return ( +
+ toggle(it.id, !!d.checked)} + aria-label={`Select ${it.title}`} + /> + +
+ + {it.playlistIndex}. {it.title} + + {it.durationLabel && {it.durationLabel}} +
+ {STATUS_LABEL[status]} +
+ ) + } + return (
@@ -418,6 +510,12 @@ export function LibraryView(): React.JSX.Element { >
+ 0 ? 'mixed' : false} + onChange={(_, d) => toggleAll(!!d.checked)} + label="Select all" + disabled={actionableItems.length === 0} + /> {items.length} videos · {pendingItems.length} pending · indexed{' '} {relTime(src.lastIndexedAt)} @@ -433,8 +531,9 @@ export function LibraryView(): React.JSX.Element { appearance="primary" icon={} onClick={downloadSelected} + disabled={selectedActionable.length === 0} > - Download {Math.min(selected.size, MAX_ENQUEUE_BATCH)} selected + Download {selectedActionable.length} selected ) : ( @@ -445,7 +544,7 @@ export function LibraryView(): React.JSX.Element { onClick={downloadPending} disabled={pendingItems.length === 0} > - Download {Math.min(pendingItems.length, MAX_ENQUEUE_BATCH)} pending + Download {pendingItems.length} pending )} @@ -475,54 +574,28 @@ export function LibraryView(): React.JSX.Element {
- {groups.map((g) => { - const allOn = g.items.every((it) => selected.has(it.id)) - return ( -
-
- - {g.title} - {g.items.length} - -
-
- {g.items.map((it) => { - const status = effStatus(it) - return ( -
- toggle(it.id, !!d.checked)} - aria-label={`Select ${it.title}`} - /> - -
- - {it.playlistIndex}. {it.title} - - {it.durationLabel && ( - {it.durationLabel} - )} -
- {STATUS_LABEL[status]} -
- ) - })} -
-
- ) - })} + {batchNote && {batchNote}} + + {flatRows.length > VIRTUALIZE_AT ? ( + // Big source: a fixed-height, internally-scrolling virtualized + // panel so 1000s of rows stay light. A definite height (not + // max-height) gives the virtualizer a viewport to measure. + (flatRows[i].kind === 'header' ? 40 : 46)} + getKey={(row) => rowKey(row)} + renderItem={(row) => renderRow(row)} + /> + ) : ( + // Small source: render inline so the card grows naturally. +
+ {flatRows.map((row) => ( +
{renderRow(row)}
+ ))} +
+ )}
))} diff --git a/src/renderer/src/components/QueueItem.tsx b/src/renderer/src/components/QueueItem.tsx index 394d5bc..89eb1e0 100644 --- a/src/renderer/src/components/QueueItem.tsx +++ b/src/renderer/src/components/QueueItem.tsx @@ -1,3 +1,4 @@ +import { memo } from 'react' import { Text, Caption1, @@ -99,7 +100,11 @@ function pct(progress: number): string { return `${Math.round(progress * 100)}%` } -export function QueueItem({ item }: { item: DownloadItem }): React.JSX.Element { +export const QueueItem = memo(function QueueItem({ + item +}: { + item: DownloadItem +}): React.JSX.Element { const styles = useStyles() const cancel = useDownloads((s) => s.cancel) const remove = useDownloads((s) => s.remove) @@ -234,4 +239,4 @@ export function QueueItem({ item }: { item: DownloadItem }): React.JSX.Element {
) -} +}) diff --git a/src/renderer/src/components/VirtualList.tsx b/src/renderer/src/components/VirtualList.tsx new file mode 100644 index 0000000..9bd9bc0 --- /dev/null +++ b/src/renderer/src/components/VirtualList.tsx @@ -0,0 +1,70 @@ +import { useRef, type CSSProperties, type ReactNode } from 'react' +import { useVirtualizer } from '@tanstack/react-virtual' + +/** + * Windowed list: renders only the rows near the viewport, so a list thousands of + * items long stays light — one DOM subtree per visible row, not per item. This + * keeps the Downloads queue and a big channel's Library list responsive when a + * source has 1000s of videos. + * + * It owns its own scroll container, so size that container via `style`/`className` + * (e.g. `flex: 1` + `minHeight: 0`, or a `maxHeight`). Owning the scroller means + * the virtualizer anchors at scroll offset 0 and never has to track where the + * list sits within the page — robust even though the app shares one outer + * scroll area with content above each list. + * + * Row heights are measured (measureElement), so variable-height rows are fine; + * `estimateSize` only needs to be a rough first guess before measurement. + */ +export function VirtualList({ + items, + estimateSize, + getKey, + renderItem, + overscan = 8, + gap = 0, + className, + style +}: { + items: T[] + estimateSize: (index: number) => number + getKey: (item: T, index: number) => string + renderItem: (item: T, index: number) => ReactNode + overscan?: number + gap?: number + className?: string + style?: CSSProperties +}): React.JSX.Element { + const scrollRef = useRef(null) + const virtualizer = useVirtualizer({ + count: items.length, + getScrollElement: () => scrollRef.current, + estimateSize, + overscan, + gap, + getItemKey: (index) => getKey(items[index], index) + }) + + return ( +
+
+ {virtualizer.getVirtualItems().map((vrow) => ( +
+ {renderItem(items[vrow.index], vrow.index)} +
+ ))} +
+
+ ) +} diff --git a/src/renderer/src/store/downloads.ts b/src/renderer/src/store/downloads.ts index 14ce7ca..afc8784 100644 --- a/src/renderer/src/store/downloads.ts +++ b/src/renderer/src/store/downloads.ts @@ -54,29 +54,41 @@ export const QUALITY_OPTIONS: Record = { audio: ['Best (MP3)', '320 kbps', '192 kbps', '128 kbps'] } +/** Options accepted when enqueuing a URL (shared by addFromUrl + addMany). */ +export interface AddOptions { + format?: ChosenFormat + title?: string + channel?: string + durationLabel?: string + thumbnail?: string + options?: DownloadOptions + extraArgs?: string + incognito?: boolean + collection?: CollectionContext + mediaItemId?: string +} + +/** One URL to enqueue in a single addMany batch. */ +export interface AddEntry { + url: string + kind: MediaKind + quality: string + opts?: AddOptions +} + // True when running in the standalone browser preview (no Electron preload). // The real preload injects window.electron; the browser stub does not. const PREVIEW = typeof window === 'undefined' || !window.electron interface DownloadState { items: DownloadItem[] - addFromUrl: ( - url: string, - kind: MediaKind, - quality: string, - opts?: { - format?: ChosenFormat - title?: string - channel?: string - durationLabel?: string - thumbnail?: string - options?: DownloadOptions - extraArgs?: string - incognito?: boolean - collection?: CollectionContext - mediaItemId?: string - } - ) => void + addFromUrl: (url: string, kind: MediaKind, quality: string, opts?: AddOptions) => void + /** + * Enqueue many URLs at once with a single state update and a single pump, so + * queuing an entire channel (1000s of items) doesn't churn the store O(n²) the + * way looping addFromUrl would. + */ + addMany: (entries: AddEntry[]) => void retry: (id: string) => void cancel: (id: string) => void remove: (id: string) => void @@ -95,6 +107,38 @@ function newId(): string { return `item-${++idCounter}` } +/** + * Build a fresh queued DownloadItem from a URL + options. Pure (no store access) + * so addFromUrl and addMany share it. If the caller already probed the URL, its + * metadata is carried as probedMeta so main can skip a redundant probe (audit P2). + */ +function buildItem(url: string, kind: MediaKind, quality: string, opts?: AddOptions): DownloadItem { + const probedMeta: DownloadMeta | undefined = + opts?.title || opts?.channel || opts?.durationLabel + ? { title: opts?.title, channel: opts?.channel, durationLabel: opts?.durationLabel } + : undefined + return { + id: newId(), + url, + title: opts?.title ?? titleFromUrl(url), + channel: opts?.channel ?? 'Resolving…', + durationLabel: opts?.durationLabel, + thumbnail: opts?.thumbnail, + kind, + quality, + status: 'queued', + progress: 0, + formatId: opts?.format?.id, + formatHasAudio: opts?.format?.hasAudio, + options: opts?.options, + extraArgs: opts?.extraArgs, + incognito: opts?.incognito, + collection: opts?.collection, + mediaItemId: opts?.mediaItemId, + probedMeta + } +} + function titleFromUrl(url: string): string { try { const u = new URL(url) @@ -291,35 +335,14 @@ export const useDownloads = create((set, get) => { pump, addFromUrl: (url, kind, quality, opts) => { - const id = newId() - // If the caller already probed the URL, carry that metadata so main can skip - // its own redundant probe (audit P2). Only set it when something real was - // provided — a bare paste with no probe leaves it undefined. - const probedMeta: DownloadMeta | undefined = - opts?.title || opts?.channel || opts?.durationLabel - ? { title: opts?.title, channel: opts?.channel, durationLabel: opts?.durationLabel } - : undefined - const item: DownloadItem = { - id, - url, - title: opts?.title ?? titleFromUrl(url), - channel: opts?.channel ?? 'Resolving…', - durationLabel: opts?.durationLabel, - thumbnail: opts?.thumbnail, - kind, - quality, - status: 'queued', - progress: 0, - formatId: opts?.format?.id, - formatHasAudio: opts?.format?.hasAudio, - options: opts?.options, - extraArgs: opts?.extraArgs, - incognito: opts?.incognito, - collection: opts?.collection, - mediaItemId: opts?.mediaItemId, - probedMeta - } - set((s) => ({ items: [item, ...s.items] })) + set((s) => ({ items: [buildItem(url, kind, quality, opts), ...s.items] })) + pump() + }, + + addMany: (entries) => { + if (entries.length === 0) return + const built = entries.map((e) => buildItem(e.url, e.kind, e.quality, e.opts)) + set((s) => ({ items: [...built, ...s.items] })) pump() }, diff --git a/src/renderer/src/store/sources.ts b/src/renderer/src/store/sources.ts index 1b89a4c..0cd1a37 100644 --- a/src/renderer/src/store/sources.ts +++ b/src/renderer/src/store/sources.ts @@ -6,11 +6,6 @@ import { useSettings } from './settings' // True in the standalone browser preview (no Electron preload). const PREVIEW = typeof window === 'undefined' || !window.electron -// Cap on how many items one "Download" click enqueues, so an entire channel -// can't flood the live queue at once — the rest stay pending for a follow-up -// click. (The automatic incremental feeder is Phase I; see ROADMAP-PINCHFLAT.md.) -export const MAX_ENQUEUE_BATCH = 100 - // --- Preview seed data ------------------------------------------------------ function seedItems(sourceId: string, playlist: string, n: number, downloadedUpTo = 0): MediaItem[] { @@ -87,8 +82,10 @@ interface SourcesState { reindexSource: (id: string) => Promise removeSource: (id: string) => void /** - * Enqueue the given media items into the download queue with folder context, - * capped at MAX_ENQUEUE_BATCH. Returns how many were actually enqueued. + * Enqueue the given media items into the download queue with folder context. + * Queues all of them — one click can fill the queue with an entire channel, + * which maxConcurrent then gates into download slots. Returns how many were + * enqueued. */ enqueueItems: (sourceId: string, items: MediaItem[]) => number /** mark an item downloaded once its queue download completes (called by the downloads store) */ @@ -185,22 +182,26 @@ export const useSources = create((set, get) => ({ const settings = useSettings.getState() const kind = settings.defaultKind const quality = kind === 'video' ? settings.defaultVideoQuality : settings.defaultAudioQuality - const batch = items.slice(0, MAX_ENQUEUE_BATCH) - const add = useDownloads.getState().addFromUrl - for (const it of batch) { - add(it.url, kind, quality, { - title: it.title, - channel: src?.channel, - durationLabel: it.durationLabel, - collection: { - channel: src?.channel || src?.title || 'Channel', - playlist: it.playlistTitle, - index: it.playlistIndex - }, - mediaItemId: it.id - }) - } - return batch.length + // One batched state update + pump, so queuing a whole channel stays O(n). + useDownloads.getState().addMany( + items.map((it) => ({ + url: it.url, + kind, + quality, + opts: { + title: it.title, + channel: src?.channel, + durationLabel: it.durationLabel, + collection: { + channel: src?.channel || src?.title || 'Channel', + playlist: it.playlistTitle, + index: it.playlistIndex + }, + mediaItemId: it.id + } + })) + ) + return items.length }, markDownloaded: (itemId, filePath) => {