Add Pinchflat-style media manager: index channels into playlist folders
Index an entire YouTube channel or playlist once, then download it into organized <Channel>/<Playlist>/<NNN> - <Title> folders, with incremental re-sync. Built in six rebuild-gated phases (F-K); see ROADMAP-PINCHFLAT.md. - F: channel-walk indexer (/playlists + /videos) -> persisted Source + MediaItem JSON stores; pure classify/dedup logic in indexerCore.ts - G: Windows-safe folder paths + dir sanitizer (collectionOutputTemplate) - H: Library tab + source tree + "Download pending" into the existing queue - I: state-preserving re-index merge + persist-downloaded-on-complete - J: watched sources, RSS fast-check, Task Scheduler + --sync launch (the OS-level scheduling/RSS need a real-install smoke test) - K: .info.json / thumbnail / .description sidecars for media servers Also gitignore .gitea-token. 106 unit tests pass; typecheck + build clean. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,204 @@
|
||||
/**
|
||||
* Source indexing orchestration (Pinchflat-style media manager; see
|
||||
* ROADMAP-PINCHFLAT.md). Walks a channel (its /playlists + /videos tabs) or a
|
||||
* single playlist with `yt-dlp --flat-playlist`, merges the result into a deduped
|
||||
* MediaItem list, and persists it as a Source. The download step (Phase G+) then
|
||||
* pulls from that persisted list rather than the live queue holding it all.
|
||||
*
|
||||
* The pure URL-classification + merge logic lives in indexerCore.ts (unit-tested);
|
||||
* this module is the impure shell that spawns yt-dlp and writes to disk.
|
||||
*/
|
||||
|
||||
import { execFile } from 'child_process'
|
||||
import { existsSync } from 'fs'
|
||||
import { getYtdlpPath } from './binaries'
|
||||
import { cleanError } from './log'
|
||||
import { assertHttpUrl } from './url'
|
||||
import {
|
||||
classifySource,
|
||||
buildMediaItems,
|
||||
buildFeedUrl,
|
||||
entryUrl,
|
||||
stripTabSuffix,
|
||||
stableSourceId,
|
||||
type RawEntry,
|
||||
type NamedPlaylist
|
||||
} from './indexerCore'
|
||||
import { getSource, upsertSource, mergeMediaItems } from './sources'
|
||||
import type { IndexProgress, IndexSourceResult, Source, SourceKind } from '@shared/ipc'
|
||||
|
||||
/** The slice of `yt-dlp -J --flat-playlist` output the indexer reads. */
|
||||
interface FlatInfo {
|
||||
_type?: string
|
||||
id?: string
|
||||
title?: string
|
||||
uploader?: string
|
||||
channel?: string
|
||||
/** the UC… channel id, used to build the RSS feed URL */
|
||||
channel_id?: string
|
||||
entries?: RawEntry[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Run `yt-dlp -J --flat-playlist` on a URL and parse the JSON. Rejects on a
|
||||
* non-zero exit or unparseable output. maxBuffer is large because a big channel's
|
||||
* flat upload list can be a few MB of JSON; the timeout is generous for the same
|
||||
* reason. `--` terminates option parsing so the URL can't be read as a flag.
|
||||
*/
|
||||
function probeFlat(url: string): Promise<FlatInfo> {
|
||||
return new Promise((resolve, reject) => {
|
||||
execFile(
|
||||
getYtdlpPath(),
|
||||
['-J', '--flat-playlist', '--no-warnings', '--', url],
|
||||
{ windowsHide: true, maxBuffer: 256 * 1024 * 1024, timeout: 180_000 },
|
||||
(err, stdout, stderr) => {
|
||||
if (err) {
|
||||
const msg = (err as { killed?: boolean }).killed
|
||||
? 'Timed out indexing source. Check the link or your connection.'
|
||||
: cleanError(stderr) || err.message
|
||||
reject(new Error(msg))
|
||||
return
|
||||
}
|
||||
try {
|
||||
resolve(JSON.parse(stdout) as FlatInfo)
|
||||
} catch {
|
||||
reject(new Error('Could not parse source info from yt-dlp.'))
|
||||
}
|
||||
}
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/** Probe a channel tab (e.g. /videos, /playlists); never throws — returns null. */
|
||||
function probeTab(url: string): Promise<FlatInfo | null> {
|
||||
return probeFlat(url).catch(() => null)
|
||||
}
|
||||
|
||||
/**
|
||||
* Index (or re-index) a Source. Resolves to the persisted Source + item count, or
|
||||
* an error. `onProgress` is invoked throughout so the renderer can show a live
|
||||
* status line — it's best-effort and never affects the result.
|
||||
*/
|
||||
export async function indexSource(
|
||||
url: string,
|
||||
onProgress: (p: IndexProgress) => void
|
||||
): Promise<IndexSourceResult> {
|
||||
try {
|
||||
assertHttpUrl(url)
|
||||
} catch (e) {
|
||||
return { ok: false, error: (e as Error).message }
|
||||
}
|
||||
if (!existsSync(getYtdlpPath())) {
|
||||
return { ok: false, error: 'yt-dlp.exe not found. Drop it into resources/bin/.' }
|
||||
}
|
||||
|
||||
const cls = classifySource(url)
|
||||
onProgress({ url, phase: 'start', message: 'Reading source…' })
|
||||
|
||||
try {
|
||||
let kind: SourceKind
|
||||
let title: string
|
||||
let channel: string | undefined
|
||||
let feedId: string | undefined
|
||||
const playlists: NamedPlaylist[] = []
|
||||
let uploads: RawEntry[] = []
|
||||
|
||||
if (cls?.kind === 'channel') {
|
||||
kind = 'channel'
|
||||
|
||||
// 1. Enumerate the channel's playlists, then each playlist's videos.
|
||||
onProgress({ url, phase: 'playlists', message: 'Finding playlists…' })
|
||||
const playlistTab = await probeTab(`${cls.base}/playlists`)
|
||||
const playlistEntries = playlistTab?.entries ?? []
|
||||
const total = playlistEntries.length
|
||||
let done = 0
|
||||
for (const pe of playlistEntries) {
|
||||
done++
|
||||
const purl = entryUrl(pe)
|
||||
if (!purl) continue
|
||||
onProgress({
|
||||
url,
|
||||
phase: 'playlist',
|
||||
message: `Indexing playlist ${done}/${total}: ${pe.title ?? ''}`.trim(),
|
||||
current: done,
|
||||
total
|
||||
})
|
||||
const pdata = await probeTab(purl)
|
||||
if (pdata?.entries?.length) {
|
||||
playlists.push({ title: pe.title || 'Playlist', entries: pdata.entries })
|
||||
}
|
||||
}
|
||||
|
||||
// 2. The full uploads feed — the catch-all for videos in no playlist.
|
||||
onProgress({ url, phase: 'uploads', message: 'Indexing channel uploads…' })
|
||||
const videoTab = await probeTab(`${cls.base}/videos`)
|
||||
uploads = videoTab?.entries ?? []
|
||||
|
||||
title =
|
||||
stripTabSuffix(videoTab?.channel) ||
|
||||
stripTabSuffix(videoTab?.title) ||
|
||||
stripTabSuffix(playlistTab?.title) ||
|
||||
cls.base
|
||||
channel = stripTabSuffix(videoTab?.channel || videoTab?.uploader) || title
|
||||
feedId = videoTab?.channel_id || playlistTab?.channel_id
|
||||
} else {
|
||||
// A single playlist (classified) — or an unclassified URL that might still
|
||||
// resolve to a playlist when probed. A lone video has no entries → error.
|
||||
kind = cls?.kind ?? 'playlist'
|
||||
onProgress({ url, phase: 'uploads', message: 'Indexing playlist…' })
|
||||
const data = await probeFlat(cls?.base ?? url)
|
||||
const entries = data.entries ?? []
|
||||
if (entries.length === 0) {
|
||||
return {
|
||||
ok: false,
|
||||
error: 'That link is a single video, not a channel or playlist. Use the download bar for one video.'
|
||||
}
|
||||
}
|
||||
title = data.title || 'Playlist'
|
||||
channel = data.uploader || data.channel
|
||||
// A playlist feed keys off the playlist id (PL…), carried as `id` here.
|
||||
feedId = data.id
|
||||
// File everything under the playlist's own name (not the 'Uploads' fallback).
|
||||
playlists.push({ title, entries })
|
||||
}
|
||||
|
||||
const sourceId = stableSourceId(cls?.base ?? url.trim())
|
||||
const fresh = buildMediaItems(sourceId, playlists, uploads)
|
||||
if (fresh.length === 0) {
|
||||
return { ok: false, error: 'No downloadable videos found for this source.' }
|
||||
}
|
||||
|
||||
// Incremental merge: preserve the downloaded state of anything already on
|
||||
// disk, and report how many videos are new since the last index.
|
||||
const { items, newCount } = mergeMediaItems(sourceId, fresh)
|
||||
|
||||
const prev = getSource(sourceId)
|
||||
const source: Source = {
|
||||
id: sourceId,
|
||||
url: url.trim(),
|
||||
kind,
|
||||
title,
|
||||
channel,
|
||||
addedAt: prev?.addedAt ?? Date.now(),
|
||||
lastIndexedAt: Date.now(),
|
||||
itemCount: items.length,
|
||||
// Preserve the watched flag across a re-index; refresh the RSS feed URL.
|
||||
watched: prev?.watched,
|
||||
feedUrl: buildFeedUrl(kind, feedId) ?? prev?.feedUrl
|
||||
}
|
||||
upsertSource(source)
|
||||
|
||||
const newNote = newCount === items.length ? '' : ` (${newCount} new)`
|
||||
onProgress({
|
||||
url,
|
||||
phase: 'done',
|
||||
message: `Indexed ${items.length} videos${newNote}.`,
|
||||
total: items.length
|
||||
})
|
||||
return { ok: true, source, itemCount: items.length, newCount }
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e)
|
||||
onProgress({ url, phase: 'error', message: msg })
|
||||
return { ok: false, error: msg }
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user