Add Pinchflat-style media manager: index channels into playlist folders
Index an entire YouTube channel or playlist once, then download it into organized <Channel>/<Playlist>/<NNN> - <Title> folders, with incremental re-sync. Built in six rebuild-gated phases (F-K); see ROADMAP-PINCHFLAT.md. - F: channel-walk indexer (/playlists + /videos) -> persisted Source + MediaItem JSON stores; pure classify/dedup logic in indexerCore.ts - G: Windows-safe folder paths + dir sanitizer (collectionOutputTemplate) - H: Library tab + source tree + "Download pending" into the existing queue - I: state-preserving re-index merge + persist-downloaded-on-complete - J: watched sources, RSS fast-check, Task Scheduler + --sync launch (the OS-level scheduling/RSS need a real-install smoke test) - K: .info.json / thumbnail / .description sidecars for media servers Also gitignore .gitea-token. 106 unit tests pass; typecheck + build clean. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,135 @@
|
||||
/**
|
||||
* Persistence for the media-manager index (Pinchflat-style; see
|
||||
* ROADMAP-PINCHFLAT.md). Two plain-JSON stores in userData, mirroring the
|
||||
* history.ts pattern (and the same deliberate choice to stay on JSON rather than
|
||||
* better-sqlite3 — revisit if a user indexes many large channels, see the
|
||||
* Phase H risk note in the roadmap):
|
||||
*
|
||||
* sources.json — one Source record per added channel/playlist
|
||||
* media-items.json — every MediaItem across all sources (queried by sourceId)
|
||||
*
|
||||
* Per-row validation on read (validation.ts) so a hand-edited or corrupted file
|
||||
* can't feed the UI malformed records (same approach as history/errorlog).
|
||||
*/
|
||||
|
||||
import { app } from 'electron'
|
||||
import { join } from 'path'
|
||||
import { readFileSync, writeFileSync, existsSync } from 'fs'
|
||||
import type { Source, MediaItem } from '@shared/ipc'
|
||||
import { isValidSource, isValidMediaItem } from './validation'
|
||||
import { mergeItemsPreservingState } from './indexerCore'
|
||||
|
||||
// A generous global cap so a runaway index can't grow the file unbounded; large
|
||||
// enough for several big channels. When exceeded, the most-recently-written
|
||||
// source's items are kept (they're placed first by replaceMediaItems).
|
||||
const MAX_ITEMS = 20000
|
||||
|
||||
function sourcesFile(): string {
|
||||
return join(app.getPath('userData'), 'sources.json')
|
||||
}
|
||||
|
||||
function itemsFile(): string {
|
||||
return join(app.getPath('userData'), 'media-items.json')
|
||||
}
|
||||
|
||||
function readJsonArray<T>(path: string, isValid: (o: unknown) => o is T): T[] {
|
||||
try {
|
||||
if (!existsSync(path)) return []
|
||||
const data = JSON.parse(readFileSync(path, 'utf8'))
|
||||
return Array.isArray(data) ? data.filter(isValid) : []
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
function writeJson(path: string, value: unknown): void {
|
||||
try {
|
||||
writeFileSync(path, JSON.stringify(value, null, 2))
|
||||
} catch {
|
||||
/* best-effort; a read-only data dir just means no persisted index */
|
||||
}
|
||||
}
|
||||
|
||||
// --- Sources ----------------------------------------------------------------
|
||||
|
||||
export function listSources(): Source[] {
|
||||
return readJsonArray(sourcesFile(), isValidSource)
|
||||
}
|
||||
|
||||
export function getSource(id: string): Source | undefined {
|
||||
return listSources().find((s) => s.id === id)
|
||||
}
|
||||
|
||||
/** Insert or replace a source by id (a re-index updates the existing record). */
|
||||
export function upsertSource(source: Source): Source[] {
|
||||
const sources = [source, ...listSources().filter((s) => s.id !== source.id)]
|
||||
writeJson(sourcesFile(), sources)
|
||||
return sources
|
||||
}
|
||||
|
||||
/** Toggle whether a source is watched for new uploads (Phase J). Returns all sources. */
|
||||
export function setSourceWatched(id: string, watched: boolean): Source[] {
|
||||
const sources = listSources().map((s) => (s.id === id ? { ...s, watched } : s))
|
||||
writeJson(sourcesFile(), sources)
|
||||
return sources
|
||||
}
|
||||
|
||||
/** Remove a source and all of its media items. Returns the remaining sources. */
|
||||
export function removeSource(id: string): Source[] {
|
||||
const sources = listSources().filter((s) => s.id !== id)
|
||||
writeJson(sourcesFile(), sources)
|
||||
const items = listAllItems().filter((m) => m.sourceId !== id)
|
||||
writeJson(itemsFile(), items)
|
||||
return sources
|
||||
}
|
||||
|
||||
// --- Media items ------------------------------------------------------------
|
||||
|
||||
function listAllItems(): MediaItem[] {
|
||||
return readJsonArray(itemsFile(), isValidMediaItem)
|
||||
}
|
||||
|
||||
export function listMediaItems(sourceId: string): MediaItem[] {
|
||||
return listAllItems().filter((m) => m.sourceId === sourceId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace all media items for one source with a fresh set (the result of a
|
||||
* (re)index). Other sources' items are preserved; the new items are placed first
|
||||
* so they survive the MAX_ITEMS cap.
|
||||
*/
|
||||
export function replaceMediaItems(sourceId: string, items: MediaItem[]): void {
|
||||
const others = listAllItems().filter((m) => m.sourceId !== sourceId)
|
||||
writeJson(itemsFile(), [...items, ...others].slice(0, MAX_ITEMS))
|
||||
}
|
||||
|
||||
/**
|
||||
* Incrementally merge a freshly-indexed item list into the persisted store,
|
||||
* preserving the downloaded state of videos already on disk (see Phase I). The
|
||||
* fresh list defines current membership/order; returns the merged items and how
|
||||
* many were new since the last index.
|
||||
*/
|
||||
export function mergeMediaItems(
|
||||
sourceId: string,
|
||||
fresh: MediaItem[]
|
||||
): { items: MediaItem[]; newCount: number } {
|
||||
const result = mergeItemsPreservingState(listMediaItems(sourceId), fresh)
|
||||
replaceMediaItems(sourceId, result.items)
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark one media item downloaded, recording its file path + time. Returns the
|
||||
* updated list for that item's source (or [] if the id is unknown). Used by the
|
||||
* library view once a queued item completes (Phase H/I).
|
||||
*/
|
||||
export function setMediaItemDownloaded(id: string, filePath?: string): MediaItem[] {
|
||||
const all = listAllItems()
|
||||
const target = all.find((m) => m.id === id)
|
||||
if (!target) return []
|
||||
const updated = all.map((m) =>
|
||||
m.id === id ? { ...m, downloaded: true, downloadedAt: Date.now(), filePath } : m
|
||||
)
|
||||
writeJson(itemsFile(), updated)
|
||||
return updated.filter((m) => m.sourceId === target.sourceId)
|
||||
}
|
||||
Reference in New Issue
Block a user