Add Pinchflat-style media manager: index channels into playlist folders
Index an entire YouTube channel or playlist once, then download it into organized <Channel>/<Playlist>/<NNN> - <Title> folders, with incremental re-sync. Built in six rebuild-gated phases (F-K); see ROADMAP-PINCHFLAT.md. - F: channel-walk indexer (/playlists + /videos) -> persisted Source + MediaItem JSON stores; pure classify/dedup logic in indexerCore.ts - G: Windows-safe folder paths + dir sanitizer (collectionOutputTemplate) - H: Library tab + source tree + "Download pending" into the existing queue - I: state-preserving re-index merge + persist-downloaded-on-complete - J: watched sources, RSS fast-check, Task Scheduler + --sync launch (the OS-level scheduling/RSS need a real-install smoke test) - K: .info.json / thumbnail / .description sidecars for media servers Also gitignore .gitea-token. 106 unit tests pass; typecheck + build clean. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -3,6 +3,8 @@ import {
|
||||
buildArgs,
|
||||
parseExtraArgs,
|
||||
formatCommandLine,
|
||||
sanitizeDirSegment,
|
||||
collectionOutputTemplate,
|
||||
CROP_SQUARE_PPA,
|
||||
ARIA2C_ARGS,
|
||||
type AccessOptions
|
||||
@@ -360,3 +362,91 @@ describe('buildArgs extraArgs', () => {
|
||||
expect(withEmpty).toEqual(withoutParam)
|
||||
})
|
||||
})
|
||||
|
||||
// --- Media-server sidecar files (Phase K) -----------------------------------
|
||||
|
||||
describe('sidecar files', () => {
|
||||
it('emits --write-info-json / --write-thumbnail / --write-description when enabled', () => {
|
||||
const argv = build(
|
||||
opts(),
|
||||
dlo({ writeInfoJson: true, writeThumbnailFile: true, writeDescription: true })
|
||||
)
|
||||
expect(argv).toContain('--write-info-json')
|
||||
expect(argv).toContain('--write-thumbnail')
|
||||
expect(argv).toContain('--write-description')
|
||||
})
|
||||
|
||||
it('emits none of them by default', () => {
|
||||
const argv = build(opts(), dlo())
|
||||
expect(argv).not.toContain('--write-info-json')
|
||||
expect(argv).not.toContain('--write-description')
|
||||
// (--write-thumbnail is sidecar-only here; embedThumbnail uses --embed-thumbnail)
|
||||
expect(argv).not.toContain('--write-thumbnail')
|
||||
})
|
||||
})
|
||||
|
||||
// --- Collection folder paths (Phase G, media-manager) -----------------------
|
||||
|
||||
describe('sanitizeDirSegment', () => {
|
||||
it('keeps an ordinary name (incl. spaces and hyphens) intact', () => {
|
||||
expect(sanitizeDirSegment('Linus Tech Tips')).toBe('Linus Tech Tips')
|
||||
expect(sanitizeDirSegment('Two-Minute Papers')).toBe('Two-Minute Papers')
|
||||
})
|
||||
|
||||
it('replaces Windows-illegal characters with a space and collapses runs', () => {
|
||||
expect(sanitizeDirSegment('A/B\\C:D*E?F')).toBe('A B C D E F')
|
||||
expect(sanitizeDirSegment('Best of 2024 <Live>')).toBe('Best of 2024 Live')
|
||||
})
|
||||
|
||||
it('strips leading/trailing dots and spaces, neutralising `..` traversal', () => {
|
||||
expect(sanitizeDirSegment('..')).toBe('Untitled')
|
||||
expect(sanitizeDirSegment('../../etc')).toBe('etc')
|
||||
expect(sanitizeDirSegment(' trailing. ')).toBe('trailing')
|
||||
expect(sanitizeDirSegment('My Playlist.')).toBe('My Playlist')
|
||||
})
|
||||
|
||||
it('prefixes reserved Windows device names', () => {
|
||||
expect(sanitizeDirSegment('CON')).toBe('_CON')
|
||||
expect(sanitizeDirSegment('com1')).toBe('_com1')
|
||||
})
|
||||
|
||||
it('falls back to Untitled for empty / all-illegal input', () => {
|
||||
expect(sanitizeDirSegment('')).toBe('Untitled')
|
||||
expect(sanitizeDirSegment('???')).toBe('Untitled')
|
||||
})
|
||||
|
||||
it('caps length to keep the overall path bounded', () => {
|
||||
expect(sanitizeDirSegment('x'.repeat(200)).length).toBe(80)
|
||||
})
|
||||
})
|
||||
|
||||
describe('collectionOutputTemplate', () => {
|
||||
const ctx = { channel: 'DevChannel', playlist: 'Full Course', index: 3 }
|
||||
|
||||
it('files into <out>/<channel>/<playlist>/<NNN> - <filename>', () => {
|
||||
const t = collectionOutputTemplate('C:/out', ctx, '%(title)s.%(ext)s')
|
||||
// normalise separators so the assertion is OS-independent
|
||||
expect(t.replace(/\\/g, '/')).toBe('C:/out/DevChannel/Full Course/003 - %(title)s.%(ext)s')
|
||||
})
|
||||
|
||||
it('zero-pads the index to three digits and clamps bad values to 001', () => {
|
||||
expect(collectionOutputTemplate('C:/o', { ...ctx, index: 42 }, 'f').replace(/\\/g, '/')).toContain(
|
||||
'/042 - f'
|
||||
)
|
||||
expect(collectionOutputTemplate('C:/o', { ...ctx, index: 0 }, 'f').replace(/\\/g, '/')).toContain(
|
||||
'/001 - f'
|
||||
)
|
||||
expect(
|
||||
collectionOutputTemplate('C:/o', { ...ctx, index: NaN }, 'f').replace(/\\/g, '/')
|
||||
).toContain('/001 - f')
|
||||
})
|
||||
|
||||
it('sanitizes the channel and playlist folder names', () => {
|
||||
const t = collectionOutputTemplate(
|
||||
'C:/out',
|
||||
{ channel: 'A/B', playlist: '..', index: 1 },
|
||||
'%(title)s.%(ext)s'
|
||||
)
|
||||
expect(t.replace(/\\/g, '/')).toBe('C:/out/A B/Untitled/001 - %(title)s.%(ext)s')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import {
|
||||
classifySource,
|
||||
buildMediaItems,
|
||||
mergeItemsPreservingState,
|
||||
buildFeedUrl,
|
||||
parseRssVideoIds,
|
||||
entryUrl,
|
||||
fmtDuration,
|
||||
stripTabSuffix,
|
||||
stableSourceId,
|
||||
type RawEntry
|
||||
} from '../src/main/indexerCore'
|
||||
import type { MediaItem } from '@shared/ipc'
|
||||
|
||||
describe('classifySource', () => {
|
||||
it('classifies channel handle / id / c / user forms and strips the tab', () => {
|
||||
expect(classifySource('https://www.youtube.com/@LinusTechTips')).toEqual({
|
||||
kind: 'channel',
|
||||
base: 'https://www.youtube.com/@LinusTechTips'
|
||||
})
|
||||
expect(classifySource('https://youtube.com/@Foo/videos')).toEqual({
|
||||
kind: 'channel',
|
||||
base: 'https://www.youtube.com/@Foo'
|
||||
})
|
||||
expect(classifySource('https://www.youtube.com/channel/UC123/playlists')?.base).toBe(
|
||||
'https://www.youtube.com/channel/UC123'
|
||||
)
|
||||
expect(classifySource('https://www.youtube.com/c/SomeName')?.kind).toBe('channel')
|
||||
expect(classifySource('https://www.youtube.com/user/Legacy')?.kind).toBe('channel')
|
||||
})
|
||||
|
||||
it('classifies a playlist by its list= param and canonicalises it', () => {
|
||||
expect(classifySource('https://www.youtube.com/playlist?list=PL_abc')).toEqual({
|
||||
kind: 'playlist',
|
||||
base: 'https://www.youtube.com/playlist?list=PL_abc'
|
||||
})
|
||||
// a watch URL that also carries list= is treated as the playlist
|
||||
expect(classifySource('https://www.youtube.com/watch?v=xyz&list=PLfoo')?.kind).toBe('playlist')
|
||||
})
|
||||
|
||||
it('returns null for a lone video, a bad URL, or a non-YouTube host', () => {
|
||||
expect(classifySource('https://www.youtube.com/watch?v=dQw4w9WgXcQ')).toBeNull()
|
||||
expect(classifySource('not a url')).toBeNull()
|
||||
expect(classifySource('ftp://example.com/@chan')).toBeNull()
|
||||
expect(classifySource('https://vimeo.com/channels/staffpicks')).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('entryUrl', () => {
|
||||
it('prefers an explicit http(s) url, else builds a watch url from the id', () => {
|
||||
expect(entryUrl({ url: 'https://youtu.be/abc' })).toBe('https://youtu.be/abc')
|
||||
expect(entryUrl({ webpage_url: 'https://x/y' })).toBe('https://x/y')
|
||||
expect(entryUrl({ id: 'vid123' })).toBe('https://www.youtube.com/watch?v=vid123')
|
||||
})
|
||||
it('rejects a non-http url and returns null when nothing usable is present', () => {
|
||||
expect(entryUrl({ url: 'javascript:alert(1)' })).toBeNull()
|
||||
expect(entryUrl({})).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('fmtDuration', () => {
|
||||
it('formats seconds as M:SS or H:MM:SS', () => {
|
||||
expect(fmtDuration(0)).toBe('0:00')
|
||||
expect(fmtDuration(75)).toBe('1:15')
|
||||
expect(fmtDuration(3661)).toBe('1:01:01')
|
||||
expect(fmtDuration(undefined)).toBeUndefined()
|
||||
expect(fmtDuration(NaN)).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe('stripTabSuffix', () => {
|
||||
it('strips a trailing " - <Tab>" suffix only', () => {
|
||||
expect(stripTabSuffix('Creator - Videos')).toBe('Creator')
|
||||
expect(stripTabSuffix('Creator - Playlists')).toBe('Creator')
|
||||
expect(stripTabSuffix('Just A Name')).toBe('Just A Name')
|
||||
expect(stripTabSuffix('Tech - Tips')).toBe('Tech - Tips') // 'Tips' isn't a tab
|
||||
expect(stripTabSuffix(undefined)).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe('stableSourceId', () => {
|
||||
it('is deterministic and 8 hex chars', () => {
|
||||
const a = stableSourceId('https://www.youtube.com/@Foo')
|
||||
expect(a).toMatch(/^[0-9a-f]{8}$/)
|
||||
expect(stableSourceId('https://www.youtube.com/@Foo')).toBe(a)
|
||||
expect(stableSourceId('https://www.youtube.com/@Bar')).not.toBe(a)
|
||||
})
|
||||
})
|
||||
|
||||
describe('buildMediaItems', () => {
|
||||
const vids = (...ids: string[]): RawEntry[] => ids.map((id) => ({ id, title: `T-${id}` }))
|
||||
|
||||
it('files videos under their playlist with 1-based per-playlist index', () => {
|
||||
const items = buildMediaItems('src1', [{ title: 'Series A', entries: vids('a', 'b') }], [])
|
||||
expect(items).toHaveLength(2)
|
||||
expect(items[0]).toMatchObject({
|
||||
id: 'src1:a',
|
||||
sourceId: 'src1',
|
||||
videoId: 'a',
|
||||
playlistTitle: 'Series A',
|
||||
playlistIndex: 1,
|
||||
downloaded: false
|
||||
})
|
||||
expect(items[1]).toMatchObject({ videoId: 'b', playlistIndex: 2 })
|
||||
})
|
||||
|
||||
it('dedups across playlists — the first playlist a video appears in wins', () => {
|
||||
const items = buildMediaItems(
|
||||
'src1',
|
||||
[
|
||||
{ title: 'First', entries: vids('x', 'y') },
|
||||
{ title: 'Second', entries: vids('y', 'z') }
|
||||
],
|
||||
[]
|
||||
)
|
||||
expect(items.map((i) => i.videoId)).toEqual(['x', 'y', 'z'])
|
||||
expect(items.find((i) => i.videoId === 'y')?.playlistTitle).toBe('First')
|
||||
})
|
||||
|
||||
it('falls back to the synthetic Uploads folder for videos in no playlist', () => {
|
||||
const items = buildMediaItems('src1', [{ title: 'P', entries: vids('a') }], vids('a', 'b'))
|
||||
// 'a' already filed under P; only 'b' becomes an Uploads item
|
||||
expect(items).toHaveLength(2)
|
||||
expect(items.find((i) => i.videoId === 'a')?.playlistTitle).toBe('P')
|
||||
const b = items.find((i) => i.videoId === 'b')
|
||||
expect(b).toMatchObject({ playlistTitle: 'Uploads', playlistIndex: 2 })
|
||||
})
|
||||
|
||||
it('skips entries with no id or no resolvable URL', () => {
|
||||
const items = buildMediaItems('src1', [{ title: 'P', entries: [{ title: 'no id' }] }], [])
|
||||
expect(items).toHaveLength(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('mergeItemsPreservingState', () => {
|
||||
const item = (videoId: string, over: Partial<MediaItem> = {}): MediaItem => ({
|
||||
id: `s:${videoId}`,
|
||||
sourceId: 's',
|
||||
videoId,
|
||||
title: videoId,
|
||||
url: `https://youtube.com/watch?v=${videoId}`,
|
||||
playlistTitle: 'P',
|
||||
playlistIndex: 1,
|
||||
downloaded: false,
|
||||
...over
|
||||
})
|
||||
|
||||
it('carries downloaded state forward for videos already on disk', () => {
|
||||
const existing = [item('a', { downloaded: true, downloadedAt: 111, filePath: 'C:/a.mp4' })]
|
||||
const fresh = [item('a'), item('b')]
|
||||
const { items, newCount } = mergeItemsPreservingState(existing, fresh)
|
||||
expect(newCount).toBe(1) // only 'b' is new
|
||||
expect(items.find((i) => i.videoId === 'a')).toMatchObject({
|
||||
downloaded: true,
|
||||
downloadedAt: 111,
|
||||
filePath: 'C:/a.mp4'
|
||||
})
|
||||
expect(items.find((i) => i.videoId === 'b')?.downloaded).toBe(false)
|
||||
})
|
||||
|
||||
it('adopts the fresh membership/order and drops vanished videos', () => {
|
||||
const existing = [item('a', { downloaded: true }), item('gone', { downloaded: true })]
|
||||
const fresh = [item('a', { playlistTitle: 'Moved', playlistIndex: 5 })]
|
||||
const { items, newCount } = mergeItemsPreservingState(existing, fresh)
|
||||
expect(items).toHaveLength(1) // 'gone' dropped
|
||||
expect(newCount).toBe(0)
|
||||
// fresh playlist assignment wins, but downloaded state is preserved
|
||||
expect(items[0]).toMatchObject({ playlistTitle: 'Moved', playlistIndex: 5, downloaded: true })
|
||||
})
|
||||
|
||||
it('counts every video as new on a first index (empty existing)', () => {
|
||||
const { newCount } = mergeItemsPreservingState([], [item('a'), item('b'), item('c')])
|
||||
expect(newCount).toBe(3)
|
||||
})
|
||||
})
|
||||
|
||||
describe('buildFeedUrl', () => {
|
||||
it('builds a channel feed by channel_id and a playlist feed by playlist_id', () => {
|
||||
expect(buildFeedUrl('channel', 'UCabc')).toBe(
|
||||
'https://www.youtube.com/feeds/videos.xml?channel_id=UCabc'
|
||||
)
|
||||
expect(buildFeedUrl('playlist', 'PLxyz')).toBe(
|
||||
'https://www.youtube.com/feeds/videos.xml?playlist_id=PLxyz'
|
||||
)
|
||||
})
|
||||
it('returns undefined when the id is missing', () => {
|
||||
expect(buildFeedUrl('channel', undefined)).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe('parseRssVideoIds', () => {
|
||||
it('extracts every <yt:videoId> from a feed body, in order', () => {
|
||||
const xml = `
|
||||
<feed><entry><yt:videoId>aaa111</yt:videoId></entry>
|
||||
<entry><yt:videoId> bbb222 </yt:videoId></entry></feed>`
|
||||
expect(parseRssVideoIds(xml)).toEqual(['aaa111', 'bbb222'])
|
||||
})
|
||||
it('returns an empty array for a feed with no entries', () => {
|
||||
expect(parseRssVideoIds('<feed></feed>')).toEqual([])
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user