AeroFetch/src/main/indexer.ts

/**
 * Source indexing orchestration (Pinchflat-style media manager; see
 * ROADMAP-PINCHFLAT.md). Walks a channel (its /playlists + /videos tabs) or a
 * single playlist with `yt-dlp --flat-playlist`, merges the result into a deduped
 * MediaItem list, and persists it as a Source. The download step (Phase G+) then
 * pulls from that persisted list rather than the live queue holding it all.
 *
 * The pure URL-classification + merge logic lives in indexerCore.ts (unit-tested);
 * this module is the impure shell that spawns yt-dlp and writes to disk.
 */

import { execFile } from 'child_process'
import { existsSync } from 'fs'
import { getYtdlpPath } from './binaries'
import { cleanError } from './log'
import { assertHttpUrl } from './url'
import {
  classifySource,
  buildMediaItems,
  buildFeedUrl,
  entryUrl,
  stripTabSuffix,
  stableSourceId,
  type RawEntry,
  type NamedPlaylist
} from './indexerCore'
import { getSource, upsertSource, mergeMediaItems } from './sources'
import type { IndexProgress, IndexSourceResult, Source, SourceKind } from '@shared/ipc'

/** The slice of `yt-dlp -J --flat-playlist` output the indexer reads. */
interface FlatInfo {
  _type?: string
  id?: string
  title?: string
  uploader?: string
  channel?: string
  /** the UC… channel id, used to build the RSS feed URL */
  channel_id?: string
  entries?: RawEntry[]
}

/**
 * Run `yt-dlp -J --flat-playlist` on a URL and parse the JSON. Rejects on a
 * non-zero exit or unparseable output. maxBuffer is large because a big channel's
 * flat upload list can be a few MB of JSON; the timeout is generous for the same
 * reason. `--` terminates option parsing so the URL can't be read as a flag.
 */
function probeFlat(url: string): Promise<FlatInfo> {
  return new Promise((resolve, reject) => {
    execFile(
      getYtdlpPath(),
      ['-J', '--flat-playlist', '--no-warnings', '--', url],
      { windowsHide: true, maxBuffer: 256 * 1024 * 1024, timeout: 180_000 },
      (err, stdout, stderr) => {
        if (err) {
          const msg = (err as { killed?: boolean }).killed
            ? 'Timed out indexing source. Check the link or your connection.'
            : cleanError(stderr) || err.message
          reject(new Error(msg))
          return
        }
        try {
          resolve(JSON.parse(stdout) as FlatInfo)
        } catch {
          reject(new Error('Could not parse source info from yt-dlp.'))
        }
      }
    )
  })
}

/** Probe a channel tab (e.g. /videos, /playlists); never throws — returns null. */
function probeTab(url: string): Promise<FlatInfo | null> {
  return probeFlat(url).catch(() => null)
}

/**
 * Index (or re-index) a Source. Resolves to the persisted Source + item count, or
 * an error. `onProgress` is invoked throughout so the renderer can show a live
 * status line — it's best-effort and never affects the result.
 */
export async function indexSource(
  url: string,
  onProgress: (p: IndexProgress) => void
): Promise<IndexSourceResult> {
  try {
    assertHttpUrl(url)
  } catch (e) {
    return { ok: false, error: (e as Error).message }
  }
  if (!existsSync(getYtdlpPath())) {
    return { ok: false, error: 'yt-dlp.exe not found. Drop it into resources/bin/.' }
  }

  const cls = classifySource(url)
  onProgress({ url, phase: 'start', message: 'Reading source…' })

  try {
    let kind: SourceKind
    let title: string
    let channel: string | undefined
    let feedId: string | undefined
    const playlists: NamedPlaylist[] = []
    let uploads: RawEntry[] = []

    if (cls?.kind === 'channel') {
      kind = 'channel'

      // 1. Enumerate the channel's playlists, then each playlist's videos.
      onProgress({ url, phase: 'playlists', message: 'Finding playlists…' })
      const playlistTab = await probeTab(`${cls.base}/playlists`)
      const playlistEntries = playlistTab?.entries ?? []
      const total = playlistEntries.length
      let done = 0
      for (const pe of playlistEntries) {
        done++
        const purl = entryUrl(pe)
        if (!purl) continue
        onProgress({
          url,
          phase: 'playlist',
          message: `Indexing playlist ${done}/${total}: ${pe.title ?? ''}`.trim(),
          current: done,
          total
        })
        const pdata = await probeTab(purl)
        if (pdata?.entries?.length) {
          playlists.push({ title: pe.title || 'Playlist', entries: pdata.entries })
        }
      }

      // 2. The full uploads feed — the catch-all for videos in no playlist.
      onProgress({ url, phase: 'uploads', message: 'Indexing channel uploads…' })
      const videoTab = await probeTab(`${cls.base}/videos`)
      uploads = videoTab?.entries ?? []

      title =
        stripTabSuffix(videoTab?.channel) ||
        stripTabSuffix(videoTab?.title) ||
        stripTabSuffix(playlistTab?.title) ||
        cls.base
      channel = stripTabSuffix(videoTab?.channel || videoTab?.uploader) || title
      feedId = videoTab?.channel_id || playlistTab?.channel_id
    } else {
      // A single playlist (classified) — or an unclassified URL that might still
      // resolve to a playlist when probed. A lone video has no entries → error.
      kind = cls?.kind ?? 'playlist'
      onProgress({ url, phase: 'uploads', message: 'Indexing playlist…' })
      const data = await probeFlat(cls?.base ?? url)
      const entries = data.entries ?? []
      if (entries.length === 0) {
        return {
          ok: false,
          error: 'That link is a single video, not a channel or playlist. Use the download bar for one video.'
        }
      }
      title = data.title || 'Playlist'
      channel = data.uploader || data.channel
      // A playlist feed keys off the playlist id (PL…), carried as `id` here.
      feedId = data.id
      // File everything under the playlist's own name (not the 'Uploads' fallback).
      playlists.push({ title, entries })
    }

    const sourceId = stableSourceId(cls?.base ?? url.trim())
    const fresh = buildMediaItems(sourceId, playlists, uploads)
    if (fresh.length === 0) {
      return { ok: false, error: 'No downloadable videos found for this source.' }
    }

    // Incremental merge: preserve the downloaded state of anything already on
    // disk, and report how many videos are new since the last index.
    const { items, newCount } = mergeMediaItems(sourceId, fresh)

    const prev = getSource(sourceId)
    const source: Source = {
      id: sourceId,
      url: url.trim(),
      kind,
      title,
      channel,
      addedAt: prev?.addedAt ?? Date.now(),
      lastIndexedAt: Date.now(),
      itemCount: items.length,
      // Preserve the watched flag across a re-index; refresh the RSS feed URL.
      watched: prev?.watched,
      feedUrl: buildFeedUrl(kind, feedId) ?? prev?.feedUrl
    }
    upsertSource(source)

    const newNote = newCount === items.length ? '' : ` (${newCount} new)`
    onProgress({
      url,
      phase: 'done',
      message: `Indexed ${items.length} videos${newNote}.`,
      total: items.length
    })
    return { ok: true, source, itemCount: items.length, newCount }
  } catch (e) {
    const msg = e instanceof Error ? e.message : String(e)
    onProgress({ url, phase: 'error', message: msg })
    return { ok: false, error: msg }
  }
}