Fix R8: compact JSON for the large media-items store
writeJsonAtomic always pretty-printed (JSON.stringify(value, null, 2)), which needlessly inflates size and write time for the media-items store (up to MAX_ITEMS=20000 rows). Added an optional `pretty` flag (default true, so history/sources/templates/errorlog stay hand-inspectable) threaded through createJsonStore, and set it false for media-items.json. typecheck + 242 tests + eslint green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+1
-1
@@ -1350,7 +1350,7 @@ cosmetics. `R` IDs.
|
|||||||
- [ ] **R7 — `safeStorage` unavailable ⇒ secrets silently stored as plaintext.** `encryptSecret` falls back
|
- [ ] **R7 — `safeStorage` unavailable ⇒ secrets silently stored as plaintext.** `encryptSecret` falls back
|
||||||
to plaintext when DPAPI/safeStorage isn't available, with no indication — compounds H7 (cookies) and the
|
to plaintext when DPAPI/safeStorage isn't available, with no indication — compounds H7 (cookies) and the
|
||||||
backup-cleartext issue (M22). **Fix:** warn (or refuse to store) when encryption is unavailable.
|
backup-cleartext issue (M22). **Fix:** warn (or refuse to store) when encryption is unavailable.
|
||||||
- [ ] **R8 — Pretty-printed JSON for large stores.** `JSON.stringify(value, null, 2)` inflates size/write
|
- [x] **R8 — Pretty-printed JSON for large stores.** `JSON.stringify(value, null, 2)` inflates size/write
|
||||||
time for the potentially-20k-item media store (ties R3). **Fix:** compact JSON for the large stores.
|
time for the potentially-20k-item media store (ties R3). **Fix:** compact JSON for the large stores.
|
||||||
- [ ] **R9 — Clock-skew sensitivity.** `shouldAutoCheckYtdlp` and the scheduled-download promoter compare
|
- [ ] **R9 — Clock-skew sensitivity.** `shouldAutoCheckYtdlp` and the scheduled-download promoter compare
|
||||||
`Date.now()`; a backward system-clock correction can skip or double-fire a check/schedule. Minor; note for awareness.
|
`Date.now()`; a backward system-clock correction can skip or double-fire a check/schedule. Minor; note for awareness.
|
||||||
|
|||||||
+18
-13
@@ -7,13 +7,13 @@ import { readFileSync, writeFileSync, renameSync, existsSync, copyFileSync } fro
|
|||||||
* error log, templates, sources, media-items). Replaces the per-module
|
* error log, templates, sources, media-items). Replaces the per-module
|
||||||
* read/parse/write copies (SIMP1) and fixes three data-safety blockers:
|
* read/parse/write copies (SIMP1) and fixes three data-safety blockers:
|
||||||
*
|
*
|
||||||
* - R1 — atomic writes: write a temp file then `rename` over the target, so a
|
* - R1 -- atomic writes: write a temp file then `rename` over the target, so a
|
||||||
* crash / power-cut mid-write can never truncate the real file. (`electron-store`,
|
* crash / power-cut mid-write can never truncate the real file. (`electron-store`,
|
||||||
* used for settings, is already atomic; these stores were not.)
|
* used for settings, is already atomic; these stores were not.)
|
||||||
* - R2 — corruption is no longer silent data loss: on a parse error the bad file
|
* - R2 -- corruption is no longer silent data loss: on a parse error the bad file
|
||||||
* is copied aside (`<file>.corrupt-<ts>`) before we fall back to empty, so a
|
* is copied aside (`<file>.corrupt-<ts>`) before we fall back to empty, so a
|
||||||
* single bad byte can't wipe the user's history/sources from their perspective.
|
* single bad byte can't wipe the user's history/sources from their perspective.
|
||||||
* - R3 — an in-memory cache + debounced batched writes: the previous code
|
* - R3 -- an in-memory cache + debounced batched writes: the previous code
|
||||||
* re-read + re-parsed the entire (multi-MB) file and rewrote it synchronously
|
* re-read + re-parsed the entire (multi-MB) file and rewrote it synchronously
|
||||||
* on *every* download completion (≈O(n²) over a large channel, blocking the
|
* on *every* download completion (≈O(n²) over a large channel, blocking the
|
||||||
* main thread). Reads now hit the cache; writes coalesce into one atomic flush.
|
* main thread). Reads now hit the cache; writes coalesce into one atomic flush.
|
||||||
@@ -44,22 +44,24 @@ export function readJsonArraySafe<T>(path: string, isValid: (o: unknown) => o is
|
|||||||
try {
|
try {
|
||||||
copyFileSync(path, `${path}.corrupt-${Date.now()}`)
|
copyFileSync(path, `${path}.corrupt-${Date.now()}`)
|
||||||
} catch {
|
} catch {
|
||||||
/* best-effort — if we can't back it up, still don't crash the read */
|
/* best-effort -- if we can't back it up, still don't crash the read */
|
||||||
}
|
}
|
||||||
return []
|
return []
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Atomically write `value` as pretty JSON to `path`: serialise to `<path>.tmp`,
|
* Atomically write `value` as JSON to `path`: serialise to `<path>.tmp`, then
|
||||||
* then `rename` it over the target (an atomic operation on the same volume), so a
|
* `rename` it over the target (an atomic operation on the same volume), so a
|
||||||
* crash mid-write leaves either the old file or the new one — never a truncated
|
* crash mid-write leaves either the old file or the new one -- never a truncated
|
||||||
* one (R1). Best-effort: a read-only data dir just means nothing is persisted.
|
* one (R1). `pretty` indents for human-readable files; pass false for large
|
||||||
|
* machine-only stores to avoid inflating size/write time (R8). Best-effort: a
|
||||||
|
* read-only data dir just means nothing is persisted.
|
||||||
*/
|
*/
|
||||||
export function writeJsonAtomic(path: string, value: unknown): void {
|
export function writeJsonAtomic(path: string, value: unknown, pretty = true): void {
|
||||||
const tmp = `${path}.tmp`
|
const tmp = `${path}.tmp`
|
||||||
try {
|
try {
|
||||||
writeFileSync(tmp, JSON.stringify(value, null, 2))
|
writeFileSync(tmp, pretty ? JSON.stringify(value, null, 2) : JSON.stringify(value))
|
||||||
renameSync(tmp, path)
|
renameSync(tmp, path)
|
||||||
} catch {
|
} catch {
|
||||||
/* best-effort; e.g. a read-only data dir */
|
/* best-effort; e.g. a read-only data dir */
|
||||||
@@ -86,7 +88,10 @@ const registry: JsonStore<unknown>[] = []
|
|||||||
export function createJsonStore<T>(
|
export function createJsonStore<T>(
|
||||||
filename: string,
|
filename: string,
|
||||||
isValid: (o: unknown) => o is T,
|
isValid: (o: unknown) => o is T,
|
||||||
cap: number
|
cap: number,
|
||||||
|
// Pretty-print by default (small, hand-inspectable files); pass false for a
|
||||||
|
// large machine-only store to keep writes compact (R8).
|
||||||
|
pretty = true
|
||||||
): JsonStore<T> {
|
): JsonStore<T> {
|
||||||
let cache: T[] | null = null
|
let cache: T[] | null = null
|
||||||
let timer: ReturnType<typeof setTimeout> | null = null
|
let timer: ReturnType<typeof setTimeout> | null = null
|
||||||
@@ -106,7 +111,7 @@ export function createJsonStore<T>(
|
|||||||
}
|
}
|
||||||
if (!dirty) return
|
if (!dirty) return
|
||||||
dirty = false
|
dirty = false
|
||||||
writeJsonAtomic(filePath(), cache ?? [])
|
writeJsonAtomic(filePath(), cache ?? [], pretty)
|
||||||
}
|
}
|
||||||
|
|
||||||
function write(items: T[]): T[] {
|
function write(items: T[]): T[] {
|
||||||
@@ -124,7 +129,7 @@ export function createJsonStore<T>(
|
|||||||
return store
|
return store
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Synchronously flush every store's pending write — call on app quit. */
|
/** Synchronously flush every store's pending write -- call on app quit. */
|
||||||
export function flushAllStores(): void {
|
export function flushAllStores(): void {
|
||||||
for (const s of registry) s.flush()
|
for (const s of registry) s.flush()
|
||||||
}
|
}
|
||||||
|
|||||||
+3
-1
@@ -27,7 +27,9 @@ const MAX_ITEMS = 20000
|
|||||||
// rewrite the whole (≤MAX_ITEMS) file on every completion; now reads hit the
|
// rewrite the whole (≤MAX_ITEMS) file on every completion; now reads hit the
|
||||||
// cache and writes are batched. Sources are few, so that store is uncapped.
|
// cache and writes are batched. Sources are few, so that store is uncapped.
|
||||||
const sourcesStore = createJsonStore('sources.json', isValidSource, Infinity)
|
const sourcesStore = createJsonStore('sources.json', isValidSource, Infinity)
|
||||||
const itemsStore = createJsonStore('media-items.json', isValidMediaItem, MAX_ITEMS)
|
// Compact (not pretty) -- this store can hold up to MAX_ITEMS rows; indentation
|
||||||
|
// would needlessly inflate its size and write time (R8).
|
||||||
|
const itemsStore = createJsonStore('media-items.json', isValidMediaItem, MAX_ITEMS, false)
|
||||||
|
|
||||||
// --- Sources ----------------------------------------------------------------
|
// --- Sources ----------------------------------------------------------------
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user