Fix R8: compact JSON for the large media-items store
writeJsonAtomic always pretty-printed (JSON.stringify(value, null, 2)), which needlessly inflates size and write time for the media-items store (up to MAX_ITEMS=20000 rows). Added an optional `pretty` flag (default true, so history/sources/templates/errorlog stay hand-inspectable) threaded through createJsonStore, and set it false for media-items.json. typecheck + 242 tests + eslint green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+18
-13
@@ -7,13 +7,13 @@ import { readFileSync, writeFileSync, renameSync, existsSync, copyFileSync } fro
|
||||
* error log, templates, sources, media-items). Replaces the per-module
|
||||
* read/parse/write copies (SIMP1) and fixes three data-safety blockers:
|
||||
*
|
||||
* - R1 — atomic writes: write a temp file then `rename` over the target, so a
|
||||
* - R1 -- atomic writes: write a temp file then `rename` over the target, so a
|
||||
* crash / power-cut mid-write can never truncate the real file. (`electron-store`,
|
||||
* used for settings, is already atomic; these stores were not.)
|
||||
* - R2 — corruption is no longer silent data loss: on a parse error the bad file
|
||||
* - R2 -- corruption is no longer silent data loss: on a parse error the bad file
|
||||
* is copied aside (`<file>.corrupt-<ts>`) before we fall back to empty, so a
|
||||
* single bad byte can't wipe the user's history/sources from their perspective.
|
||||
* - R3 — an in-memory cache + debounced batched writes: the previous code
|
||||
* - R3 -- an in-memory cache + debounced batched writes: the previous code
|
||||
* re-read + re-parsed the entire (multi-MB) file and rewrote it synchronously
|
||||
* on *every* download completion (≈O(n²) over a large channel, blocking the
|
||||
* main thread). Reads now hit the cache; writes coalesce into one atomic flush.
|
||||
@@ -44,22 +44,24 @@ export function readJsonArraySafe<T>(path: string, isValid: (o: unknown) => o is
|
||||
try {
|
||||
copyFileSync(path, `${path}.corrupt-${Date.now()}`)
|
||||
} catch {
|
||||
/* best-effort — if we can't back it up, still don't crash the read */
|
||||
/* best-effort -- if we can't back it up, still don't crash the read */
|
||||
}
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Atomically write `value` as pretty JSON to `path`: serialise to `<path>.tmp`,
|
||||
* then `rename` it over the target (an atomic operation on the same volume), so a
|
||||
* crash mid-write leaves either the old file or the new one — never a truncated
|
||||
* one (R1). Best-effort: a read-only data dir just means nothing is persisted.
|
||||
* Atomically write `value` as JSON to `path`: serialise to `<path>.tmp`, then
|
||||
* `rename` it over the target (an atomic operation on the same volume), so a
|
||||
* crash mid-write leaves either the old file or the new one -- never a truncated
|
||||
* one (R1). `pretty` indents for human-readable files; pass false for large
|
||||
* machine-only stores to avoid inflating size/write time (R8). Best-effort: a
|
||||
* read-only data dir just means nothing is persisted.
|
||||
*/
|
||||
export function writeJsonAtomic(path: string, value: unknown): void {
|
||||
export function writeJsonAtomic(path: string, value: unknown, pretty = true): void {
|
||||
const tmp = `${path}.tmp`
|
||||
try {
|
||||
writeFileSync(tmp, JSON.stringify(value, null, 2))
|
||||
writeFileSync(tmp, pretty ? JSON.stringify(value, null, 2) : JSON.stringify(value))
|
||||
renameSync(tmp, path)
|
||||
} catch {
|
||||
/* best-effort; e.g. a read-only data dir */
|
||||
@@ -86,7 +88,10 @@ const registry: JsonStore<unknown>[] = []
|
||||
export function createJsonStore<T>(
|
||||
filename: string,
|
||||
isValid: (o: unknown) => o is T,
|
||||
cap: number
|
||||
cap: number,
|
||||
// Pretty-print by default (small, hand-inspectable files); pass false for a
|
||||
// large machine-only store to keep writes compact (R8).
|
||||
pretty = true
|
||||
): JsonStore<T> {
|
||||
let cache: T[] | null = null
|
||||
let timer: ReturnType<typeof setTimeout> | null = null
|
||||
@@ -106,7 +111,7 @@ export function createJsonStore<T>(
|
||||
}
|
||||
if (!dirty) return
|
||||
dirty = false
|
||||
writeJsonAtomic(filePath(), cache ?? [])
|
||||
writeJsonAtomic(filePath(), cache ?? [], pretty)
|
||||
}
|
||||
|
||||
function write(items: T[]): T[] {
|
||||
@@ -124,7 +129,7 @@ export function createJsonStore<T>(
|
||||
return store
|
||||
}
|
||||
|
||||
/** Synchronously flush every store's pending write — call on app quit. */
|
||||
/** Synchronously flush every store's pending write -- call on app quit. */
|
||||
export function flushAllStores(): void {
|
||||
for (const s of registry) s.flush()
|
||||
}
|
||||
|
||||
+3
-1
@@ -27,7 +27,9 @@ const MAX_ITEMS = 20000
|
||||
// rewrite the whole (≤MAX_ITEMS) file on every completion; now reads hit the
|
||||
// cache and writes are batched. Sources are few, so that store is uncapped.
|
||||
const sourcesStore = createJsonStore('sources.json', isValidSource, Infinity)
|
||||
const itemsStore = createJsonStore('media-items.json', isValidMediaItem, MAX_ITEMS)
|
||||
// Compact (not pretty) -- this store can hold up to MAX_ITEMS rows; indentation
|
||||
// would needlessly inflate its size and write time (R8).
|
||||
const itemsStore = createJsonStore('media-items.json', isValidMediaItem, MAX_ITEMS, false)
|
||||
|
||||
// --- Sources ----------------------------------------------------------------
|
||||
|
||||
|
||||
Reference in New Issue
Block a user