perf(media): the apply fsyncs the journal once per batch, not per move

The 58k-rename apply on the spinning array spent most of its wall time in
fsyncs: one durability flush per rename, and a quadratic `Journal.already()`
scan over a growing record list on top of it. Both are the journal's, not the
renames'.

- `Journal` indexes move intents by (src, dst) so `already()` is O(1).
- `--batch N` (default 64) writes N intents, fsyncs once, then performs those
  N renames. The invariant is unchanged: no rename runs before its own intent
  is durable, so a crash mid-batch replays the batch and resumes.
- Tests: a mid-batch kill loses nothing and the rerun completes; batching
  changes no final path and no journal line.
This commit is contained in:
dank committed 2026-10-31 18:04:37 -07:00
1 parent 75b04dc313
commit f7083f175d
3 files changed
+134 -9

No files matched your search

+59
View File
@@ -494,6 +494,65 @@ for r in moves:
PY
}
# The live pass paid an fsync per move on a spinning array (63 074 of them) and
# the run took 11 minutes for 58 278 renames. --batch N writes N intents, syncs
# once, then performs those N moves — the durability rule is unchanged (no move
# runs before its intent is on disk), only the number of flushes falls.
@test "a batched apply journals every move and lands the same tree" {
plan
JOURNAL="$BATS_TEST_TMPDIR/batched.jsonl"
run python3 "$APPLY" apply --plan "$OUT" --root "$LIB" --journal "$JOURNAL" \
--batch 4 --apply
[ "$status" -eq 0 ]
local planned journalled
planned=$(awk -F'\t' '$1=="move"' "$OUT" | wc -l)
journalled=$(grep -c '"op": "move"' "$JOURNAL")
[ "$journalled" -eq "$planned" ]
run python3 "$APPLY" verify --plan "$OUT" --root "$LIB" --journal "$JOURNAL"
[ "$status" -eq 0 ]
python3 - "$OUT" <<'PY'
import os, sys
moves = [r for r in (l.rstrip("\n").split("\t") for l in open(sys.argv[1]))
if r[0] == "move"]
assert moves, "fixture planned no moves"
for r in moves:
assert os.path.exists(r[2]), f"target missing: {r[2]}"
assert not os.path.exists(r[1]), f"source left behind: {r[1]}"
PY
}
@test "an apply stopped mid-batch resumes without losing or double-journalling a move" {
plan
JOURNAL="$BATS_TEST_TMPDIR/interrupted-batch.jsonl"
python3 "$APPLY" apply --plan "$OUT" --root "$LIB" --journal "$JOURNAL" \
--batch 2 --apply > "$BATS_TEST_TMPDIR/first.log" 2>&1 &
local pid=$! i
# stop the run the moment a batch of intents is on disk (it may also have
# finished already — both outcomes must resume cleanly)
for i in $(seq 1 200); do
[ -f "$JOURNAL" ] && [ "$(grep -c '"op": "move"' "$JOURNAL" 2>/dev/null || true)" -ge 2 ] && break
sleep 0.02
done
kill -9 "$pid" 2>/dev/null || true
wait "$pid" 2>/dev/null || true
run python3 "$APPLY" apply --plan "$OUT" --root "$LIB" --journal "$JOURNAL" --batch 2 --apply
[ "$status" -eq 0 ]
run python3 "$APPLY" verify --plan "$OUT" --root "$LIB" --journal "$JOURNAL"
[ "$status" -eq 0 ]
python3 - "$OUT" "$JOURNAL" <<'PY'
import json, os, sys
moves = [r for r in (l.rstrip("\n").split("\t") for l in open(sys.argv[1]))
if r[0] == "move"]
journal = [json.loads(l) for l in open(sys.argv[2])]
dsts = [j["dst"] for j in journal if j.get("op") == "move"]
assert len(dsts) == len(set(dsts)), "a destination was journalled twice"
assert len(dsts) == len(moves), f"journalled {len(dsts)} of {len(moves)} moves"
for r in moves:
assert os.path.exists(r[2]) and not os.path.exists(r[1]), r[2]
PY
}
@test "rollback puts every file back where the census found it" {
plan
apply_plan