perf(media): the apply fsyncs the journal once per batch, not per move
The 58k-rename apply on the spinning array spent most of its wall time in fsyncs: one durability flush per rename, and a quadratic `Journal.already()` scan over a growing record list on top of it. Both are the journal's, not the renames'. - `Journal` indexes move intents by (src, dst) so `already()` is O(1). - `--batch N` (default 64) writes N intents, fsyncs once, then performs those N renames. The invariant is unchanged: no rename runs before its own intent is durable, so a crash mid-batch replays the batch and resumes. - Tests: a mid-batch kill loses nothing and the rerun completes; batching changes no final path and no journal line.
This commit is contained in:
1 parent
75b04dc313
commit
f7083f175d
3 files changed
+134
-9
No files matched your search
@@ -494,6 +494,65 @@ for r in moves:
|
||||
PY
|
||||
}
|
||||
|
||||
# The live pass paid an fsync per move on a spinning array (63 074 of them) and
|
||||
# the run took 11 minutes for 58 278 renames. --batch N writes N intents, syncs
|
||||
# once, then performs those N moves — the durability rule is unchanged (no move
|
||||
# runs before its intent is on disk), only the number of flushes falls.
|
||||
@test "a batched apply journals every move and lands the same tree" {
|
||||
plan
|
||||
JOURNAL="$BATS_TEST_TMPDIR/batched.jsonl"
|
||||
run python3 "$APPLY" apply --plan "$OUT" --root "$LIB" --journal "$JOURNAL" \
|
||||
--batch 4 --apply
|
||||
[ "$status" -eq 0 ]
|
||||
local planned journalled
|
||||
planned=$(awk -F'\t' '$1=="move"' "$OUT" | wc -l)
|
||||
journalled=$(grep -c '"op": "move"' "$JOURNAL")
|
||||
[ "$journalled" -eq "$planned" ]
|
||||
run python3 "$APPLY" verify --plan "$OUT" --root "$LIB" --journal "$JOURNAL"
|
||||
[ "$status" -eq 0 ]
|
||||
python3 - "$OUT" <<'PY'
|
||||
import os, sys
|
||||
moves = [r for r in (l.rstrip("\n").split("\t") for l in open(sys.argv[1]))
|
||||
if r[0] == "move"]
|
||||
assert moves, "fixture planned no moves"
|
||||
for r in moves:
|
||||
assert os.path.exists(r[2]), f"target missing: {r[2]}"
|
||||
assert not os.path.exists(r[1]), f"source left behind: {r[1]}"
|
||||
PY
|
||||
}
|
||||
|
||||
@test "an apply stopped mid-batch resumes without losing or double-journalling a move" {
|
||||
plan
|
||||
JOURNAL="$BATS_TEST_TMPDIR/interrupted-batch.jsonl"
|
||||
python3 "$APPLY" apply --plan "$OUT" --root "$LIB" --journal "$JOURNAL" \
|
||||
--batch 2 --apply > "$BATS_TEST_TMPDIR/first.log" 2>&1 &
|
||||
local pid=$! i
|
||||
# stop the run the moment a batch of intents is on disk (it may also have
|
||||
# finished already — both outcomes must resume cleanly)
|
||||
for i in $(seq 1 200); do
|
||||
[ -f "$JOURNAL" ] && [ "$(grep -c '"op": "move"' "$JOURNAL" 2>/dev/null || true)" -ge 2 ] && break
|
||||
sleep 0.02
|
||||
done
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
wait "$pid" 2>/dev/null || true
|
||||
|
||||
run python3 "$APPLY" apply --plan "$OUT" --root "$LIB" --journal "$JOURNAL" --batch 2 --apply
|
||||
[ "$status" -eq 0 ]
|
||||
run python3 "$APPLY" verify --plan "$OUT" --root "$LIB" --journal "$JOURNAL"
|
||||
[ "$status" -eq 0 ]
|
||||
python3 - "$OUT" "$JOURNAL" <<'PY'
|
||||
import json, os, sys
|
||||
moves = [r for r in (l.rstrip("\n").split("\t") for l in open(sys.argv[1]))
|
||||
if r[0] == "move"]
|
||||
journal = [json.loads(l) for l in open(sys.argv[2])]
|
||||
dsts = [j["dst"] for j in journal if j.get("op") == "move"]
|
||||
assert len(dsts) == len(set(dsts)), "a destination was journalled twice"
|
||||
assert len(dsts) == len(moves), f"journalled {len(dsts)} of {len(moves)} moves"
|
||||
for r in moves:
|
||||
assert os.path.exists(r[2]) and not os.path.exists(r[1]), r[2]
|
||||
PY
|
||||
}
|
||||
|
||||
@test "rollback puts every file back where the census found it" {
|
||||
plan
|
||||
apply_plan
|
||||
|
||||
Reference in new issue
Block a user