Frank
35ceae6d7b
`ssh host 'cmd | tail -f log | sed -n /ready/q'` reported success on jobs that never finished, three structural ways: the broker aborts the long connection, a readiness pattern can match a log header printed before the work starts, and `sed -n /x/q` exits the instant it matches — tearing down the very command it was watching. The replacement runs the job detached on the host and writes NAME.out, NAME.rc, then NAME.done LAST, so status is a cheap read of files over a short-lived connection. That survives the connection, the restart, and the job that restarts the endpoint the agent itself is running on. Black-box tests pin the sentinel ordering (rc before done), that a finished-but- failed job is DONE rc=N with the report still exiting 0, and that a name with no state is never reported DONE. Refs #53
136 lines
4.5 KiB
Bash
Executable File
136 lines
4.5 KiB
Bash
Executable File
#!/usr/bin/env bats
|
|
# tests/remote-job.bats — black-box tests for scripts/remote-job.sh (Refs #53)
|
|
#
|
|
# Contract: a job runs detached, writes its output, then its exit code, then touches .done LAST.
|
|
# Status is then a cheap independent read that survives the session that started the job — the
|
|
# whole point is replacing `ssh host 'tail -f | sed'`, which reported success on jobs that had not
|
|
# finished. Exit status answers "is everything finished"; the rc= field answers "did it work".
|
|
#
|
|
# Jobs really run here, against a temp job dir. No fake filesystem state is pre-seeded, because a
|
|
# fixture that pre-creates .done proves nothing about the code that is supposed to create it.
|
|
|
|
setup() {
|
|
TEST_TMP="$(mktemp -d)"
|
|
SCRIPT="$BATS_TEST_DIRNAME/../scripts/remote-job.sh"
|
|
export REMOTE_JOBS_DIR="$TEST_TMP/jobs"
|
|
}
|
|
|
|
teardown() {
|
|
# Jobs are detached on purpose; leave nothing behind that could outlive the test run.
|
|
pkill -P 1 -f "$TEST_TMP" 2>/dev/null || true
|
|
rm -rf "$TEST_TMP"
|
|
}
|
|
|
|
# until_done NAME SECONDS — poll instead of sleeping a fixed amount, so the suite is neither slow
|
|
# nor flaky on a loaded machine.
|
|
until_done() {
|
|
local name="$1" limit="${2:-10}" waited=0
|
|
while [[ ! -e "$REMOTE_JOBS_DIR/$name.done" && $waited -lt $((limit * 10)) ]]; do
|
|
sleep 0.1; waited=$((waited + 1))
|
|
done
|
|
[[ -e "$REMOTE_JOBS_DIR/$name.done" ]]
|
|
}
|
|
|
|
@test "start returns immediately while the job is still running" {
|
|
run timeout 10 "$SCRIPT" start slow -- sleep 2
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == "started slow" ]]
|
|
# A 2-second job must not have blocked a start that is supposed to return at once, and while it
|
|
# is still in flight the report must say unfinished — never DONE.
|
|
run "$SCRIPT" status slow
|
|
[[ "$output" == *RUN* || "$output" == *PENDING* ]]
|
|
[ "$status" -eq 1 ]
|
|
}
|
|
|
|
@test "a successful job is reported DONE with its output" {
|
|
"$SCRIPT" start ok -- bash -c 'echo marker-4711'
|
|
until_done ok
|
|
run "$SCRIPT" status ok
|
|
[[ "$output" == *"ok"*DONE*rc=0* ]]
|
|
[ "$status" -eq 0 ]
|
|
run "$SCRIPT" out ok
|
|
[[ "$output" == *marker-4711* ]]
|
|
}
|
|
|
|
@test "a failing job's exit code survives into the report" {
|
|
# This is the reason the .rc file exists: a supervisor that cannot see the failure code is the
|
|
# bug the runner was written to fix.
|
|
"$SCRIPT" start boom -- bash -c 'echo it failed here >&2; exit 7'
|
|
until_done boom
|
|
run "$SCRIPT" status boom
|
|
[[ "$output" == *"boom"*DONE*rc=7* ]]
|
|
run "$SCRIPT" out boom
|
|
[[ "$output" == *"it failed here"* ]]
|
|
}
|
|
|
|
@test "the .done sentinel is never written before the exit code" {
|
|
# If .done landed first, a waiter could read an empty rc and call the job successful.
|
|
"$SCRIPT" start order -- bash -c 'echo x'
|
|
until_done order
|
|
[ -f "$REMOTE_JOBS_DIR/order.rc" ]
|
|
[ -s "$REMOTE_JOBS_DIR/order.rc" ]
|
|
run cat "$REMOTE_JOBS_DIR/order.rc"
|
|
[[ "$output" =~ ^[0-9]+$ ]]
|
|
}
|
|
|
|
@test "wait prints DONE with the exit code and a tail of the output" {
|
|
"$SCRIPT" start fin -- bash -c 'echo one; echo two'
|
|
run "$SCRIPT" wait fin 15
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *"DONE fin rc=0"* ]]
|
|
[[ "$output" == *two* ]]
|
|
}
|
|
|
|
@test "wait gives up on its own and says so" {
|
|
"$SCRIPT" start never -- sleep 30
|
|
run "$SCRIPT" wait never 1
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *TIMEOUT*never* ]]
|
|
"$SCRIPT" status never | grep -q . # the job is still there, unfinished
|
|
}
|
|
|
|
@test "status distinguishes running from pending from done" {
|
|
run "$SCRIPT" status
|
|
[[ "$output" == *"(no jobs)"* ]]
|
|
[ "$status" -eq 0 ]
|
|
|
|
"$SCRIPT" start later -- sleep 3
|
|
run "$SCRIPT" status later
|
|
[[ "$output" == *later* ]]
|
|
[[ "$output" == *RUN* || "$output" == *PENDING* ]]
|
|
[ "$status" -eq 1 ] # unfinished: a supervisor must be able to branch on this
|
|
}
|
|
|
|
@test "two jobs in flight keep separate state" {
|
|
"$SCRIPT" start a -- bash -c 'echo A; exit 0'
|
|
"$SCRIPT" start b -- bash -c 'echo B; exit 2'
|
|
until_done a && until_done b
|
|
run "$SCRIPT" status
|
|
[[ "$output" == *"a"*DONE*rc=0* ]]
|
|
[[ "$output" == *"b"*DONE*rc=2* ]]
|
|
run "$SCRIPT" out a
|
|
[[ "$output" == A ]]
|
|
run "$SCRIPT" out b
|
|
[[ "$output" == B ]]
|
|
}
|
|
|
|
@test "out honours a line limit" {
|
|
"$SCRIPT" start many -- bash -c 'seq 1 40'
|
|
until_done many
|
|
run "$SCRIPT" out many 3
|
|
[ "$(printf '%s\n' "$output" | grep -c .)" -eq 3 ]
|
|
[[ "$output" == *38* ]] # the tail, not the head
|
|
}
|
|
|
|
@test "an unknown verb is a usage error" {
|
|
run "$SCRIPT" frobnicate
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *usage* ]]
|
|
}
|
|
|
|
@test "the job directory is created on first use" {
|
|
[ ! -d "$REMOTE_JOBS_DIR" ]
|
|
"$SCRIPT" status
|
|
[ -d "$REMOTE_JOBS_DIR" ]
|
|
}
|