command-center/tests/remote-job.bats

#!/usr/bin/env bats
# tests/remote-job.bats — black-box tests for scripts/remote-job.sh (Refs #53)
#
# Contract: a job runs detached, writes its output, then its exit code, then touches .done LAST.
# Status is then a cheap independent read that survives the session that started the job — the
# whole point is replacing `ssh host 'tail -f | sed'`, which reported success on jobs that had not
# finished. Exit status answers "is everything finished"; the rc= field answers "did it work".
#
# Jobs really run here, against a temp job dir. No fake filesystem state is pre-seeded, because a
# fixture that pre-creates .done proves nothing about the code that is supposed to create it.

setup() {
  TEST_TMP="$(mktemp -d)"
  SCRIPT="$BATS_TEST_DIRNAME/../scripts/remote-job.sh"
  export REMOTE_JOBS_DIR="$TEST_TMP/jobs"
}

teardown() {
  # Jobs are detached on purpose; leave nothing behind that could outlive the test run.
  pkill -P 1 -f "$TEST_TMP" 2>/dev/null || true
  rm -rf "$TEST_TMP"
}

# until_done NAME SECONDS — poll instead of sleeping a fixed amount, so the suite is neither slow
# nor flaky on a loaded machine.
until_done() {
  local name="$1" limit="${2:-10}" waited=0
  while [[ ! -e "$REMOTE_JOBS_DIR/$name.done" && $waited -lt $((limit * 10)) ]]; do
    sleep 0.1; waited=$((waited + 1))
  done
  [[ -e "$REMOTE_JOBS_DIR/$name.done" ]]
}

@test "start returns immediately while the job is still running" {
  run timeout 10 "$SCRIPT" start slow -- sleep 2
  [ "$status" -eq 0 ]
  [[ "$output" == "started slow" ]]
  # A 2-second job must not have blocked a start that is supposed to return at once, and while it
  # is still in flight the report must say unfinished — never DONE.
  run "$SCRIPT" status slow
  [[ "$output" == *RUN* || "$output" == *PENDING* ]]
  [ "$status" -eq 1 ]
}

@test "a successful job is reported DONE with its output" {
  "$SCRIPT" start ok -- bash -c 'echo marker-4711'
  until_done ok
  run "$SCRIPT" status ok
  [[ "$output" == *"ok"*DONE*rc=0* ]]
  [ "$status" -eq 0 ]
  run "$SCRIPT" out ok
  [[ "$output" == *marker-4711* ]]
}

@test "a failing job's exit code survives into the report" {
  # This is the reason the .rc file exists: a supervisor that cannot see the failure code is the
  # bug the runner was written to fix.
  "$SCRIPT" start boom -- bash -c 'echo it failed here >&2; exit 7'
  until_done boom
  run "$SCRIPT" status boom
  [[ "$output" == *"boom"*DONE*rc=7* ]]
  run "$SCRIPT" out boom
  [[ "$output" == *"it failed here"* ]]
}

@test "the .done sentinel is never written before the exit code" {
  # If .done landed first, a waiter could read an empty rc and call the job successful.
  "$SCRIPT" start order -- bash -c 'echo x'
  until_done order
  [ -f "$REMOTE_JOBS_DIR/order.rc" ]
  [ -s "$REMOTE_JOBS_DIR/order.rc" ]
  run cat "$REMOTE_JOBS_DIR/order.rc"
  [[ "$output" =~ ^[0-9]+$ ]]
}

@test "wait prints DONE with the exit code and a tail of the output" {
  "$SCRIPT" start fin -- bash -c 'echo one; echo two'
  run "$SCRIPT" wait fin 15
  [ "$status" -eq 0 ]
  [[ "$output" == *"DONE fin rc=0"* ]]
  [[ "$output" == *two* ]]
}

@test "wait gives up on its own and says so" {
  "$SCRIPT" start never -- sleep 30
  run "$SCRIPT" wait never 1
  [ "$status" -eq 1 ]
  [[ "$output" == *TIMEOUT*never* ]]
  "$SCRIPT" status never | grep -q .   # the job is still there, unfinished
}

@test "status distinguishes running from pending from done" {
  run "$SCRIPT" status
  [[ "$output" == *"(no jobs)"* ]]
  [ "$status" -eq 0 ]

  "$SCRIPT" start later -- sleep 3
  run "$SCRIPT" status later
  [[ "$output" == *later* ]]
  [[ "$output" == *RUN* || "$output" == *PENDING* ]]
  [ "$status" -eq 1 ]   # unfinished: a supervisor must be able to branch on this
}

@test "two jobs in flight keep separate state" {
  "$SCRIPT" start a -- bash -c 'echo A; exit 0'
  "$SCRIPT" start b -- bash -c 'echo B; exit 2'
  until_done a && until_done b
  run "$SCRIPT" status
  [[ "$output" == *"a"*DONE*rc=0* ]]
  [[ "$output" == *"b"*DONE*rc=2* ]]
  run "$SCRIPT" out a
  [[ "$output" == A ]]
  run "$SCRIPT" out b
  [[ "$output" == B ]]
}

@test "out honours a line limit" {
  "$SCRIPT" start many -- bash -c 'seq 1 40'
  until_done many
  run "$SCRIPT" out many 3
  [ "$(printf '%s\n' "$output" | grep -c .)" -eq 3 ]
  [[ "$output" == *38* ]]   # the tail, not the head
}

@test "status treats a job name as a name, not a pattern" {
  "$SCRIPT" start 'x*' -- bash -c 'echo star'
  until_done 'x*'
  touch "$TEST_TMP/anchor"
  # The report is read from a directory full of unrelated files on purpose: a name that is
  # expanded as a glob resolves against the CALLER's cwd, so the tool would describe `anchor`
  # as a job and never mention the one that really ran.
  run bash -c 'cd "$1" && "$2" status "$3"' _ "$TEST_TMP" "$SCRIPT" 'x*'
  [[ "$output" == "  x*"*"DONE   rc=0"* ]]
  [[ "$output" != *anchor* ]]
}

@test "tail on an empty harness and on a missing job both say so" {
  # An unexpanded `*.out` reaching tail is reported as a missing FILE, which reads as a broken
  # script; the honest answer is that there is nothing to follow yet.
  run timeout 5 "$SCRIPT" tail
  [ "$status" -ne 0 ]
  [[ "$output" == *"no jobs"* ]]
  [[ "$output" != *".out"* ]]
  run timeout 5 "$SCRIPT" tail nosuchjob
  [ "$status" -ne 0 ]
  [[ "$output" == *"nosuchjob"* ]]
  [[ "$output" != *"No such file"* ]]
}

@test "an unknown verb is a usage error" {
  run "$SCRIPT" frobnicate
  [ "$status" -eq 2 ]
  [[ "$output" == *usage* ]]
}

@test "the job directory is created on first use" {
  [ ! -d "$REMOTE_JOBS_DIR" ]
  "$SCRIPT" status
  [ -d "$REMOTE_JOBS_DIR" ]
}