Frank
35ceae6d7b
`ssh host 'cmd | tail -f log | sed -n /ready/q'` reported success on jobs that never finished, three structural ways: the broker aborts the long connection, a readiness pattern can match a log header printed before the work starts, and `sed -n /x/q` exits the instant it matches — tearing down the very command it was watching. The replacement runs the job detached on the host and writes NAME.out, NAME.rc, then NAME.done LAST, so status is a cheap read of files over a short-lived connection. That survives the connection, the restart, and the job that restarts the endpoint the agent itself is running on. Black-box tests pin the sentinel ordering (rc before done), that a finished-but- failed job is DONE rc=N with the report still exiting 0, and that a name with no state is never reported DONE. Refs #53
116 lines
4.5 KiB
Bash
Executable File
116 lines
4.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# scripts/remote-job.sh — a detached job whose result outlives the session that started it
|
|
#
|
|
# Why this exists rather than `ssh host 'cmd | tail -f | sed -n /ready/q'`, which lied about success
|
|
# repeatedly, for three structural reasons — all of them the pipe's fault: the broker aborts
|
|
# long-lived ssh; a readiness pattern can match a log *header* before the work has begun; and
|
|
# `sed -n /x/q` exits the instant it matches, so the supervisor reports green on a job that never
|
|
# finished.
|
|
#
|
|
# Here the job runs detached on the box it is invoked on (over ssh, or locally). It writes NAME.out,
|
|
# then NAME.rc, then touches NAME.done LAST. Status is then a cheap read of a directory: no
|
|
# long-lived connection to lose, and the answer survives the session, the terminal, and the agent —
|
|
# including the case where the job restarts the very service the agent is running on.
|
|
#
|
|
# remote-job.sh start NAME [--] CMD... launch detached, return at once
|
|
# remote-job.sh status [NAME] one line per job
|
|
# remote-job.sh wait NAME [SECONDS] [N] block until DONE or timeout, then tail N lines
|
|
# remote-job.sh out NAME [LINES] print the tail of a job's output
|
|
# remote-job.sh tail [NAME] follow one job, or every job at once
|
|
# remote-job.sh watch refreshing status, for a spare pane
|
|
#
|
|
# Job directory: $REMOTE_JOBS_DIR, default $HOME/jobs. Ship this file to the host and run it there;
|
|
# nothing about it is box-specific, which is the point.
|
|
#
|
|
# Exit status: 0 when every job reported is finished · 1 when something is unfinished or a wait
|
|
# timed out · 2 usage. A job that FAILED still counts as finished for `status`: "is it done" and
|
|
# "did it work" are different questions, and the second one is the rc= column.
|
|
|
|
set -uo pipefail
|
|
|
|
JOBS="${REMOTE_JOBS_DIR:-$HOME/jobs}"
|
|
mkdir -p "$JOBS"
|
|
|
|
case "${1:-}" in
|
|
start)
|
|
[[ $# -ge 2 ]] || { echo "usage: remote-job.sh start NAME [--] CMD..." >&2; exit 2; }
|
|
name="$2"; shift 2
|
|
[[ "${1:-}" == "--" ]] && shift
|
|
[[ $# -ge 1 ]] || { echo "remote-job: start needs a command for '$name'" >&2; exit 2; }
|
|
# Clear the previous run entirely: a stale .out would make the new run's line count a lie.
|
|
rm -f "$JOBS/$name.out" "$JOBS/$name.rc" "$JOBS/$name.done"
|
|
# setsid leaves the invoking process group, so a dropped ssh or a killed agent takes nothing
|
|
# down with it. rc lands BEFORE the sentinel: a waiter must never see DONE with no exit code.
|
|
JOBS="$JOBS" setsid nohup bash -c '
|
|
name="$1"; shift
|
|
"$@" > "$JOBS/$name.out" 2>&1
|
|
echo $? > "$JOBS/$name.rc"
|
|
touch "$JOBS/$name.done"
|
|
' _ "$name" "$@" < /dev/null > /dev/null 2>&1 &
|
|
printf 'started %s\n' "$name"
|
|
;;
|
|
|
|
status)
|
|
if [[ -n "${2:-}" ]]; then
|
|
names="$2"
|
|
else
|
|
names=$(cd "$JOBS" && ls *.out 2>/dev/null | sed 's/\.out$//' | sort)
|
|
fi
|
|
pending=0
|
|
for n in $names; do
|
|
if [[ -f "$JOBS/$n.done" ]]; then
|
|
printf ' %-20s DONE rc=%s %s lines\n' "$n" \
|
|
"$(cat "$JOBS/$n.rc" 2>/dev/null)" "$(wc -l < "$JOBS/$n.out" 2>/dev/null | tr -d ' ')"
|
|
elif [[ -f "$JOBS/$n.out" ]]; then
|
|
printf ' %-20s RUN %s lines so far\n' "$n" "$(wc -l < "$JOBS/$n.out" 2>/dev/null | tr -d ' ')"
|
|
pending=1
|
|
else
|
|
printf ' %-20s PENDING (no output yet)\n' "$n"
|
|
pending=1
|
|
fi
|
|
done
|
|
if [[ -z "$names" ]]; then echo " (no jobs)"; exit 0; fi
|
|
exit "$pending"
|
|
;;
|
|
|
|
wait)
|
|
[[ $# -ge 2 ]] || { echo "usage: remote-job.sh wait NAME [SECONDS] [TAIL]" >&2; exit 2; }
|
|
name="$2"; limit="${3:-3600}"; lines="${4:-25}"; waited=0
|
|
while [[ ! -e "$JOBS/$name.done" ]] && (( waited < limit )); do
|
|
sleep 1; waited=$((waited + 1))
|
|
done
|
|
if [[ -e "$JOBS/$name.done" ]]; then
|
|
printf 'DONE %s rc=%s\n' "$name" "$(cat "$JOBS/$name.rc" 2>/dev/null)"; rc=0
|
|
else
|
|
printf 'TIMEOUT %s after %ss\n' "$name" "$waited"; rc=1
|
|
fi
|
|
tail -n "$lines" "$JOBS/$name.out" 2>/dev/null
|
|
exit "$rc"
|
|
;;
|
|
|
|
out)
|
|
[[ $# -ge 2 ]] || { echo "usage: remote-job.sh out NAME [LINES]" >&2; exit 2; }
|
|
tail -n "${3:-40}" "$JOBS/$2.out" 2>/dev/null
|
|
;;
|
|
|
|
tail)
|
|
# Multiplexed follow: no NAME means every job at once, so one pane shows the whole harness.
|
|
if [[ -n "${2:-}" ]]; then exec tail -f "$JOBS/$2.out"; fi
|
|
exec tail -f "$JOBS"/*.out
|
|
;;
|
|
|
|
watch)
|
|
while true; do
|
|
clear 2>/dev/null || true
|
|
date '+%H:%M:%S'
|
|
"$0" status
|
|
sleep 5
|
|
done
|
|
;;
|
|
|
|
*)
|
|
echo "usage: remote-job.sh {start|status|wait|out|tail|watch} ..." >&2
|
|
exit 2
|
|
;;
|
|
esac
|