command-center/scripts/remote-job.sh

#!/usr/bin/env bash
# scripts/remote-job.sh — a detached job whose result outlives the session that started it
#
# Why this exists rather than `ssh host 'cmd | tail -f | sed -n /ready/q'`, which lied about success
# repeatedly, for three structural reasons — all of them the pipe's fault: the broker aborts
# long-lived ssh; a readiness pattern can match a log *header* before the work has begun; and
# `sed -n /x/q` exits the instant it matches, so the supervisor reports green on a job that never
# finished.
#
# Here the job runs detached on the box it is invoked on (over ssh, or locally). It writes NAME.out,
# then NAME.rc, then touches NAME.done LAST. Status is then a cheap read of a directory: no
# long-lived connection to lose, and the answer survives the session, the terminal, and the agent —
# including the case where the job restarts the very service the agent is running on.
#
#   remote-job.sh start NAME [--] CMD...    launch detached, return at once
#   remote-job.sh status [NAME]             one line per job
#   remote-job.sh wait NAME [SECONDS] [N]   block until DONE or timeout, then tail N lines
#   remote-job.sh out NAME [LINES]          print the tail of a job's output
#   remote-job.sh tail [NAME]               follow one job, or every job at once
#   remote-job.sh watch                     refreshing status, for a spare pane
#
# Job directory: $REMOTE_JOBS_DIR, default $HOME/jobs. Ship this file to the host and run it there;
# nothing about it is box-specific, which is the point.
#
# Exit status: 0 when every job reported is finished · 1 when something is unfinished or a wait
# timed out · 2 usage. A job that FAILED still counts as finished for `status`: "is it done" and
# "did it work" are different questions, and the second one is the rc= column.

set -uo pipefail

JOBS="${REMOTE_JOBS_DIR:-$HOME/jobs}"
mkdir -p "$JOBS"

case "${1:-}" in
  start)
    [[ $# -ge 2 ]] || { echo "usage: remote-job.sh start NAME [--] CMD..." >&2; exit 2; }
    name="$2"; shift 2
    [[ "${1:-}" == "--" ]] && shift
    [[ $# -ge 1 ]] || { echo "remote-job: start needs a command for '$name'" >&2; exit 2; }
    # Clear the previous run entirely: a stale .out would make the new run's line count a lie.
    rm -f "$JOBS/$name.out" "$JOBS/$name.rc" "$JOBS/$name.done"
    # setsid leaves the invoking process group, so a dropped ssh or a killed agent takes nothing
    # down with it. rc lands BEFORE the sentinel: a waiter must never see DONE with no exit code.
    JOBS="$JOBS" setsid nohup bash -c '
      name="$1"; shift
      "$@" > "$JOBS/$name.out" 2>&1
      echo $? > "$JOBS/$name.rc"
      touch "$JOBS/$name.done"
    ' _ "$name" "$@" < /dev/null > /dev/null 2>&1 &
    printf 'started %s\n' "$name"
    ;;

  status)
    # Job names are data, never patterns. `for n in $names` word-split AND globbed: a job named
    # `x*` was expanded against the caller's working directory, so the report described unrelated
    # files there as jobs and never mentioned the one that ran. Splitting stays (several names in
    # one argument is the documented call shape); expansion does not.
    names=()
    if [[ -n "${2:-}" ]]; then
      read -r -a names <<< "$2"
    else
      while IFS= read -r -d '' f; do
        names+=("$(basename "$f" .out)")
      done < <(find "$JOBS" -maxdepth 1 -name '*.out' -print0 2>/dev/null | sort -z)
    fi
    if [[ ${#names[@]} -eq 0 ]]; then echo "  (no jobs)"; exit 0; fi
    pending=0
    for n in "${names[@]}"; do
      if [[ -f "$JOBS/$n.done" ]]; then
        printf '  %-20s DONE   rc=%s  %s lines\n' "$n" \
          "$(cat "$JOBS/$n.rc" 2>/dev/null)" "$(wc -l < "$JOBS/$n.out" 2>/dev/null | tr -d ' ')"
      elif [[ -f "$JOBS/$n.out" ]]; then
        printf '  %-20s RUN    %s lines so far\n' "$n" "$(wc -l < "$JOBS/$n.out" 2>/dev/null | tr -d ' ')"
        pending=1
      else
        printf '  %-20s PENDING (no output yet)\n' "$n"
        pending=1
      fi
    done
    exit "$pending"
    ;;

  wait)
    [[ $# -ge 2 ]] || { echo "usage: remote-job.sh wait NAME [SECONDS] [TAIL]" >&2; exit 2; }
    name="$2"; limit="${3:-3600}"; lines="${4:-25}"; waited=0
    while [[ ! -e "$JOBS/$name.done" ]] && (( waited < limit )); do
      sleep 1; waited=$((waited + 1))
    done
    if [[ -e "$JOBS/$name.done" ]]; then
      printf 'DONE %s rc=%s\n' "$name" "$(cat "$JOBS/$name.rc" 2>/dev/null)"; rc=0
    else
      printf 'TIMEOUT %s after %ss\n' "$name" "$waited"; rc=1
    fi
    tail -n "$lines" "$JOBS/$name.out" 2>/dev/null
    exit "$rc"
    ;;

  out)
    [[ $# -ge 2 ]] || { echo "usage: remote-job.sh out NAME [LINES]" >&2; exit 2; }
    tail -n "${3:-40}" "$JOBS/$2.out" 2>/dev/null
    ;;

  tail)
    # Multiplexed follow: no NAME means every job at once, so one pane shows the whole harness.
    # Both branches check for something to follow first: an unexpanded `*.out` handed to tail
    # reports a missing *file*, which reads as a broken tool instead of an empty harness.
    if [[ -n "${2:-}" ]]; then
      [[ -f "$JOBS/$2.out" ]] || { echo "remote-job: no job '$2' (no $JOBS/$2.out)" >&2; exit 1; }
      exec tail -f "$JOBS/$2.out"
    fi
    mapfile -d '' -t follow < <(find "$JOBS" -maxdepth 1 -name '*.out' -print0 2>/dev/null | sort -z)
    (( ${#follow[@]} )) || { echo "remote-job: no jobs in $JOBS yet" >&2; exit 1; }
    exec tail -f "${follow[@]}"
    ;;

  watch)
    while true; do
      clear 2>/dev/null || true
      date '+%H:%M:%S'
      "$0" status
      sleep 5
    done
    ;;

  *)
    echo "usage: remote-job.sh {start|status|wait|out|tail|watch} ..." >&2
    exit 2
    ;;
esac