From 2339faeaa473262e38c705a72cd3fa63a3650c45 Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 30 Sep 2026 01:05:05 -0700 Subject: [PATCH] frank-primary: fix the SIGPIPE race in both image-digest gates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The classic 'podman images | grep -q' pattern races under set -o pipefail: grep exits on its first match, podman dies SIGPIPE (141) writing the remaining RepoDigests lines, and pipefail promotes 141 to the pipeline status — the gate reads 'no match' and refuses to deploy (2026-09-30, dankserver). Same bug in the serve script (it would restart-loop the same way). Fix: capture podman's output into a variable first, grep the variable. Patched in the R9V September deploys by luck of timing. --- scripts/deploy-remote.sh | 8 +++++++- scripts/serve-frank-primary.sh | 11 ++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index d0cd3fa..19ea64d 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1237,7 +1237,13 @@ deploy_llm_primary() { echo "ERROR: frank-primary.env missing FRANK_PRIMARY_IMAGE/MODEL_DIR/MODEL_SHARDS" >&2 return 1 fi - if ! podman images --format '{{.RepoDigests}}' | grep -qF "${image#*@}"; then + # Capture podman's output BEFORE grepping: under set -o pipefail, the + # classic `podman images | grep -q` races (grep exits on match, podman + # dies SIGPIPE 141 writing the rest, pipefail calls it no-match — the + # 2026-09-30 deploy refusal). + local digests + digests="$(podman images --format '{{.RepoDigests}}')" + if ! grep -qF "${image#*@}" <<<"$digests"; then echo "ERROR: pinned runtime image absent from rootless storage: $image" >&2 echo "frank-primary: fetch it (podman pull $image)" >&2 return 1 diff --git a/scripts/serve-frank-primary.sh b/scripts/serve-frank-primary.sh index 40688d0..830a47f 100755 --- a/scripts/serve-frank-primary.sh +++ b/scripts/serve-frank-primary.sh @@ -46,11 +46,12 @@ set +a : "${FRANK_PRIMARY_SEQS:?FRANK_PRIMARY_SEQS must be set}" : "${FRANK_PRIMARY_KV_BYTES:?FRANK_PRIMARY_KV_BYTES must be set (PER RANK)}" : "${FRANK_PRIMARY_KV_DTYPE:?FRANK_PRIMARY_KV_DTYPE must be set (fp8 = q8/q8 KV)}" -: "${FRANK_PRIMARY_SPEC:?FRANK_PRIMARY_SPEC must be set (0 disables)}" -: "${FRANK_PRIMARY_UTIL:?FRANK_PRIMARY_UTIL must be set}" - -# --- gates: never serve a half-finished payload or a missing image ----------- -if ! podman images --format '{{.RepoDigests}}' | grep -qF "${FRANK_PRIMARY_IMAGE#*@}"; then +# Capture podman's output BEFORE grepping: under `set -o pipefail`, the +# classic `podman images | grep -q` pattern races — grep exits on its first +# match, podman dies SIGPIPE (141) writing the remaining lines, and pipefail +# turns the 141 into "no match" (the 2026-09-30 deploy refusal). +DIGESTS="$(podman images --format '{{.RepoDigests}}')" +if ! grep -qF "${FRANK_PRIMARY_IMAGE#*@}" <<<"$DIGESTS"; then echo "serve-frank-primary: pinned image absent from rootless storage: $FRANK_PRIMARY_IMAGE" >&2 echo " fetch it: podman pull $FRANK_PRIMARY_IMAGE" >&2 exit 1