fix(deploy): hash every file a locally built image COPYs, not just its Containerfile
build_compose_image skips `podman build` when the installed image's `cfhash` label matches a hash taken from the checkout now — the skip is what keeps frank-tts-engine's ten-minute recursive source clone off every re-deploy. The hash covered the Containerfile ALONE, so any change to a file the image COPYs was invisible: edit dankclaw/voiceservice/app.py or tts-engine-warm.sh, re-run the deploy, and podman reports the existing image is fine. The redeployed container runs the previous code, /health answers 200, and the e2e gate has no way to see it. It bit the boot warmer twice this week. Each case arm now names its own extra=(…) — the paths its Containerfile COPYs — and the label hashes the Containerfile plus those. The two arms are kept apart deliberately: the façade bakes app.py and tts-start.sh, the engine bakes tts-engine-start.sh and tts-engine-warm.sh, and comfyui/loki/alloy have no extra files, so their hash is unchanged and they do not rebuild on this commit. Hashing rather than word-splitting because these are repo-relative paths under a fixed checkout root. The seam drives the real function against a stub podman and asserts observable output — did the deploy ask podman to build, or did it skip? — in five steps: first deploy builds, unchanged tree does not, editing the engine's COPY'd warm hook rebuilds the ENGINE image and not the façade, editing app.py rebuilds the façade. Reverted scripts/deploy-remote.sh to prove it red on the old hash: the warm-hook step is exactly where it fails.
This commit is contained in:
@@ -541,6 +541,14 @@ deploy_observability() {
|
|||||||
|
|
||||||
build_compose_image() {
|
build_compose_image() {
|
||||||
local service="$1" image cfhash oldhash containerfile
|
local service="$1" image cfhash oldhash containerfile
|
||||||
|
# Every file the image COPIES IN, beside the Containerfile. The skip below
|
||||||
|
# compares a hash against the image's `cfhash` label, so anything the build
|
||||||
|
# bakes must be in that hash or the deploy silently keeps serving the old
|
||||||
|
# bytes: an edited app.py or warm hook leaves the Containerfile untouched,
|
||||||
|
# the label matches, `podman build` never runs, and the health gate passes on
|
||||||
|
# a stale container. Services whose Containerfile pulls everything at
|
||||||
|
# build time (comfyui, loki, alloy) keep an empty list.
|
||||||
|
local -a extra=()
|
||||||
case "$service" in
|
case "$service" in
|
||||||
comfyui)
|
comfyui)
|
||||||
image=comfyui-rocm:gfx1151
|
image=comfyui-rocm:gfx1151
|
||||||
@@ -557,6 +565,7 @@ build_compose_image() {
|
|||||||
# fails on a tag that was never made.
|
# fails on a tag that was never made.
|
||||||
image=franksdank/frank-tts:latest
|
image=franksdank/frank-tts:latest
|
||||||
containerfile=dankclaw/voiceservice/Containerfile
|
containerfile=dankclaw/voiceservice/Containerfile
|
||||||
|
extra=(dankclaw/voiceservice/app.py)
|
||||||
;;
|
;;
|
||||||
frank-tts-engine)
|
frank-tts-engine)
|
||||||
# The arm A TTS engine (qwentts.cpp `tts-server`). Built from
|
# The arm A TTS engine (qwentts.cpp `tts-server`). Built from
|
||||||
@@ -565,12 +574,16 @@ build_compose_image() {
|
|||||||
# offline: the clone only happens when this Containerfile changes.
|
# offline: the clone only happens when this Containerfile changes.
|
||||||
image=franksdank/frank-tts-engine:latest
|
image=franksdank/frank-tts-engine:latest
|
||||||
containerfile=dankclaw/voiceservice/Containerfile.engine
|
containerfile=dankclaw/voiceservice/Containerfile.engine
|
||||||
|
# start.sh is the image's CMD and the warmer is the unit's
|
||||||
|
# ExecStartPost; both are COPY'd in, so both decide what this image is.
|
||||||
|
extra=(dankclaw/voiceservice/tts-engine-start.sh
|
||||||
|
dankclaw/voiceservice/tts-engine-warm.sh)
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
return 0
|
return 0
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
cfhash="$(sha256sum "$containerfile" | cut -d' ' -f1)"
|
cfhash="$(sha256sum "$containerfile" "${extra[@]}" | cut -d' ' -f1 | tr -d '\n')"
|
||||||
oldhash="$(podman image inspect "$image" --format '{{index .Config.Labels "cfhash"}}' 2>/dev/null || echo none)"
|
oldhash="$(podman image inspect "$image" --format '{{index .Config.Labels "cfhash"}}' 2>/dev/null || echo none)"
|
||||||
if [[ "$oldhash" != "$cfhash" ]]; then
|
if [[ "$oldhash" != "$cfhash" ]]; then
|
||||||
# --format docker is required, not cosmetic: podman defaults to the OCI
|
# --format docker is required, not cosmetic: podman defaults to the OCI
|
||||||
|
|||||||
@@ -826,6 +826,92 @@ PY
|
|||||||
! printf '%s' "$handler" | grep -qE '^[[:space:]]*systemctl --user enable'
|
! printf '%s' "$handler" | grep -qE '^[[:space:]]*systemctl --user enable'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@test "an image rebuild is triggered by every file the image bakes in" {
|
||||||
|
# build_compose_image skips `podman build` when the image's cfhash label
|
||||||
|
# matches a hash — the thing that keeps the engine's 10-minute source clone
|
||||||
|
# from running on every re-deploy. If that hash covers only the Containerfile,
|
||||||
|
# then editing app.py (COPY'd into the façade) or the warm hook (COPY'd into
|
||||||
|
# the engine) rebuilds NOTHING: the deploy reports success, /health answers,
|
||||||
|
# and the container is still running last week's code. Observable output, not
|
||||||
|
# a grep of the handler: did the deploy ask podman to build, or did it skip?
|
||||||
|
awk '/^build_compose_image\(\) \{/,/^\}/' "$REMOTE" > "$TEST_TMP/bci.sh"
|
||||||
|
[ -s "$TEST_TMP/bci.sh" ] || { echo "build_compose_image not extractable from $REMOTE"; false; }
|
||||||
|
|
||||||
|
repo="$TEST_TMP/repo"
|
||||||
|
mkdir -p "$repo/dankclaw/voiceservice"
|
||||||
|
: > "$repo/dankclaw/voiceservice/Containerfile"
|
||||||
|
: > "$repo/dankclaw/voiceservice/Containerfile.engine"
|
||||||
|
: > "$repo/dankclaw/voiceservice/app.py"
|
||||||
|
: > "$repo/dankclaw/voiceservice/tts-engine-start.sh"
|
||||||
|
: > "$repo/dankclaw/voiceservice/tts-engine-warm.sh"
|
||||||
|
|
||||||
|
# Stub podman: `image inspect` replays the label this stub stored on the last
|
||||||
|
# build of THAT image (nothing the first time), `build` records the tag and
|
||||||
|
# writes the label it was given. One state file per image, so the pair's two
|
||||||
|
# images cannot borrow each other's freshness.
|
||||||
|
mkdir -p "$TEST_TMP/bin" "$TEST_TMP/labels"
|
||||||
|
cat > "$TEST_TMP/bin/podman" <<'STUB'
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# Image tags contain a slash (`franksdank/foo:latest`), so the state file name
|
||||||
|
# flattens it — otherwise the write lands in a directory that does not exist and
|
||||||
|
# every image looks permanently unbuilt.
|
||||||
|
case "$1" in
|
||||||
|
image) # invoked as `podman image inspect <image> --format …`, so $3 is the tag
|
||||||
|
f="$LABEL_DIR/${3//\//_}"
|
||||||
|
[[ -s "$f" ]] && { cat "$f"; exit 0; }
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
build)
|
||||||
|
# Two passes: `podman build` puts `--label cfhash=…` BEFORE `-t <tag>`, so
|
||||||
|
# the tag is not known when the label argument goes by.
|
||||||
|
tag=""
|
||||||
|
label=""
|
||||||
|
for a in "$@"; do
|
||||||
|
case "$a" in
|
||||||
|
*:*) tag="$a" ;;
|
||||||
|
cfhash=*) label="${a#cfhash=}" ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
[[ -n "$label" ]] && printf '%s' "$label" > "$LABEL_DIR/${tag//\//_}"
|
||||||
|
printf 'BUILD %s\n' "$tag" >> "$BUILD_LOG"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
STUB
|
||||||
|
chmod +x "$TEST_TMP/bin/podman"
|
||||||
|
cat > "$TEST_TMP/drive.sh" <<'DRV'
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
source "$BCI"
|
||||||
|
build_compose_image "$1"
|
||||||
|
DRV
|
||||||
|
: > "$TEST_TMP/builds"
|
||||||
|
builds() { grep -cF "BUILD $1" "$TEST_TMP/builds"; }
|
||||||
|
drive() {
|
||||||
|
( cd "$repo" && BCI="$TEST_TMP/bci.sh" PATH="$TEST_TMP/bin:$PATH" \
|
||||||
|
LABEL_DIR="$TEST_TMP/labels" BUILD_LOG="$TEST_TMP/builds" \
|
||||||
|
bash "$TEST_TMP/drive.sh" "$1" )
|
||||||
|
}
|
||||||
|
|
||||||
|
drive frank-tts-engine
|
||||||
|
[ "$(builds franksdank/frank-tts-engine:latest)" -eq 1 ] \
|
||||||
|
|| { echo "the first deploy of an unbuilt image must build"; false; }
|
||||||
|
drive frank-tts-engine
|
||||||
|
[ "$(builds franksdank/frank-tts-engine:latest)" -eq 1 ] \
|
||||||
|
|| { echo "an unchanged tree must not rebuild — that is the 10-minute clone"; false; }
|
||||||
|
|
||||||
|
printf '# the boot warmer grew a line\n' >> "$repo/dankclaw/voiceservice/tts-engine-warm.sh"
|
||||||
|
drive frank-tts-engine
|
||||||
|
[ "$(builds franksdank/frank-tts-engine:latest)" -eq 2 ] \
|
||||||
|
|| { echo "editing a script the image COPYs must rebuild it"; false; }
|
||||||
|
|
||||||
|
drive frank-tts
|
||||||
|
[ "$(builds franksdank/frank-tts:latest)" -eq 1 ] || { echo "the façade must build once"; false; }
|
||||||
|
printf '# the shim grew a route\n' >> "$repo/dankclaw/voiceservice/app.py"
|
||||||
|
drive frank-tts
|
||||||
|
[ "$(builds franksdank/frank-tts:latest)" -eq 2 ] \
|
||||||
|
|| { echo "editing app.py must rebuild the façade image"; false; }
|
||||||
|
}
|
||||||
|
|
||||||
@test "the voice stack's homepage state: one card for the pair, and it is hidden while undeployed" {
|
@test "the voice stack's homepage state: one card for the pair, and it is hidden while undeployed" {
|
||||||
# The engine is not on the homepage and must not be: it publishes no port, so
|
# The engine is not on the homepage and must not be: it publishes no port, so
|
||||||
# there is no address the checker could dial, and a card that can never be
|
# there is no address the checker could dial, and a card that can never be
|
||||||
|
|||||||
Reference in New Issue
Block a user