diff --git a/scripts/kld-then-calib.sh b/scripts/kld-then-calib.sh new file mode 100755 index 0000000..8754c7f --- /dev/null +++ b/scripts/kld-then-calib.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# kld-then-calib.sh — RUNS ON DANKSERVER. Campaign chain: wait for the KLD +# record to finish, verify it, then boot the 1.2.0 calibration server and +# push the 10M-token corpus. One GPU, so the phases serialize; prod is +# restored by each run script's trap. +set -u +KLOG=/home/dank/logs/swift15-kld-driver.log +CLOG=/home/dank/logs/swift15-calib-driver.log +S=/home/dank/serverAppConfigs-swift15/scripts + +# Phase 1: KLD record. +while ! grep -q "exited rc=" "$KLOG" 2>/dev/null; do sleep 60; done +RC=$(grep -o "exited rc=[0-9]*" "$KLOG" | tail -1) +if [ "$RC" != "exited rc=0" ]; then echo "KLD FAILED ($RC); calib NOT started"; exit 1; fi +N=$(find /home/dank/corpora/kld-ref -type f | wc -l) +SZ=$(du -sm /home/dank/corpora/kld-ref | cut -f1) +echo "KLD OK files=$N size_mib=$SZ" +[ "$N" -ge 1 ] && [ "$SZ" -ge 1000 ] || { echo "kld-ref looks thin; calib NOT started"; exit 1; } + +# Phase 2: calibration serve (1.2.0) + corpus push. +nohup bash "$S/run-swift15-bf16.sh" calib > "$CLOG" 2>&1 & +CP=$! +for _ in $(seq 1 160); do + sleep 15 + curl -sf localhost:7770/health >/dev/null 2>&1 && break +done +if ! curl -sf localhost:7770/health >/dev/null 2>&1; then + echo "CALIB BOOT FAILED; see /home/dank/logs/swift15_bf16.log"; kill "$CP" 2>/dev/null; exit 1 +fi +echo "CALIB UP, pushing corpus" +python3 "$S/calib-push.py" --port 7770 --parallel 16 --container swift15ref --max-rounds 6 +PUSH=$? +kill "$CP" 2>/dev/null +sleep 10 +echo "CALIB_CHAIN_DONE push=$PUSH grams=$(find /home/dank/corpora/calib -name 'gram.r*.bin' 2>/dev/null | wc -l)"