Frank
07af1a9366
One-GPU serialization: the chain waits for the kld driver log to close rc=0, sanity-checks kld-ref, boots the calibration server, pushes the corpus at parallel 16 (4 engine slots; queue absorbs the rest), then kills the server so its trap restores production.
36 lines
1.5 KiB
Bash
Executable File
36 lines
1.5 KiB
Bash
Executable File
#!/bin/bash
|
|
# kld-then-calib.sh — RUNS ON DANKSERVER. Campaign chain: wait for the KLD
|
|
# record to finish, verify it, then boot the 1.2.0 calibration server and
|
|
# push the 10M-token corpus. One GPU, so the phases serialize; prod is
|
|
# restored by each run script's trap.
|
|
set -u
|
|
KLOG=/home/dank/logs/swift15-kld-driver.log
|
|
CLOG=/home/dank/logs/swift15-calib-driver.log
|
|
S=/home/dank/serverAppConfigs-swift15/scripts
|
|
|
|
# Phase 1: KLD record.
|
|
while ! grep -q "exited rc=" "$KLOG" 2>/dev/null; do sleep 60; done
|
|
RC=$(grep -o "exited rc=[0-9]*" "$KLOG" | tail -1)
|
|
if [ "$RC" != "exited rc=0" ]; then echo "KLD FAILED ($RC); calib NOT started"; exit 1; fi
|
|
N=$(find /home/dank/corpora/kld-ref -type f | wc -l)
|
|
SZ=$(du -sm /home/dank/corpora/kld-ref | cut -f1)
|
|
echo "KLD OK files=$N size_mib=$SZ"
|
|
[ "$N" -ge 1 ] && [ "$SZ" -ge 1000 ] || { echo "kld-ref looks thin; calib NOT started"; exit 1; }
|
|
|
|
# Phase 2: calibration serve (1.2.0) + corpus push.
|
|
nohup bash "$S/run-swift15-bf16.sh" calib > "$CLOG" 2>&1 &
|
|
CP=$!
|
|
for _ in $(seq 1 160); do
|
|
sleep 15
|
|
curl -sf localhost:7770/health >/dev/null 2>&1 && break
|
|
done
|
|
if ! curl -sf localhost:7770/health >/dev/null 2>&1; then
|
|
echo "CALIB BOOT FAILED; see /home/dank/logs/swift15_bf16.log"; kill "$CP" 2>/dev/null; exit 1
|
|
fi
|
|
echo "CALIB UP, pushing corpus"
|
|
python3 "$S/calib-push.py" --port 7770 --parallel 16 --container swift15ref --max-rounds 6
|
|
PUSH=$?
|
|
kill "$CP" 2>/dev/null
|
|
sleep 10
|
|
echo "CALIB_CHAIN_DONE push=$PUSH grams=$(find /home/dank/corpora/calib -name 'gram.r*.bin' 2>/dev/null | wc -l)"
|