Frank
bd5c28c4e9
Replaces the tcclaviger vLLM fork with FreeToken at PR #587 head 77a7201 built on dankserver into localhost/frank-primary-freetoken:pr587-77a7201 (scripts/docker/frank-primary-freetoken.Dockerfile: digest-pinned rocm/pytorch rocm7.14 torch-2.11.0 base, libhiprtc dev symlink, curated --no-deps install per docs/install_amd.md). Shape (first-boot conservative): ONE R9700 (second card idle; --tensor-parallel-size exists for future 2-card tuning), KV pool --num-tokens 300000 (7.2 GiB at 24 KiB/token; clears the 177,356-token harness worst case and one native 262,144 conversation), MoE LRU --moe-cache-rate 0.10, --max-running-requests 2, --max-output-tokens 36864. Parsers AUTO: thinking -> reasoning_content natively (models.yml flips reasoningContentField on the harness side). Experts + PLE pin ~170 GiB host RAM — root unit keeps LimitMEMLOCK=infinity. Container identity renamed to the unit name (Frank's call): frank-primary; vLLM-era frank-primary-qwen joins the legacy cleanup list. Deployer image gate becomes podman image exists (a local build carries no RepoDigests; the old digest grep refused exactly this shape). Rollback tag: frank-primary-vllm-shapeG-20261003.
66 lines
3.3 KiB
Desktop File
66 lines
3.3 KiB
Desktop File
# frank-primary.service — FreeToken (PR #587 ROCm) endpoint (orcarouter
|
|
# Qwen3.8-Flash-Next-Uncensored-FP8, 131-shard payload) on ONE R9700
|
|
# (gfx1201), dankserver :7770. 2026-10-05 cutover from the tcclaviger vLLM
|
|
# fork (git tag frank-primary-vllm-shapeG-20261003 is the pre-cutover
|
|
# state; the retired unit/env/serve trio is in that tag).
|
|
#
|
|
# Same port, same served alias, same container identity as the unit name;
|
|
# the harness models.yml id is unchanged. The serve script drives
|
|
# `podman run` directly (the image carries `ft serve`) and then waits on
|
|
# the container. The image is built ON THE HOST from
|
|
# scripts/docker/frank-primary-freetoken.Dockerfile (FreeToken PR head
|
|
# 77a7201 in the PR's own ROCm 7.14 base image).
|
|
#
|
|
# WHY A ROOT UNIT (not a rootless quadlet): FreeToken pins ~120 GiB of
|
|
# expert weights plus the ~47.7 GiB PLE ngram table in HOST RAM. Pinned
|
|
# pages are charged against RLIMIT_MEMLOCK; dank's login session on this
|
|
# host carries 8 MiB and a user quadlet is capped by the user manager's own
|
|
# hard limit — raising that means restarting user@1000.service and bouncing
|
|
# every user container on the host. A root unit grants
|
|
# LimitMEMLOCK=infinity to the dank process via PID1, and the image stays
|
|
# in dank's rootless podman storage.
|
|
[Unit]
|
|
Description=frank-primary — FreeToken (PR587) Qwen3.8-Flash-Next-Uncensored-FP8 on 1x R9700
|
|
After=network-online.target
|
|
Wants=network-online.target
|
|
# Never let the start-rate limiter give up after a crash loop (a failed LLM
|
|
# endpoint that systemd refuses to retry is worse than a retrying one).
|
|
StartLimitIntervalSec=0
|
|
|
|
[Service]
|
|
User=dank
|
|
Group=dank
|
|
# The expert LRU + PLE pin ~170 GiB of host RAM; an RLIMIT_MEMLOCK smaller
|
|
# than the pinned set fails the load mid-start. PID1 grants infinity here —
|
|
# parity with the fleet's other LLM units.
|
|
LimitMEMLOCK=infinity
|
|
Environment="FRANK_PRIMARY_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-primary.env"
|
|
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-primary.sh
|
|
# systemd stop: wait alone cannot stop the container child; stop it
|
|
# explicitly. ft serve drops connections on SIGTERM; no graceful drain
|
|
# contract is claimed (the OpenAI front end holds no queue to drain —
|
|
# in-flight decodes are lost, which a stop during a bench window accepts).
|
|
ExecStop=/usr/bin/podman stop -t 120 frank-primary
|
|
# Boot cost: 131-shard FP8 page-in + PLE pin + expert upload. vLLM-era
|
|
# budget (1800 s) carried forward; the FreeToken first boot measured here
|
|
# once it ran (see the runbook for the real number).
|
|
TimeoutStartSec=1800
|
|
# Restart=always, NOT on-failure (the 2026-10-01 vLLM lesson carries over:
|
|
# a clean API-server exit after a worker fault reads as success to
|
|
# systemd). `always` restarts on any container death; an explicit
|
|
# `systemctl stop` still wins, so bench windows and deploys are unaffected.
|
|
Restart=always
|
|
RestartSec=10
|
|
# ExecStop allows the engine 120 s to drain (`podman stop -t 120`); without
|
|
# an explicit TimeoutStopSec systemd's 90 s default cuts the stop short,
|
|
# kills it, and records 'Failed with result timeout' (measured 2026-10-01
|
|
# 13:27:17). Must exceed the podman grace with margin.
|
|
TimeoutStopSec=180
|
|
# The engine pins tens of GiB of host RAM (expert rows + PLE table); it is
|
|
# the last process on this host that should be chosen by the kernel OOM
|
|
# killer.
|
|
OOMScoreAdjust=-500
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|