llama.cpp runtime arm: serve-script runtime switch, harness containerless support, master build
This commit is contained in:
1 parent
6513672f31
commit
cde91c1a00
3 files changed
+63
-8
No files matched your search
@@ -88,6 +88,10 @@ FRANK_LARGE_UTIL=0.45
|
||||
FRANK_LARGE_SPEC=
|
||||
FRANK_LARGE_SPEC_DIR=/home/dank/models/Qwen3.5-9B
|
||||
FRANK_LARGE_SPEC_TOKENS=2
|
||||
# AUTORESEARCH: torchao int8 weight-only (quark int8 + all fp8 kernels
|
||||
# rejected on gfx1151; torchao is a separate Triton dequant stack)
|
||||
FRANK_LARGE_QUANT=torchao
|
||||
FRANK_LARGE_QUANT_CONFIG='{"torchao_config":"int8_weight_only"}'
|
||||
|
||||
# --- client contract ----------------------------------------------------------
|
||||
# vLLM emits the thinking text in `reasoning` (NOT llama.cpp/gufo's
|
||||
|
||||
Reference in new issue
Block a user