Frank
7cb6eeb67a
fa on aborted graph build: build_attn_qsa called ggml_fill on the F16-converted kq mask (GGML_ASSERT F32 at ggml.c:5371). Patch mirrors the MLA sparse path in llm_graph_context::build_attn (F32 base mask for the top-k surgery, F16 cast only at the attention call, gated on cparams.flash_attn). Vendored at scripts/patches/qwen4exp-qsa-f32-mask.patch (sha256 f2f4a486...); the build-qwen38 binary on Dankclaw is rebuilt with it and validated live at CTX_SIZE=32768 (health 200, real completion).
32 lines
1.5 KiB
Diff
32 lines
1.5 KiB
Diff
--- a/src/models/qwen4exp.cpp
|
|
+++ b/src/models/qwen4exp.cpp
|
|
@@ -560,9 +565,13 @@ ggml_tensor * llama_model_qwen4exp::graph::build_attn_qsa(
|
|
ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, k_cur, k_idxs, il));
|
|
ggml_build_forward_expand(gf, mctx_cur->cpy_v(ctx0, v_cur, v_idxs, il));
|
|
}
|
|
|
|
- ggml_tensor * kq_mask = inp->get_kq_mask();
|
|
+ // get_kq_mask() returns the F16-converted mask when flash attention is
|
|
+ // on, but ggml_fill/set_rows require F32. Mirror the MLA sparse path
|
|
+ // (llm_graph_context::build_attn): run the top-k mask surgery on the F32
|
|
+ // base mask and cast to the attention type only at the end.
|
|
+ ggml_tensor * kq_mask = inp->self_kq_mask;
|
|
|
|
// prepare new kq mask - starts filled with -INFINITY
|
|
ggml_tensor * kq_mask_all = ggml_fill(ctx0, kq_mask, -INFINITY);
|
|
|
|
@@ -592,9 +601,12 @@ ggml_tensor * llama_model_qwen4exp::graph::build_attn_qsa(
|
|
ggml_tensor * q = q_cur;
|
|
ggml_tensor * k = mctx_cur->get_k(ctx0, il);
|
|
ggml_tensor * v = mctx_cur->get_v(ctx0, il);
|
|
|
|
- ggml_tensor * cur = build_attn_mha(q, k, v, nullptr, kq_mask_top_k, nullptr, nullptr, kq_scale, il);
|
|
+ // flash attention consumes an F16 mask; softmax consumes F32 as-is
|
|
+ ggml_tensor * kq_mask_attn = inp->cparams.flash_attn ? ggml_cast(ctx0, kq_mask_top_k, GGML_TYPE_F16) : kq_mask_top_k;
|
|
+
|
|
+ ggml_tensor * cur = build_attn_mha(q, k, v, nullptr, kq_mask_attn, nullptr, nullptr, kq_scale, il);
|
|
cb(cur, "kqv_out", il);
|
|
|
|
return cur;
|
|
}
|