abort on unknown KV type, single-source fa_vec_legal_ne

ggml/src/ggml-metal/ggml-metal-tuning.h

@@ -4,6 +4,7 @@
 #include "ggml.h"
 
 #include <cstdint>
+#include <vector>
 
 namespace ggml_metal_tuning {
 
@@ -52,6 +53,19 @@ struct fa_vec_entry_t {
     fa_vec_cfg_t cfg;
 };
 
+// legal NE values for a (dk,dv): NL = 32/NE, require (dk/4)%NL==0 && (dv/4)%NL==0.
+// single source shared by the offline tuner and test-backend-ops.
+inline std::vector<int> fa_vec_legal_ne(int dk, int dv) {
+    std::vector<int> r;
+    for (int ne : { 1, 2, 4 }) {
+        const int nl = 32 / ne;
+        if ((dk/4) % nl == 0 && (dv/4) % nl == 0) {
+            r.push_back(ne);
+        }
+    }
+    return r;
+}
+
 // test/tune-only override; when set, fa_vec_pick returns it directly.
 void         fa_vec_set_override(fa_vec_cfg_t cfg);
 void         fa_vec_clear_override();

tests/test-backend-ops.cpp

@@ -10161,7 +10161,8 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_from_file(const c
 using set_fa_vec_override_t   = void (*)(int, int);
 using clear_fa_vec_override_t = void (*)(void);
 
-// legal NE for a (dk,dv): NL = 32/NE, require (dk/4)%NL==0 && (dv/4)%NL==0
+// legal NE for a (dk,dv): NL = 32/NE, require (dk/4)%NL==0 && (dv/4)%NL==0.
+// keep in sync with ggml_metal_tuning::fa_vec_legal_ne in ggml-metal-tuning.h (used by the tool)
 static std::vector<int> fa_vec_legal_ne(int dk, int dv) {
     std::vector<int> r;
     for (int ne : {1, 2, 4}) {
@@ -10178,6 +10179,7 @@ static std::vector<int> fa_vec_legal_ne(int dk, int dv) {
 // per test case in the backend-agnostic case list. Covers padded rows (ne01 % Q != 0),
 // per-qq sinks, kvpad, the nsg-dependent shmem offsets / parallel-reduce stride
 // (ne11 -> nsg 1/2/4) and the quantized dequant-once path.
+// single-threaded; g_override_set is backend-global. called only after all parallel workers have joined.
 static bool run_fa_vec_slice(ggml_backend_t backend, ggml_backend_t backend_cpu) {
     auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend));
 

tools/tuning/CMakeLists.txt

@@ -3,6 +3,7 @@ set(TARGET ggml-metal-tuning)
 add_executable(${TARGET} main.cpp bench.cpp fa-vec.cpp)
 target_link_libraries(${TARGET} PRIVATE ggml ${CMAKE_THREAD_LIBS_INIT})
 target_compile_features(${TARGET} PRIVATE cxx_std_17)
+target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/ggml/src/ggml-metal)
 
 if(LLAMA_TOOLS_INSTALL)
     install(TARGETS ${TARGET} RUNTIME)

tools/tuning/bench.cpp

@@ -153,7 +153,7 @@ cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int rep
     for (size_t i = 0; i < order.size(); ++i) {
         res.t[order[i]] = measure_one(backend, cell, reps, set_cand, clear_cand, order[i]);
 
-        if (i % 4 != 0) {
+        if (i % 4 != 0) {  // re-check anchor every 4 candidates: balances drift detection latency against overhead
             continue;
         }
 

tools/tuning/fa-vec.cpp

@@ -3,6 +3,7 @@
 
 #include "ggml.h"
 #include "ggml-backend.h"
+#include "ggml-metal-tuning.h"
 
 #include <algorithm>
 #include <cmath>
@@ -155,7 +156,7 @@ static void fa_init_kq_mask(ggml_tensor * t, std::mt19937 & rng, float min, floa
 static unsigned fa_cell_seed(const fa_shape & s, unsigned base) {
     unsigned h = base;
     for (int v : { s.dk, s.dv, s.ne01, s.ne11, (int) s.type_kv }) {
-        h = h*1000003u + (unsigned) v;
+        h = h*1000003u + (unsigned) v;  // small prime, standard multiplicative hash mixing
     }
     return h;
 }
@@ -175,18 +176,6 @@ static void fa_init_tensors(ggml_context * ctx, const fa_shape & s, unsigned bas
     }
 }
 
-// legal NE for a (dk,dv): NL = 32/NE, require (dk/4)%NL==0 && (dv/4)%NL==0
-static std::vector<int> fa_legal_ne(int dk, int dv) {
-    std::vector<int> r;
-    for (int ne : { 1, 2, 4 }) {
-        const int nl = 32 / ne;
-        if ((dk/4) % nl == 0 && (dv/4) % nl == 0) {
-            r.push_back(ne);
-        }
-    }
-    return r;
-}
-
 using set_override_t   = void (*)(int, int);
 using clear_override_t = void (*)(void);
 using bucket_t         = int  (*)(int64_t);
@@ -227,7 +216,7 @@ static const char * fa_type_token(ggml_type t) {
         case GGML_TYPE_Q5_0: return "GGML_TYPE_Q5_0";
         case GGML_TYPE_Q5_1: return "GGML_TYPE_Q5_1";
         case GGML_TYPE_Q8_0: return "GGML_TYPE_Q8_0";
-        default:             return "GGML_TYPE_F16";
+        default:             GGML_ABORT("unhandled KV type in fa_type_token: %d", (int) t);
     }
 }
 
@@ -256,7 +245,7 @@ static std::vector<fa_cand> fa_build_cands(const fa_procs & procs, int dk, int d
 
     std::vector<fa_cand> cands;
     base_i = -1;
-    for (int ne : fa_legal_ne(dk, dv)) {
+    for (int ne : ggml_metal_tuning::fa_vec_legal_ne(dk, dv)) {
         for (int Q : { 1, 2, 4 }) {
             if (Q == 1 && ne == base_ne) {
                 base_i = (int) cands.size();