From 8045779cff04265c7ba2c43275471e9b14ce49b3 Mon Sep 17 00:00:00 2001 From: forforever73 <690105611@qq.com> Date: Mon, 3 Aug 2026 11:44:22 +0800 Subject: [PATCH] cleanup --- ggml/src/ggml-metal/ggml-metal-tuning.h | 7 +- tests/test-backend-ops.cpp | 14 +- tools/tuning/bench.cpp | 91 ++++---- tools/tuning/bench.h | 54 ++--- tools/tuning/fa-vec.cpp | 297 ++++++++++++------------ tools/tuning/fa-vec.h | 20 +- tools/tuning/main.cpp | 7 +- 7 files changed, 236 insertions(+), 254 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal-tuning.h b/ggml/src/ggml-metal/ggml-metal-tuning.h index fd9a7ffd94..640ce53efb 100644 --- a/ggml/src/ggml-metal/ggml-metal-tuning.h +++ b/ggml/src/ggml-metal/ggml-metal-tuning.h @@ -19,9 +19,8 @@ int fa_vec_ne11_bucket(int64_t ne11); int fa_vec_ne01_bucket(int64_t ne01); // NE baked into each (dk,dv) baseline instantiation in kernels/fa.metal. -// Hand-maintained mirror; keep in sync with those instantiations. test-backend-ops forces -// every legal (Q,NE) on dk=128 and dk=576 under Metal, so a missing instantiation for -// those two surfaces there; the other head sizes are only covered by the offline tuner. +// Hand-maintained mirror; keep in sync with those instantiations. +// The Metal test slice covers every legal config for dk=128 and dk=576. int fa_vec_baseline_ne(int dk, int dv); // Tuned table has two row kinds. Exact rows key a (ne11_b, ne01_b) bucket. Default rows @@ -59,7 +58,7 @@ inline std::vector fa_vec_legal_ne(int dk, int dv) { std::vector r; for (int ne : { 1, 2, 4 }) { const int nl = 32 / ne; - if ((dk/4) % nl == 0 && (dv/4) % nl == 0) { + if ((dk / 4) % nl == 0 && (dv / 4) % nl == 0) { r.push_back(ne); } } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 573ecfd358..bddc8bb169 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10157,12 +10157,10 @@ static std::vector> make_test_cases_from_file(const c } // ---- FA vec (Q,NE): forced-config numerical slice (Metal only) ---- -// metal proc_address bridges (resolved by string, not symbol linkage) using set_fa_vec_override_t = void (*)(int, int); using clear_fa_vec_override_t = void (*)(void); -// legal NE for a (dk,dv): NL = 32/NE, require (dk/4)%NL==0 && (dv/4)%NL==0. -// keep in sync with ggml_metal_tuning::fa_vec_legal_ne in ggml-metal-tuning.h (used by the tool) +// NL = 32/NE must divide both dk/4 and dv/4. static std::vector fa_vec_legal_ne(int dk, int dv) { std::vector r; for (int ne : {1, 2, 4}) { @@ -10174,12 +10172,8 @@ static std::vector fa_vec_legal_ne(int dk, int dv) { return r; } -// Forces every legal (Q,NE) on two representative shapes and compares against the CPU -// reference. Metal-only: the override is a backend-global switch, so it cannot be expressed -// per test case in the backend-agnostic case list. Covers padded rows (ne01 % Q != 0), -// per-qq sinks, kvpad, the nsg-dependent shmem offsets / parallel-reduce stride -// (ne11 -> nsg 1/2/4) and the quantized dequant-once path. -// single-threaded; g_override_set is backend-global. called only after all parallel workers have joined. +// Covers padded rows, sinks, kvpad, multi-SIMDgroup reduction, quantized K/V, and MLA views. +// The override is backend-global, so this runs after all parallel workers have joined. static bool run_fa_vec_slice(ggml_backend_t backend, ggml_backend_t backend_cpu) { auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend)); @@ -10366,8 +10360,6 @@ static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mo output_printer->print_summary(test_summary_info(n_ok, tests_run, false)); output_printer->print_failed_tests(failed_tests); - // Metal-only: force every legal (Q,NE) on a bounded slice of shapes. Reuses the - // reference CPU backend above; a no-op on backends without the override proc. const bool slice_ok = run_fa_vec_slice(backend, backend_cpu.get()); return n_ok == tests_run && slice_ok; diff --git a/tools/tuning/bench.cpp b/tools/tuning/bench.cpp index 38527ffca1..59945506c3 100644 --- a/tools/tuning/bench.cpp +++ b/tools/tuning/bench.cpp @@ -7,14 +7,16 @@ #include #include -perf_cell build_perf_cell(ggml_backend_t backend, const build_graph_fn & build, - const init_tensors_fn & init, const op_flops_fn & flops) { +perf_cell build_perf_cell(ggml_backend_t backend, + const build_graph_fn & build, + const init_tensors_fn & init, + const op_flops_fn & flops) { perf_cell cell; const size_t graph_nodes = 1024; ggml_init_params params = { - /* .mem_size = */ ggml_tensor_overhead()*128 + ggml_graph_overhead_custom(graph_nodes, false), + /* .mem_size = */ ggml_tensor_overhead() * 128 + ggml_graph_overhead_custom(graph_nodes, false), /* .mem_base = */ NULL, /* .no_alloc = */ true, }; @@ -28,7 +30,7 @@ perf_cell build_perf_cell(ggml_backend_t backend, const build_graph_fn & build, } cell.buf.reset(ggml_backend_alloc_ctx_tensors(cell.ctx.get(), backend)); - if (cell.buf == NULL) { + if (!cell.buf) { return cell; } @@ -38,24 +40,24 @@ perf_cell build_perf_cell(ggml_backend_t backend, const build_graph_fn & build, ggml_build_forward_expand(cell.gf, out); // replicate the op to amortize overhead (target ~50 GFLOP/compute, capped to bound graph size) - cell.n_runs = 1; - if (flops(out) > 0) { + cell.n_runs = 1; + const uint64_t n_flops = flops(out); + if (n_flops > 0) { const uint64_t target_flops = 50ULL * 1000 * 1000 * 1000; const int cap = 512; - const int by_flops = (int) std::min(cap, (int64_t) (target_flops / flops(out))); - cell.n_runs = std::max(1, std::min(by_flops, (int) (ggml_graph_size(cell.gf) - ggml_graph_n_nodes(cell.gf)))); + const int by_flops = (int) std::min(cap, (int64_t) (target_flops / n_flops)); + cell.n_runs = + std::max(1, std::min(by_flops, (int) (ggml_graph_size(cell.gf) - ggml_graph_n_nodes(cell.gf)))); } for (int i = 1; i < cell.n_runs; ++i) { ggml_graph_add_node(cell.gf, out); } - cell.ok = true; - return cell; } double time_cell_median(ggml_backend_t backend, const perf_cell & cell, int reps) { - if (!cell.ok) { + if (cell.gf == nullptr) { return -1.0; } @@ -70,15 +72,17 @@ double time_cell_median(ggml_backend_t backend, const perf_cell & cell, int reps ggml_backend_synchronize(backend); samples.push_back((double) (ggml_time_us() - t0)); } - std::nth_element(samples.begin(), samples.begin() + samples.size()/2, samples.end()); + std::nth_element(samples.begin(), samples.begin() + samples.size() / 2, samples.end()); - return samples[samples.size()/2] / cell.n_runs; + return samples[samples.size() / 2] / cell.n_runs; } -// times one candidate and returns its time; -1 on failure -static double measure_one(ggml_backend_t backend, const perf_cell & cell, int reps, - const set_candidate_fn & set_cand, const clear_candidate_fn & clear_cand, - int cand) { +static double measure_one(ggml_backend_t backend, + const perf_cell & cell, + int reps, + const set_candidate_fn & set_cand, + const clear_candidate_fn & clear_cand, + int cand) { set_cand(cand); const double t = time_cell_median(backend, cell, reps); clear_cand(); @@ -88,17 +92,22 @@ static double measure_one(ggml_backend_t backend, const perf_cell & cell, int re // waits for the anchor to come back within eps of anchor_ref, with exponential backoff. // returns the converged anchor, or -1 if it never converged within max_wait. -static double cool_until_steady(ggml_backend_t backend, const perf_cell & cell, int reps, - const set_candidate_fn & set_cand, const clear_candidate_fn & clear_cand, - int baseline_cand, double & anchor_ref, const cooldown_opts & cool, - const char * cell_label) { +static double cool_until_steady(ggml_backend_t backend, + const perf_cell & cell, + int reps, + const set_candidate_fn & set_cand, + const clear_candidate_fn & clear_cand, + int baseline_cand, + double & anchor_ref, + const cooldown_opts & cool, + const char * cell_label) { int total_wait = 0; - for (int sleep_s = 2; total_wait < cool.max_wait; sleep_s = std::min(sleep_s*2, 32)) { + for (int sleep_s = 2; total_wait < cool.max_wait; sleep_s = std::min(sleep_s * 2, 32)) { const int this_wait = std::min(sleep_s, cool.max_wait - total_wait); - fprintf(stderr, "# COOL sleeping %ds (%ds/%ds) %s\n", - this_wait, total_wait + this_wait, cool.max_wait, cell_label); + fprintf(stderr, "# COOL sleeping %ds (%ds/%ds) %s\n", this_wait, total_wait + this_wait, cool.max_wait, + cell_label); std::this_thread::sleep_for(std::chrono::seconds(this_wait)); total_wait += this_wait; @@ -112,7 +121,7 @@ static double cool_until_steady(ggml_backend_t backend, const perf_cell & cell, anchor_ref = a; } - if (a <= anchor_ref*(1.0 + cool.eps)) { + if (a <= anchor_ref * (1.0 + cool.eps)) { fprintf(stderr, "# COOL steady after %ds %s\n", total_wait, cell_label); return a; } @@ -123,14 +132,17 @@ static double cool_until_steady(ggml_backend_t backend, const perf_cell & cell, return -1.0; } -cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int reps, - int n_cands, const std::vector & order, - const set_candidate_fn & set_cand, +cell_result measure_cell(ggml_backend_t backend, + const perf_cell & cell, + int reps, + const std::vector & order, + const set_candidate_fn & set_cand, const clear_candidate_fn & clear_cand, - int baseline_cand, const cooldown_opts & cool, - const char * cell_label) { + int baseline_cand, + const cooldown_opts & cool, + const char * cell_label) { cell_result res; - res.t.assign(n_cands, 0.0); + res.t.assign(order.size(), 0.0); double anchor_ref = 0.0; @@ -140,8 +152,8 @@ cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int rep std::vector> anchors; auto window_start = [&]() -> size_t { - for (size_t i = anchors.size(); i-- > 0; ) { - if (anchors[i].first <= anchor_ref*(1.0 + cool.eps)) { + for (size_t i = anchors.size(); i-- > 0;) { + if (anchors[i].first <= anchor_ref * (1.0 + cool.eps)) { return anchors[i].second; } } @@ -153,7 +165,7 @@ cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int rep for (size_t i = 0; i < order.size(); ++i) { res.t[order[i]] = measure_one(backend, cell, reps, set_cand, clear_cand, order[i]); - if (i % 4 != 0) { // re-check anchor every 4 candidates: balances drift detection latency against overhead + if (i % 4 != 0) { continue; } @@ -171,7 +183,7 @@ cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int rep continue; } - const double drift = std::fabs(a - anchor_ref)/anchor_ref; + const double drift = std::fabs(a - anchor_ref) / anchor_ref; // a cooler anchor than any so far becomes the reference: whatever was measured // before it was measured on a hotter machine @@ -184,7 +196,7 @@ cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int rep continue; } - fprintf(stderr, "# WARN throttling? anchor drift %.1f%% %s\n", 100.0*drift, cell_label); + fprintf(stderr, "# WARN throttling? anchor drift %.1f%% %s\n", 100.0 * drift, cell_label); if (!cool.enabled) { anchors.push_back({ a, i }); @@ -199,10 +211,8 @@ cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int rep const size_t dirty_from = window_start(); - res.n_cooldowns++; - - const double a_cool = cool_until_steady(backend, cell, reps, set_cand, clear_cand, - baseline_cand, anchor_ref, cool, cell_label); + const double a_cool = + cool_until_steady(backend, cell, reps, set_cand, clear_cand, baseline_cand, anchor_ref, cool, cell_label); if (a_cool <= 0.0) { res.trusted = false; return res; @@ -217,7 +227,6 @@ cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int rep fprintf(stderr, "# REDO candidates %zu..%zu %s\n", dirty_from, i, cell_label); for (size_t j = dirty_from; j <= i; ++j) { res.t[order[j]] = measure_one(backend, cell, reps, set_cand, clear_cand, order[j]); - res.n_remeasures++; } } diff --git a/tools/tuning/bench.h b/tools/tuning/bench.h index 6349d970bd..10167ce39f 100644 --- a/tools/tuning/bench.h +++ b/tools/tuning/bench.h @@ -1,67 +1,57 @@ #pragma once -#include "ggml.h" #include "ggml-backend.h" #include "ggml-cpp.h" +#include "ggml.h" #include #include #include -// one prebuilt op graph, replicated n_runs times so a single graph_compute amortizes -// dispatch/sync overhead. reused across candidates: an override only changes which -// pipeline is picked at encode time, so the (large) input tensors stay allocated. +// A prebuilt graph replicated to amortize dispatch and synchronization overhead. struct perf_cell { ggml_context_ptr ctx; ggml_backend_buffer_ptr buf; ggml_cgraph * gf = nullptr; int n_runs = 0; - bool ok = false; }; -// builds the op graph for one shape. returns the output tensor, or null if unsupported. -using build_graph_fn = std::function; -// fills the allocated tensors of ctx with input data +using build_graph_fn = std::function; using init_tensors_fn = std::function; -// flops of one op instance, used to size n_runs -using op_flops_fn = std::function; +using op_flops_fn = std::function; -perf_cell build_perf_cell(ggml_backend_t backend, const build_graph_fn & build, - const init_tensors_fn & init, const op_flops_fn & flops); +perf_cell build_perf_cell(ggml_backend_t backend, + const build_graph_fn & build, + const init_tensors_fn & init, + const op_flops_fn & flops); -// median per-op time (us) over the prebuilt cell for whatever config is currently set double time_cell_median(ggml_backend_t backend, const perf_cell & cell, int reps); struct cooldown_opts { bool enabled = true; double drift = 0.10; // anchor drift that triggers a cooldown double eps = 0.03; // anchor tolerance to call the GPU cool again - int max_wait = 120; // seconds of cooling per cell before giving up - int max_retry = 2; // re-measure rounds per cell before giving up + int max_wait = 120; // seconds of cooling per cell before giving up + int max_retry = 2; // re-measure rounds per cell before giving up }; -// applies candidate i (an index into the tuner's own candidate list) -using set_candidate_fn = std::function; -// undoes the last set_candidate +using set_candidate_fn = std::function; using clear_candidate_fn = std::function; -// giving up on a cell returns early with trusted == false, so a trusted cell is one every -// candidate of was measured; callers may still see a non-positive t[] from a failed measure. struct cell_result { - std::vector t; // time (us) per candidate index, <= 0 if not measured - bool trusted = true; // false -> caller must drop this cell + std::vector t; + bool trusted = true; double anchor_min = 0.0; double anchor_max = 0.0; - int n_cooldowns = 0; - int n_remeasures = 0; }; -// times every candidate over the prebuilt cell, re-measuring a periodic baseline anchor -// to watch for thermal drift. order[] gives the (shuffled) visiting order; baseline_cand is -// the candidate the anchor forces, so drift is measured against a config the tuner controls. -cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int reps, - int n_cands, const std::vector & order, - const set_candidate_fn & set_cand, +// Times candidates in order while using baseline_cand as a thermal-drift anchor. +cell_result measure_cell(ggml_backend_t backend, + const perf_cell & cell, + int reps, + const std::vector & order, + const set_candidate_fn & set_cand, const clear_candidate_fn & clear_cand, - int baseline_cand, const cooldown_opts & cool, - const char * cell_label); + int baseline_cand, + const cooldown_opts & cool, + const char * cell_label); diff --git a/tools/tuning/fa-vec.cpp b/tools/tuning/fa-vec.cpp index 828920acc9..2dd1c95be5 100644 --- a/tools/tuning/fa-vec.cpp +++ b/tools/tuning/fa-vec.cpp @@ -1,9 +1,9 @@ #include "fa-vec.h" -#include "bench.h" -#include "ggml.h" +#include "bench.h" #include "ggml-backend.h" #include "ggml-metal-tuning.h" +#include "ggml.h" #include #include @@ -16,9 +16,9 @@ // GQA spec-decode shape: enough query heads to keep the GPU busy so the Q>1 K/V-reuse // benefit is visible. nh KV heads, nr2 query heads each, nr3 batches. -static const int FA_NH = 4; -static const int FA_NR2 = 8; -static const int FA_NR3 = 1; +static const int FA_NH = 4; +static const int FA_NR2 = 8; +static const int FA_NR3 = 1; struct fa_shape { int dk; @@ -34,13 +34,12 @@ static ggml_tensor * fa_build_graph(ggml_context * ctx, const fa_shape & s) { const int64_t dk_padded = GGML_PAD(s.dk, ggml_blck_size(s.type_kv)); const int64_t dv_padded = GGML_PAD(s.dv, ggml_blck_size(s.type_kv)); - ggml_tensor * q = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, dk_padded, s.ne01, FA_NH*FA_NR2, FA_NR3); + ggml_tensor * q = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, dk_padded, s.ne01, FA_NH * FA_NR2, FA_NR3); ggml_set_name(q, "q"); // K/V are views of a 2x-tall parent, as they are of the KV cache in production - ggml_tensor * k0 = ggml_new_tensor_4d(ctx, s.type_kv, dk_padded, 2*s.ne11, FA_NH, FA_NR3); - ggml_tensor * k = ggml_view_4d(ctx, k0, dk_padded, s.ne11, FA_NH, FA_NR3, - k0->nb[1], k0->nb[2], k0->nb[3], 0); + ggml_tensor * k0 = ggml_new_tensor_4d(ctx, s.type_kv, dk_padded, 2 * s.ne11, FA_NH, FA_NR3); + ggml_tensor * k = ggml_view_4d(ctx, k0, dk_padded, s.ne11, FA_NH, FA_NR3, k0->nb[1], k0->nb[2], k0->nb[3], 0); ggml_set_name(k, "k"); ggml_tensor * v = nullptr; @@ -48,16 +47,15 @@ static ggml_tensor * fa_build_graph(ggml_context * ctx, const fa_shape & s) { // MLA: the V cache is a sub-view of the K cache v = ggml_view_4d(ctx, k, dv_padded, s.ne11, FA_NH, FA_NR3, k->nb[1], k->nb[2], k->nb[3], 0); } else { - ggml_tensor * v0 = ggml_new_tensor_4d(ctx, s.type_kv, dv_padded, 2*s.ne11, FA_NH, FA_NR3); - v = ggml_view_4d(ctx, v0, dv_padded, s.ne11, FA_NH, FA_NR3, - v0->nb[1], v0->nb[2], v0->nb[3], 0); + ggml_tensor * v0 = ggml_new_tensor_4d(ctx, s.type_kv, dv_padded, 2 * s.ne11, FA_NH, FA_NR3); + v = ggml_view_4d(ctx, v0, dv_padded, s.ne11, FA_NH, FA_NR3, v0->nb[1], v0->nb[2], v0->nb[3], 0); } ggml_set_name(v, "v"); ggml_tensor * m = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, s.ne11, s.ne01, 1, FA_NR3); ggml_set_name(m, "m"); - ggml_tensor * out = ggml_flash_attn_ext(ctx, q, k, v, m, 1.0f/sqrtf((float) s.dk), 0.0f, 0.0f); + ggml_tensor * out = ggml_flash_attn_ext(ctx, q, k, v, m, 1.0f / sqrtf((float) s.dk), 0.0f, 0.0f); ggml_flash_attn_ext_set_prec(out, GGML_PREC_F32); ggml_set_name(out, "out"); @@ -66,21 +64,20 @@ static ggml_tensor * fa_build_graph(ggml_context * ctx, const fa_shape & s) { static uint64_t fa_op_flops(const fa_shape & s) { // Q*K^T is ne01 x dk x ne11, P*V is ne01 x ne11 x dv, per head - return (uint64_t) 2*FA_NH*FA_NR2*s.ne01*(s.dk + s.dv)*s.ne11*FA_NR3; + return (uint64_t) 2 * FA_NH * FA_NR2 * s.ne01 * (s.dk + s.dv) * s.ne11 * FA_NR3; } -// mirrors init_tensor_uniform: uniform f32 data, quantized in place for quantized types static void fa_init_uniform(ggml_tensor * t, std::mt19937 & rng, float min, float max) { const size_t nels = ggml_nelements(t); - std::vector data(nels); + std::vector data(nels); std::uniform_real_distribution dist(min, max); for (size_t i = 0; i < nels; i++) { data[i] = dist(rng); } if (t->type == GGML_TYPE_F32) { - ggml_backend_tensor_set(t, data.data(), 0, nels*sizeof(float)); + ggml_backend_tensor_set(t, data.data(), 0, nels * sizeof(float)); return; } @@ -88,10 +85,10 @@ static void fa_init_uniform(ggml_tensor * t, std::mt19937 & rng, float min, floa GGML_ASSERT(nels % ggml_blck_size(t->type) == 0); std::vector imatrix(t->ne[0], 1.0f); - const float * im = imatrix.data(); + const float * im = imatrix.data(); if (!ggml_quantize_requires_imatrix(t->type)) { // when the imatrix is optional, exercise both paths; pick via one of the random numbers - if (data[0] > 0.5f*(min + max)) { + if (data[0] > 0.5f * (min + max)) { im = nullptr; } } @@ -116,8 +113,8 @@ static void fa_init_kq_mask(ggml_tensor * t, std::mt19937 & rng, float min, floa const int32_t ne2 = (int32_t) t->ne[2]; const int32_t ne3 = (int32_t) t->ne[3]; - std::vector data_f32(size_t(ne0)*ne1*ne2*ne3); - std::vector data_f16(size_t(ne0)*ne1*ne2*ne3); + std::vector data_f32(size_t(ne0) * ne1 * ne2 * ne3); + std::vector data_f16(size_t(ne0) * ne1 * ne2 * ne3); std::uniform_real_distribution dis(min, max); for (size_t i = 0; i < data_f32.size(); i++) { @@ -127,7 +124,7 @@ static void fa_init_kq_mask(ggml_tensor * t, std::mt19937 & rng, float min, floa const int blck0 = 128; const int blck1 = 64; - const int n_inf_zero_blocks = 0.2*(ne0*ne1*ne2*ne3)/(blck0*blck1); + const int n_inf_zero_blocks = 0.2 * (ne0 * ne1 * ne2 * ne3) / (blck0 * blck1); for (int b = 0; b < n_inf_zero_blocks; b++) { const int p3 = (int) (rng() % ne3); @@ -138,7 +135,7 @@ static void fa_init_kq_mask(ggml_tensor * t, std::mt19937 & rng, float min, floa const bool inf = rng() & 1; for (int i1 = 0; i1 < blck1 && p1 + i1 < ne1; i1++) { - const int idx = p3*ne2*ne1*ne0 + p2*ne1*ne0 + (p1 + i1)*ne0 + p0; + const int idx = p3 * ne2 * ne1 * ne0 + p2 * ne1 * ne0 + (p1 + i1) * ne0 + p0; for (int i0 = 0; i0 < blck0 && p0 + i0 < ne0; i0++) { data_f32[idx + i0] = inf ? -INFINITY : 0.0f; @@ -146,17 +143,15 @@ static void fa_init_kq_mask(ggml_tensor * t, std::mt19937 & rng, float min, floa } } - ggml_fp32_to_fp16_row(data_f32.data(), data_f16.data(), ne0*ne1*ne2*ne3); + ggml_fp32_to_fp16_row(data_f32.data(), data_f16.data(), ne0 * ne1 * ne2 * ne3); - ggml_backend_tensor_set(t, data_f16.data(), 0, data_f16.size()*sizeof(ggml_fp16_t)); + ggml_backend_tensor_set(t, data_f16.data(), 0, data_f16.size() * sizeof(ggml_fp16_t)); } -// per-cell deterministic seed: the shape decides it, so a cell is reproducible -// regardless of what else the sweep visited before it static unsigned fa_cell_seed(const fa_shape & s, unsigned base) { unsigned h = base; for (int v : { s.dk, s.dv, s.ne01, s.ne11, (int) s.type_kv }) { - h = h*1000003u + (unsigned) v; // small prime, standard multiplicative hash mixing + h = h * 1000003u + (unsigned) v; } return h; } @@ -178,9 +173,9 @@ static void fa_init_tensors(ggml_context * ctx, const fa_shape & s, unsigned bas using set_override_t = void (*)(int, int); using clear_override_t = void (*)(void); -using bucket_t = int (*)(int64_t); -using baseline_ne_t = int (*)(int, int); -using device_token_t = const char * (*)(ggml_backend_dev_t); +using bucket_t = int (*)(int64_t); +using baseline_ne_t = int (*)(int, int); +using device_token_t = const char * (*) (ggml_backend_dev_t); struct fa_procs { set_override_t set_ov = nullptr; @@ -190,37 +185,25 @@ struct fa_procs { baseline_ne_t baseline_ne = nullptr; device_token_t dev_token = nullptr; - bool ok() const { - return set_ov && clr_ov && ne11_bucket && ne01_bucket && baseline_ne && dev_token; - } + bool ok() const { return set_ov && clr_ov && ne11_bucket && ne01_bucket && baseline_ne && dev_token; } }; static fa_procs fa_resolve_procs(ggml_backend_dev_t dev) { ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev); fa_procs p; - p.set_ov = (set_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_set_fa_vec_override"); - p.clr_ov = (clear_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_clear_fa_vec_override"); - p.ne11_bucket = (bucket_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_ne11_bucket"); - p.ne01_bucket = (bucket_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_ne01_bucket"); - p.baseline_ne = (baseline_ne_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_baseline_ne"); - p.dev_token = (device_token_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_device_token"); + p.set_ov = (set_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_set_fa_vec_override"); + p.clr_ov = + (clear_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_clear_fa_vec_override"); + p.ne11_bucket = (bucket_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_ne11_bucket"); + p.ne01_bucket = (bucket_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_ne01_bucket"); + p.baseline_ne = + (baseline_ne_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_baseline_ne"); + p.dev_token = (device_token_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_device_token"); return p; } -static const char * fa_type_token(ggml_type t) { - switch (t) { - case GGML_TYPE_Q4_0: return "GGML_TYPE_Q4_0"; - case GGML_TYPE_Q4_1: return "GGML_TYPE_Q4_1"; - case GGML_TYPE_Q5_0: return "GGML_TYPE_Q5_0"; - case GGML_TYPE_Q5_1: return "GGML_TYPE_Q5_1"; - case GGML_TYPE_Q8_0: return "GGML_TYPE_Q8_0"; - default: GGML_ABORT("unhandled KV type in fa_type_token: %d", (int) t); - } -} - -// "f16,q4_0" -> does it contain ggml_type_name(t)? null filter accepts everything static bool fa_filter_has(const char * filter, const char * name) { if (!filter) { return true; @@ -231,15 +214,16 @@ static bool fa_filter_has(const char * filter, const char * name) { return f.find(std::string(",") + name + ",") != std::string::npos; } -struct fa_cand { int Q, NE; }; - -struct fa_point { // one swept grid point with its candidate times - int dk, dv, ne11, ne01; - std::vector t; // indexed like the shape's candidate list +struct fa_cand { + int Q, NE; }; -// candidate list for one shape, identical for every grid point of it. base_i is the index of -// the (Q=1, baseline NE) candidate: the anchor config, and what the tuning gates compare to. +struct fa_point { + int dk, dv, ne11, ne01; + std::vector t; +}; + +// base_i identifies the (Q=1, baseline NE) anchor configuration. static std::vector fa_build_cands(const fa_procs & procs, int dk, int dv, int & base_i) { const int base_ne = procs.baseline_ne(dk, dv); @@ -267,34 +251,56 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune const char * dev_token = procs.dev_token(dev); - struct shape_t { int dk, dv; }; - const shape_t shapes[] = { { 32, 32 }, { 64, 64 }, { 96, 96 }, { 128, 128 }, { 192, 192 }, - { 192, 128 }, { 256, 256 }, { 320, 256 }, { 512, 512 }, { 576, 512 } }; + struct shape_t { + int dk, dv; + }; + + const shape_t shapes[] = { + { 32, 32 }, + { 64, 64 }, + { 96, 96 }, + { 128, 128 }, + { 192, 192 }, + { 192, 128 }, + { 256, 256 }, + { 320, 256 }, + { 512, 512 }, + { 576, 512 } + }; const int ne11_rep[] = { 512, 2048, 8192, 32768 }; // ne11 bucket representatives const int ne01_rep[] = { 1, 2, 3, 4, 5, 6, 7, 8, 16 }; // point buckets (1-4) + tail mod-4 cycle + anchor - const ggml_type types[] = { GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, - GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0 }; + + struct dtype_t { + ggml_type type; + const char * token; + }; + + const dtype_t dtypes[] = { + { GGML_TYPE_F16, "GGML_TYPE_F16" }, + { GGML_TYPE_Q4_0, "GGML_TYPE_Q4_0" }, + { GGML_TYPE_Q4_1, "GGML_TYPE_Q4_1" }, + { GGML_TYPE_Q5_0, "GGML_TYPE_Q5_0" }, + { GGML_TYPE_Q5_1, "GGML_TYPE_Q5_1" }, + { GGML_TYPE_Q8_0, "GGML_TYPE_Q8_0" }, + }; const double TUNE_TAU = 0.05; // max POINTWISE regret to ride a domain default const double TUNE_THETA = 1.05; // min AGGREGATE bucket speedup vs baseline to tune at all - cooldown_opts cool; - cool.enabled = opts.cooldown; - cool.drift = opts.cool_drift; - cool.eps = opts.cool_eps; - cool.max_wait = opts.cool_max_wait; - cool.max_retry = opts.cool_max_retry; + const cooldown_opts cool = { + opts.cooldown, opts.cool_drift, opts.cool_eps, opts.cool_max_wait, opts.cool_max_retry, + }; - fprintf(stderr, "seed=%u reps=%d cooldown=%s (drift=%.2f eps=%.2f max_wait=%ds max_retry=%d)\n", - opts.seed, opts.reps, cool.enabled ? "on" : "off", - cool.drift, cool.eps, cool.max_wait, cool.max_retry); + fprintf(stderr, "seed=%u reps=%d cooldown=%s (drift=%.2f eps=%.2f max_wait=%ds max_retry=%d)\n", opts.seed, + opts.reps, cool.enabled ? "on" : "off", cool.drift, cool.eps, cool.max_wait, cool.max_retry); fprintf(stderr, "device token: %s\n", dev_token); int n_untrusted = 0; printf("// ==== BEGIN fa_vec_tuned_table rows (%s) ====\n", dev_token); - for (ggml_type type_kv : types) { + for (const auto & dtype : dtypes) { + const ggml_type type_kv = dtype.type; if (!fa_filter_has(opts.dtype_filter, ggml_type_name(type_kv))) { continue; } @@ -308,19 +314,19 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune continue; } - int base_i = 0; - std::vector cands = fa_build_cands(procs, s.dk, s.dv, base_i); + int base_i = 0; + std::vector cands = fa_build_cands(procs, s.dk, s.dv, base_i); for (int ne11 : ne11_rep) { for (int ne01 : ne01_rep) { const fa_shape sh = { s.dk, s.dv, ne01, ne11, type_kv }; - perf_cell cell = build_perf_cell(backend, - [&](ggml_context * ctx) { return fa_build_graph(ctx, sh); }, + perf_cell cell = build_perf_cell( + backend, [&](ggml_context * ctx) { return fa_build_graph(ctx, sh); }, [&](ggml_context * ctx) { fa_init_tensors(ctx, sh, opts.seed); }, [&](ggml_tensor *) { return fa_op_flops(sh); }); - if (!cell.ok) { + if (cell.gf == nullptr) { continue; } @@ -334,22 +340,18 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune char label[128]; snprintf(label, sizeof(label), "dk=%d ne11=%d", s.dk, ne11); - cell_result r = measure_cell(backend, cell, opts.reps, - (int) cands.size(), order, - [&](int i) { procs.set_ov(cands[i].Q, cands[i].NE); }, - [&]() { procs.clr_ov(); }, - base_i, cool, label); + cell_result r = measure_cell( + backend, cell, opts.reps, order, [&](int i) { procs.set_ov(cands[i].Q, cands[i].NE); }, + [&]() { procs.clr_ov(); }, base_i, cool, label); - // per-cell noise floor: spread of the repeated same-config anchor if (r.anchor_min > 0.0) { - fprintf(stderr, "# noise dk=%d dv=%d ne11=%d ne01=%d spread=%.1f%%\n", - s.dk, s.dv, ne11, ne01, 100.0*(r.anchor_max - r.anchor_min)/r.anchor_min); + fprintf(stderr, "# noise dk=%d dv=%d ne11=%d ne01=%d spread=%.1f%%\n", s.dk, s.dv, ne11, ne01, + 100.0 * (r.anchor_max - r.anchor_min) / r.anchor_min); } if (!r.trusted) { n_untrusted++; - fprintf(stderr, "# DROP untrusted cell dk=%d dv=%d ne11=%d ne01=%d\n", - s.dk, s.dv, ne11, ne01); + fprintf(stderr, "# DROP untrusted cell dk=%d dv=%d ne11=%d ne01=%d\n", s.dk, s.dv, ne11, ne01); continue; } @@ -360,17 +362,17 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune } } const double base_t = r.t[base_i]; - const bool keep = best_i >= 0 && base_t > 0.0 && r.t[best_i] < base_t*0.98; + const bool keep = best_i >= 0 && base_t > 0.0 && r.t[best_i] < base_t * 0.98; - fprintf(stderr, "# dtype=%s dk=%d dv=%d ne11=%d ne01=%d:", - ggml_type_name(type_kv), s.dk, s.dv, ne11, ne01); + fprintf(stderr, "# dtype=%s dk=%d dv=%d ne11=%d ne01=%d:", ggml_type_name(type_kv), s.dk, s.dv, + ne11, ne01); for (size_t i = 0; i < cands.size(); ++i) { fprintf(stderr, " Q%dNE%d=%.1f%s", cands[i].Q, cands[i].NE, r.t[i], (int) i == best_i ? "*" : ""); } if (keep) { - fprintf(stderr, " => Q%d,NE%d %.2fx\n", - cands[best_i].Q, cands[best_i].NE, base_t/r.t[best_i]); + fprintf(stderr, " => Q%d,NE%d %.2fx\n", cands[best_i].Q, cands[best_i].NE, + base_t / r.t[best_i]); } else { fprintf(stderr, " => baseline\n"); } @@ -385,49 +387,52 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune // the default's pointwise regret vs the bucket target, or its aggregate slowdown vs // baseline, exceeds TUNE_TAU. std::vector rows_out; - char rbuf[192]; + char rbuf[192]; for (auto s : shapes) { if (!fa_filter_has(opts.dk_filter, std::to_string(s.dk).c_str())) { continue; } - int base_i = 0; - std::vector cands = fa_build_cands(procs, s.dk, s.dv, base_i); + int base_i = 0; + std::vector cands = fa_build_cands(procs, s.dk, s.dv, base_i); struct bkt_t { - int b11, b01, Ti; - std::vector agg; - double base_agg; + int b11, b01, Ti; + std::vector agg; std::vector bp; }; - // bucket the grid points with the runtime's bucketers, so keys match fa_vec_pick. - // short-KV points (ne11 bucket 0) are dropped: the runtime serves those from baseline. - std::set> seen; - for (const auto & p : pts) { - if (p.dk != s.dk || p.dv != s.dv) { - continue; - } - const int b11 = procs.ne11_bucket(p.ne11); + std::set> buckets; + for (int ne11 : ne11_rep) { + const int b11 = procs.ne11_bucket(ne11); if (b11 == 0) { continue; } - seen.insert({ b11, procs.ne01_bucket(p.ne01) }); + for (int ne01 : ne01_rep) { + buckets.insert({ b11, procs.ne01_bucket(ne01) }); + } } std::vector bks; - for (const auto & bb : seen) { + for (const auto & bb : buckets) { const int b11 = bb.first, b01 = bb.second; std::vector bp; for (const auto & p : pts) { - if (p.dk == s.dk && p.dv == s.dv && - procs.ne11_bucket(p.ne11) == b11 && procs.ne01_bucket(p.ne01) == b01) { + if (p.dk == s.dk && p.dv == s.dv && procs.ne11_bucket(p.ne11) == b11 && + procs.ne01_bucket(p.ne01) == b01) { bp.push_back(&p); } } + fprintf(stderr, "# bucket dk=%d dv=%d ne11_b=%d ne01_b=%d samples=%zu\n", s.dk, s.dv, b11, b01, + bp.size()); + if (bp.empty()) { + fprintf(stderr, "# WARN empty bucket dk=%d dv=%d ne11_b=%d ne01_b=%d\n", s.dk, s.dv, b11, b01); + continue; + } + std::vector agg(cands.size(), 0.0), worst(cands.size(), 0.0); for (const auto * p : bp) { double bestt = 0.0; @@ -439,35 +444,24 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune for (size_t i = 0; i < cands.size(); ++i) { agg[i] += p->t[i]; if (p->t[i] > 0.0 && bestt > 0.0) { - worst[i] = std::max(worst[i], p->t[i]/bestt); + worst[i] = std::max(worst[i], p->t[i] / bestt); } } } int robust = 0; for (size_t i = 1; i < cands.size(); ++i) { - if (worst[i] < worst[robust] || - (worst[i] == worst[robust] && (cands[i].Q < cands[robust].Q || - (cands[i].Q == cands[robust].Q && cands[i].NE < cands[robust].NE)))) { + if (worst[i] < worst[robust] || (worst[i] == worst[robust] && (cands[i].Q < cands[robust].Q || + (cands[i].Q == cands[robust].Q && + cands[i].NE < cands[robust].NE)))) { robust = (int) i; } } const bool tune = robust != base_i && agg[base_i] > 0.0 && agg[robust] > 0.0 && - agg[base_i]/agg[robust] >= TUNE_THETA; + agg[base_i] / agg[robust] >= TUNE_THETA; - bks.push_back({ b11, b01, tune ? robust : base_i, agg, agg[base_i], bp }); - } - - // bucket coverage: a hardcoded sampling grid can't produce a wrong key, only miss - // a bucket, so report what each bucket actually got - for (const auto & b : bks) { - fprintf(stderr, "# bucket dk=%d dv=%d ne11_b=%d ne01_b=%d samples=%zu\n", - s.dk, s.dv, b.b11, b.b01, b.bp.size()); - if (b.bp.empty()) { - fprintf(stderr, "# WARN empty bucket dk=%d dv=%d ne11_b=%d ne01_b=%d\n", - s.dk, s.dv, b.b11, b.b01); - } + bks.push_back({ b11, b01, tune ? robust : base_i, agg, bp }); } // pointwise regret of default cfg d vs the bucket target: a ratio-of-sums lets a @@ -477,7 +471,7 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune for (const auto * p : b->bp) { const double td = p->t[d], tT = p->t[b->Ti]; if (td > 0.0 && tT > 0.0) { - r = std::max(r, td/tT - 1.0); + r = std::max(r, td / tT - 1.0); } } return r; @@ -495,14 +489,15 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune } // default cfg = the one minimizing (#rows, total achieved time, Q, NE) - int bestD = -1, bestRows = 1 << 30; + int bestD = -1, bestRows = 1 << 30; double bestTot = 0.0; for (size_t d = 0; d < cands.size(); ++d) { - int rows = ((int) d != base_i) ? 1 : 0; - double tot = 0.0; + int rows = ((int) d != base_i) ? 1 : 0; + double tot = 0.0; for (const auto * b : db) { - const double reg = reg_pointwise(b, (int) d); - const double slow = b->base_agg > 0.0 ? b->agg[d]/b->base_agg - 1.0 : 0.0; + const double base_agg = b->agg[base_i]; + const double reg = reg_pointwise(b, (int) d); + const double slow = base_agg > 0.0 ? b->agg[d] / base_agg - 1.0 : 0.0; if (reg > TUNE_TAU || slow > TUNE_TAU) { rows++; tot += b->agg[b->Ti]; @@ -510,33 +505,33 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune tot += b->agg[d]; } } - const bool better = bestD < 0 || rows < bestRows || - (rows == bestRows && (tot < bestTot || - (tot == bestTot && (cands[d].Q < cands[bestD].Q || - (cands[d].Q == cands[bestD].Q && cands[d].NE < cands[bestD].NE))))); + const bool better = + bestD < 0 || rows < bestRows || + (rows == bestRows && + (tot < bestTot || + (tot == bestTot && (cands[d].Q < cands[bestD].Q || + (cands[d].Q == cands[bestD].Q && cands[d].NE < cands[bestD].NE))))); if (better) { - bestD = (int) d; + bestD = (int) d; bestRows = rows; - bestTot = tot; + bestTot = tot; } } - const int dom_id = (dom == 0) ? 0 : 1; // FA_VEC_DOMAIN_DECODE / FA_VEC_DOMAIN_BATCH if (bestD != base_i) { - snprintf(rbuf, sizeof(rbuf), " { { %s, %s, %d, %d, -1, %d }, { %d, %d } },", - dev_token, fa_type_token(type_kv), s.dk, s.dv, dom_id, - cands[bestD].Q, cands[bestD].NE); + snprintf(rbuf, sizeof(rbuf), " { { %s, %s, %d, %d, -1, %d }, { %d, %d } },", dev_token, + dtype.token, s.dk, s.dv, dom, cands[bestD].Q, cands[bestD].NE); rows_out.emplace_back(rbuf); } for (const auto * b : db) { - const double reg = reg_pointwise(b, bestD); - const double slow = b->base_agg > 0.0 ? b->agg[bestD]/b->base_agg - 1.0 : 0.0; + const double base_agg = b->agg[base_i]; + const double reg = reg_pointwise(b, bestD); + const double slow = base_agg > 0.0 ? b->agg[bestD] / base_agg - 1.0 : 0.0; if (reg <= TUNE_TAU && slow <= TUNE_TAU) { - continue; // rides the default / baseline + continue; } - snprintf(rbuf, sizeof(rbuf), " { { %s, %s, %d, %d, %d, %d }, { %d, %d } },", - dev_token, fa_type_token(type_kv), s.dk, s.dv, b->b11, b->b01, - cands[b->Ti].Q, cands[b->Ti].NE); + snprintf(rbuf, sizeof(rbuf), " { { %s, %s, %d, %d, %d, %d }, { %d, %d } },", dev_token, + dtype.token, s.dk, s.dv, b->b11, b->b01, cands[b->Ti].Q, cands[b->Ti].NE); rows_out.emplace_back(rbuf); } } diff --git a/tools/tuning/fa-vec.h b/tools/tuning/fa-vec.h index b015a8051f..b815f18673 100644 --- a/tools/tuning/fa-vec.h +++ b/tools/tuning/fa-vec.h @@ -2,19 +2,17 @@ #include "ggml-backend.h" -// options shared by all tuners; parsed in main.cpp struct tuner_opts { - const char * dtype_filter = nullptr; // comma-separated, e.g. "f16,q4_0"; null = all - const char * dk_filter = nullptr; // comma-separated dk values, e.g. "128,192"; null = all - int reps = 7; - unsigned seed = 1234; - bool cooldown = true; - double cool_drift = 0.10; - double cool_eps = 0.03; - int cool_max_wait = 120; + const char * dtype_filter = nullptr; // comma-separated, e.g. "f16,q4_0"; null = all + const char * dk_filter = nullptr; // comma-separated dk values, e.g. "128,192"; null = all + int reps = 7; + unsigned seed = 1234; + bool cooldown = true; + double cool_drift = 0.10; + double cool_eps = 0.03; + int cool_max_wait = 120; int cool_max_retry = 2; }; -// runs the FA-vec (Q,NE) sweep and prints a pasteable table block on stdout. -// returns false only on environment failure (missing procs), never on perf results. +// Returns false only when the required Metal proc bridges are unavailable. bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tuner_opts & opts); diff --git a/tools/tuning/main.cpp b/tools/tuning/main.cpp index ce9eb0a993..fbe0505936 100644 --- a/tools/tuning/main.cpp +++ b/tools/tuning/main.cpp @@ -1,7 +1,6 @@ #include "fa-vec.h" - -#include "ggml.h" #include "ggml-backend.h" +#include "ggml.h" #include #include @@ -42,8 +41,8 @@ static void usage(const char * argv0) { } int main(int argc, char ** argv) { - const char * tuner = nullptr; - const char * bname = nullptr; + const char * tuner = nullptr; + const char * bname = nullptr; tuner_opts opts; for (int i = 1; i < argc; i++) {