LLM-Inference/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-08-12 22:31:11 +04:00
Code Issues Packages Projects Releases Wiki Activity

cleanup

Browse Source
This commit is contained in:
forforever73
2026-08-05 16:22:33 +08:00
committed by YiChen Lv
parent 9be171f5fa
commit 8045779cff
7 changed files with 236 additions and 254 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-metal/ggml-metal-tuning.h
+3 -4
View File
@@ -19,9 +19,8 @@ int fa_vec_ne11_bucket(int64_t ne11);
int fa_vec_ne01_bucket(int64_t ne01);
// NE baked into each (dk,dv) baseline instantiation in kernels/fa.metal.
// Hand-maintained mirror; keep in sync with those instantiations. test-backend-ops forces
// every legal (Q,NE) on dk=128 and dk=576 under Metal, so a missing instantiation for
// those two surfaces there; the other head sizes are only covered by the offline tuner.
// Hand-maintained mirror; keep in sync with those instantiations.
// The Metal test slice covers every legal config for dk=128 and dk=576.
int fa_vec_baseline_ne(int dk, int dv);
// Tuned table has two row kinds. Exact rows key a (ne11_b, ne01_b) bucket. Default rows
@@ -59,7 +58,7 @@ inline std::vector<int> fa_vec_legal_ne(int dk, int dv) {
std::vector<int> r;
for (int ne : { 1, 2, 4 }) {
const int nl = 32 / ne;
if ((dk/4) % nl == 0 && (dv/4) % nl == 0) {
if ((dk / 4) % nl == 0 && (dv / 4) % nl == 0) {
r.push_back(ne);
}
}
tests/test-backend-ops.cpp
+3 -11
View File
@@ -10157,12 +10157,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_from_file(const c
}
// ---- FA vec (Q,NE): forced-config numerical slice (Metal only) ----
// metal proc_address bridges (resolved by string, not symbol linkage)
using set_fa_vec_override_t = void (*)(int, int);
using clear_fa_vec_override_t = void (*)(void);
// legal NE for a (dk,dv): NL = 32/NE, require (dk/4)%NL==0 && (dv/4)%NL==0.
// keep in sync with ggml_metal_tuning::fa_vec_legal_ne in ggml-metal-tuning.h (used by the tool)
// NL = 32/NE must divide both dk/4 and dv/4.
static std::vector<int> fa_vec_legal_ne(int dk, int dv) {
std::vector<int> r;
for (int ne : {1, 2, 4}) {
@@ -10174,12 +10172,8 @@ static std::vector<int> fa_vec_legal_ne(int dk, int dv) {
return r;
}
// Forces every legal (Q,NE) on two representative shapes and compares against the CPU
// reference. Metal-only: the override is a backend-global switch, so it cannot be expressed
// per test case in the backend-agnostic case list. Covers padded rows (ne01 % Q != 0),
// per-qq sinks, kvpad, the nsg-dependent shmem offsets / parallel-reduce stride
// (ne11 -> nsg 1/2/4) and the quantized dequant-once path.
// single-threaded; g_override_set is backend-global. called only after all parallel workers have joined.
// Covers padded rows, sinks, kvpad, multi-SIMDgroup reduction, quantized K/V, and MLA views.
// The override is backend-global, so this runs after all parallel workers have joined.
static bool run_fa_vec_slice(ggml_backend_t backend, ggml_backend_t backend_cpu) {
auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend));
@@ -10366,8 +10360,6 @@ static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mo
output_printer->print_summary(test_summary_info(n_ok, tests_run, false));
output_printer->print_failed_tests(failed_tests);
// Metal-only: force every legal (Q,NE) on a bounded slice of shapes. Reuses the
// reference CPU backend above; a no-op on backends without the override proc.
const bool slice_ok = run_fa_vec_slice(backend, backend_cpu.get());
return n_ok == tests_run && slice_ok;
tools/tuning/bench.cpp
+50 -41
View File
@@ -7,14 +7,16 @@
#include <thread>
#include <utility>
perf_cell build_perf_cell(ggml_backend_t backend, const build_graph_fn & build,
const init_tensors_fn & init, const op_flops_fn & flops) {
perf_cell build_perf_cell(ggml_backend_t backend,
const build_graph_fn & build,
const init_tensors_fn & init,
const op_flops_fn & flops) {
perf_cell cell;
const size_t graph_nodes = 1024;
ggml_init_params params = {
/* .mem_size = */ ggml_tensor_overhead()*128 + ggml_graph_overhead_custom(graph_nodes, false),
/* .mem_size = */ ggml_tensor_overhead() * 128 + ggml_graph_overhead_custom(graph_nodes, false),
/* .mem_base = */ NULL,
/* .no_alloc = */ true,
};
@@ -28,7 +30,7 @@ perf_cell build_perf_cell(ggml_backend_t backend, const build_graph_fn & build,
}
cell.buf.reset(ggml_backend_alloc_ctx_tensors(cell.ctx.get(), backend));
if (cell.buf == NULL) {
if (!cell.buf) {
return cell;
}
@@ -38,24 +40,24 @@ perf_cell build_perf_cell(ggml_backend_t backend, const build_graph_fn & build,
ggml_build_forward_expand(cell.gf, out);
// replicate the op to amortize overhead (target ~50 GFLOP/compute, capped to bound graph size)
cell.n_runs = 1;
if (flops(out) > 0) {
cell.n_runs = 1;
const uint64_t n_flops = flops(out);
if (n_flops > 0) {
const uint64_t target_flops = 50ULL * 1000 * 1000 * 1000;
const int cap = 512;
const int by_flops = (int) std::min<int64_t>(cap, (int64_t) (target_flops / flops(out)));
cell.n_runs = std::max(1, std::min<int>(by_flops, (int) (ggml_graph_size(cell.gf) - ggml_graph_n_nodes(cell.gf))));
const int by_flops = (int) std::min<int64_t>(cap, (int64_t) (target_flops / n_flops));
cell.n_runs =
std::max(1, std::min<int>(by_flops, (int) (ggml_graph_size(cell.gf) - ggml_graph_n_nodes(cell.gf))));
}
for (int i = 1; i < cell.n_runs; ++i) {
ggml_graph_add_node(cell.gf, out);
}
cell.ok = true;
return cell;
}
double time_cell_median(ggml_backend_t backend, const perf_cell & cell, int reps) {
if (!cell.ok) {
if (cell.gf == nullptr) {
return -1.0;
}
@@ -70,15 +72,17 @@ double time_cell_median(ggml_backend_t backend, const perf_cell & cell, int reps
ggml_backend_synchronize(backend);
samples.push_back((double) (ggml_time_us() - t0));
}
std::nth_element(samples.begin(), samples.begin() + samples.size()/2, samples.end());
std::nth_element(samples.begin(), samples.begin() + samples.size() / 2, samples.end());
return samples[samples.size()/2] / cell.n_runs;
return samples[samples.size() / 2] / cell.n_runs;
}
// times one candidate and returns its time; -1 on failure
static double measure_one(ggml_backend_t backend, const perf_cell & cell, int reps,
const set_candidate_fn & set_cand, const clear_candidate_fn & clear_cand,
int cand) {
static double measure_one(ggml_backend_t backend,
const perf_cell & cell,
int reps,
const set_candidate_fn & set_cand,
const clear_candidate_fn & clear_cand,
int cand) {
set_cand(cand);
const double t = time_cell_median(backend, cell, reps);
clear_cand();
@@ -88,17 +92,22 @@ static double measure_one(ggml_backend_t backend, const perf_cell & cell, int re
// waits for the anchor to come back within eps of anchor_ref, with exponential backoff.
// returns the converged anchor, or -1 if it never converged within max_wait.
static double cool_until_steady(ggml_backend_t backend, const perf_cell & cell, int reps,
const set_candidate_fn & set_cand, const clear_candidate_fn & clear_cand,
int baseline_cand, double & anchor_ref, const cooldown_opts & cool,
const char * cell_label) {
static double cool_until_steady(ggml_backend_t backend,
const perf_cell & cell,
int reps,
const set_candidate_fn & set_cand,
const clear_candidate_fn & clear_cand,
int baseline_cand,
double & anchor_ref,
const cooldown_opts & cool,
const char * cell_label) {
int total_wait = 0;
for (int sleep_s = 2; total_wait < cool.max_wait; sleep_s = std::min(sleep_s*2, 32)) {
for (int sleep_s = 2; total_wait < cool.max_wait; sleep_s = std::min(sleep_s * 2, 32)) {
const int this_wait = std::min(sleep_s, cool.max_wait - total_wait);
fprintf(stderr, "# COOL sleeping %ds (%ds/%ds) %s\n",
this_wait, total_wait + this_wait, cool.max_wait, cell_label);
fprintf(stderr, "# COOL sleeping %ds (%ds/%ds) %s\n", this_wait, total_wait + this_wait, cool.max_wait,
cell_label);
std::this_thread::sleep_for(std::chrono::seconds(this_wait));
total_wait += this_wait;
@@ -112,7 +121,7 @@ static double cool_until_steady(ggml_backend_t backend, const perf_cell & cell,
anchor_ref = a;
}
if (a <= anchor_ref*(1.0 + cool.eps)) {
if (a <= anchor_ref * (1.0 + cool.eps)) {
fprintf(stderr, "# COOL steady after %ds %s\n", total_wait, cell_label);
return a;
}
@@ -123,14 +132,17 @@ static double cool_until_steady(ggml_backend_t backend, const perf_cell & cell,
return -1.0;
}
cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int reps,
int n_cands, const std::vector<int> & order,
const set_candidate_fn & set_cand,
cell_result measure_cell(ggml_backend_t backend,
const perf_cell & cell,
int reps,
const std::vector<int> & order,
const set_candidate_fn & set_cand,
const clear_candidate_fn & clear_cand,
int baseline_cand, const cooldown_opts & cool,
const char * cell_label) {
int baseline_cand,
const cooldown_opts & cool,
const char * cell_label) {
cell_result res;
res.t.assign(n_cands, 0.0);
res.t.assign(order.size(), 0.0);
double anchor_ref = 0.0;
@@ -140,8 +152,8 @@ cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int rep
std::vector<std::pair<double, size_t>> anchors;
auto window_start = [&]() -> size_t {
for (size_t i = anchors.size(); i-- > 0; ) {
if (anchors[i].first <= anchor_ref*(1.0 + cool.eps)) {
for (size_t i = anchors.size(); i-- > 0;) {
if (anchors[i].first <= anchor_ref * (1.0 + cool.eps)) {
return anchors[i].second;
}
}
@@ -153,7 +165,7 @@ cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int rep
for (size_t i = 0; i < order.size(); ++i) {
res.t[order[i]] = measure_one(backend, cell, reps, set_cand, clear_cand, order[i]);
if (i % 4 != 0) { // re-check anchor every 4 candidates: balances drift detection latency against overhead
if (i % 4 != 0) {
continue;
}
@@ -171,7 +183,7 @@ cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int rep
continue;
}
const double drift = std::fabs(a - anchor_ref)/anchor_ref;
const double drift = std::fabs(a - anchor_ref) / anchor_ref;
// a cooler anchor than any so far becomes the reference: whatever was measured
// before it was measured on a hotter machine
@@ -184,7 +196,7 @@ cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int rep
continue;
}
fprintf(stderr, "# WARN throttling? anchor drift %.1f%% %s\n", 100.0*drift, cell_label);
fprintf(stderr, "# WARN throttling? anchor drift %.1f%% %s\n", 100.0 * drift, cell_label);
if (!cool.enabled) {
anchors.push_back({ a, i });
@@ -199,10 +211,8 @@ cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int rep
const size_t dirty_from = window_start();
res.n_cooldowns++;
const double a_cool = cool_until_steady(backend, cell, reps, set_cand, clear_cand,
baseline_cand, anchor_ref, cool, cell_label);
const double a_cool =
cool_until_steady(backend, cell, reps, set_cand, clear_cand, baseline_cand, anchor_ref, cool, cell_label);
if (a_cool <= 0.0) {
res.trusted = false;
return res;
@@ -217,7 +227,6 @@ cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int rep
fprintf(stderr, "# REDO candidates %zu..%zu %s\n", dirty_from, i, cell_label);
for (size_t j = dirty_from; j <= i; ++j) {
res.t[order[j]] = measure_one(backend, cell, reps, set_cand, clear_cand, order[j]);
res.n_remeasures++;
}
}
tools/tuning/bench.h
+22 -32
View File
@@ -1,67 +1,57 @@
#pragma once
#include "ggml.h"
#include "ggml-backend.h"
#include "ggml-cpp.h"
#include "ggml.h"
#include <cstdint>
#include <functional>
#include <vector>
// one prebuilt op graph, replicated n_runs times so a single graph_compute amortizes
// dispatch/sync overhead. reused across candidates: an override only changes which
// pipeline is picked at encode time, so the (large) input tensors stay allocated.
// A prebuilt graph replicated to amortize dispatch and synchronization overhead.
struct perf_cell {
ggml_context_ptr ctx;
ggml_backend_buffer_ptr buf;
ggml_cgraph * gf = nullptr;
int n_runs = 0;
bool ok = false;
};
// builds the op graph for one shape. returns the output tensor, or null if unsupported.
using build_graph_fn = std::function<ggml_tensor *(ggml_context *)>;
// fills the allocated tensors of ctx with input data
using build_graph_fn = std::function<ggml_tensor *(ggml_context *)>;
using init_tensors_fn = std::function<void(ggml_context *)>;
// flops of one op instance, used to size n_runs
using op_flops_fn = std::function<uint64_t(ggml_tensor *)>;
using op_flops_fn = std::function<uint64_t(ggml_tensor *)>;
perf_cell build_perf_cell(ggml_backend_t backend, const build_graph_fn & build,
const init_tensors_fn & init, const op_flops_fn & flops);
perf_cell build_perf_cell(ggml_backend_t backend,
const build_graph_fn & build,
const init_tensors_fn & init,
const op_flops_fn & flops);
// median per-op time (us) over the prebuilt cell for whatever config is currently set
double time_cell_median(ggml_backend_t backend, const perf_cell & cell, int reps);
struct cooldown_opts {
bool enabled = true;
double drift = 0.10; // anchor drift that triggers a cooldown
double eps = 0.03; // anchor tolerance to call the GPU cool again
int max_wait = 120; // seconds of cooling per cell before giving up
int max_retry = 2; // re-measure rounds per cell before giving up
int max_wait = 120; // seconds of cooling per cell before giving up
int max_retry = 2; // re-measure rounds per cell before giving up
};
// applies candidate i (an index into the tuner's own candidate list)
using set_candidate_fn = std::function<void(int)>;
// undoes the last set_candidate
using set_candidate_fn = std::function<void(int)>;
using clear_candidate_fn = std::function<void()>;
// giving up on a cell returns early with trusted == false, so a trusted cell is one every
// candidate of was measured; callers may still see a non-positive t[] from a failed measure.
struct cell_result {
std::vector<double> t; // time (us) per candidate index, <= 0 if not measured
bool trusted = true; // false -> caller must drop this cell
std::vector<double> t;
bool trusted = true;
double anchor_min = 0.0;
double anchor_max = 0.0;
int n_cooldowns = 0;
int n_remeasures = 0;
};
// times every candidate over the prebuilt cell, re-measuring a periodic baseline anchor
// to watch for thermal drift. order[] gives the (shuffled) visiting order; baseline_cand is
// the candidate the anchor forces, so drift is measured against a config the tuner controls.
cell_result measure_cell(ggml_backend_t backend, const perf_cell & cell, int reps,
int n_cands, const std::vector<int> & order,
const set_candidate_fn & set_cand,
// Times candidates in order while using baseline_cand as a thermal-drift anchor.
cell_result measure_cell(ggml_backend_t backend,
const perf_cell & cell,
int reps,
const std::vector<int> & order,
const set_candidate_fn & set_cand,
const clear_candidate_fn & clear_cand,
int baseline_cand, const cooldown_opts & cool,
const char * cell_label);
int baseline_cand,
const cooldown_opts & cool,
const char * cell_label);
tools/tuning/fa-vec.cpp
+146 -151
View File
@@ -1,9 +1,9 @@
#include "fa-vec.h"
#include "bench.h"
#include "ggml.h"
#include "bench.h"
#include "ggml-backend.h"
#include "ggml-metal-tuning.h"
#include "ggml.h"
#include <algorithm>
#include <cmath>
@@ -16,9 +16,9 @@
// GQA spec-decode shape: enough query heads to keep the GPU busy so the Q>1 K/V-reuse
// benefit is visible. nh KV heads, nr2 query heads each, nr3 batches.
static const int FA_NH = 4;
static const int FA_NR2 = 8;
static const int FA_NR3 = 1;
static const int FA_NH = 4;
static const int FA_NR2 = 8;
static const int FA_NR3 = 1;
struct fa_shape {
int dk;
@@ -34,13 +34,12 @@ static ggml_tensor * fa_build_graph(ggml_context * ctx, const fa_shape & s) {
const int64_t dk_padded = GGML_PAD(s.dk, ggml_blck_size(s.type_kv));
const int64_t dv_padded = GGML_PAD(s.dv, ggml_blck_size(s.type_kv));
ggml_tensor * q = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, dk_padded, s.ne01, FA_NH*FA_NR2, FA_NR3);
ggml_tensor * q = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, dk_padded, s.ne01, FA_NH * FA_NR2, FA_NR3);
ggml_set_name(q, "q");
// K/V are views of a 2x-tall parent, as they are of the KV cache in production
ggml_tensor * k0 = ggml_new_tensor_4d(ctx, s.type_kv, dk_padded, 2*s.ne11, FA_NH, FA_NR3);
ggml_tensor * k = ggml_view_4d(ctx, k0, dk_padded, s.ne11, FA_NH, FA_NR3,
k0->nb[1], k0->nb[2], k0->nb[3], 0);
ggml_tensor * k0 = ggml_new_tensor_4d(ctx, s.type_kv, dk_padded, 2 * s.ne11, FA_NH, FA_NR3);
ggml_tensor * k = ggml_view_4d(ctx, k0, dk_padded, s.ne11, FA_NH, FA_NR3, k0->nb[1], k0->nb[2], k0->nb[3], 0);
ggml_set_name(k, "k");
ggml_tensor * v = nullptr;
@@ -48,16 +47,15 @@ static ggml_tensor * fa_build_graph(ggml_context * ctx, const fa_shape & s) {
// MLA: the V cache is a sub-view of the K cache
v = ggml_view_4d(ctx, k, dv_padded, s.ne11, FA_NH, FA_NR3, k->nb[1], k->nb[2], k->nb[3], 0);
} else {
ggml_tensor * v0 = ggml_new_tensor_4d(ctx, s.type_kv, dv_padded, 2*s.ne11, FA_NH, FA_NR3);
v = ggml_view_4d(ctx, v0, dv_padded, s.ne11, FA_NH, FA_NR3,
v0->nb[1], v0->nb[2], v0->nb[3], 0);
ggml_tensor * v0 = ggml_new_tensor_4d(ctx, s.type_kv, dv_padded, 2 * s.ne11, FA_NH, FA_NR3);
v = ggml_view_4d(ctx, v0, dv_padded, s.ne11, FA_NH, FA_NR3, v0->nb[1], v0->nb[2], v0->nb[3], 0);
}
ggml_set_name(v, "v");
ggml_tensor * m = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, s.ne11, s.ne01, 1, FA_NR3);
ggml_set_name(m, "m");
ggml_tensor * out = ggml_flash_attn_ext(ctx, q, k, v, m, 1.0f/sqrtf((float) s.dk), 0.0f, 0.0f);
ggml_tensor * out = ggml_flash_attn_ext(ctx, q, k, v, m, 1.0f / sqrtf((float) s.dk), 0.0f, 0.0f);
ggml_flash_attn_ext_set_prec(out, GGML_PREC_F32);
ggml_set_name(out, "out");
@@ -66,21 +64,20 @@ static ggml_tensor * fa_build_graph(ggml_context * ctx, const fa_shape & s) {
static uint64_t fa_op_flops(const fa_shape & s) {
// Q*K^T is ne01 x dk x ne11, P*V is ne01 x ne11 x dv, per head
return (uint64_t) 2*FA_NH*FA_NR2*s.ne01*(s.dk + s.dv)*s.ne11*FA_NR3;
return (uint64_t) 2 * FA_NH * FA_NR2 * s.ne01 * (s.dk + s.dv) * s.ne11 * FA_NR3;
}
// mirrors init_tensor_uniform: uniform f32 data, quantized in place for quantized types
static void fa_init_uniform(ggml_tensor * t, std::mt19937 & rng, float min, float max) {
const size_t nels = ggml_nelements(t);
std::vector<float> data(nels);
std::vector<float> data(nels);
std::uniform_real_distribution<float> dist(min, max);
for (size_t i = 0; i < nels; i++) {
data[i] = dist(rng);
}
if (t->type == GGML_TYPE_F32) {
ggml_backend_tensor_set(t, data.data(), 0, nels*sizeof(float));
ggml_backend_tensor_set(t, data.data(), 0, nels * sizeof(float));
return;
}
@@ -88,10 +85,10 @@ static void fa_init_uniform(ggml_tensor * t, std::mt19937 & rng, float min, floa
GGML_ASSERT(nels % ggml_blck_size(t->type) == 0);
std::vector<float> imatrix(t->ne[0], 1.0f);
const float * im = imatrix.data();
const float * im = imatrix.data();
if (!ggml_quantize_requires_imatrix(t->type)) {
// when the imatrix is optional, exercise both paths; pick via one of the random numbers
if (data[0] > 0.5f*(min + max)) {
if (data[0] > 0.5f * (min + max)) {
im = nullptr;
}
}
@@ -116,8 +113,8 @@ static void fa_init_kq_mask(ggml_tensor * t, std::mt19937 & rng, float min, floa
const int32_t ne2 = (int32_t) t->ne[2];
const int32_t ne3 = (int32_t) t->ne[3];
std::vector<float> data_f32(size_t(ne0)*ne1*ne2*ne3);
std::vector<ggml_fp16_t> data_f16(size_t(ne0)*ne1*ne2*ne3);
std::vector<float> data_f32(size_t(ne0) * ne1 * ne2 * ne3);
std::vector<ggml_fp16_t> data_f16(size_t(ne0) * ne1 * ne2 * ne3);
std::uniform_real_distribution<float> dis(min, max);
for (size_t i = 0; i < data_f32.size(); i++) {
@@ -127,7 +124,7 @@ static void fa_init_kq_mask(ggml_tensor * t, std::mt19937 & rng, float min, floa
const int blck0 = 128;
const int blck1 = 64;
const int n_inf_zero_blocks = 0.2*(ne0*ne1*ne2*ne3)/(blck0*blck1);
const int n_inf_zero_blocks = 0.2 * (ne0 * ne1 * ne2 * ne3) / (blck0 * blck1);
for (int b = 0; b < n_inf_zero_blocks; b++) {
const int p3 = (int) (rng() % ne3);
@@ -138,7 +135,7 @@ static void fa_init_kq_mask(ggml_tensor * t, std::mt19937 & rng, float min, floa
const bool inf = rng() & 1;
for (int i1 = 0; i1 < blck1 && p1 + i1 < ne1; i1++) {
const int idx = p3*ne2*ne1*ne0 + p2*ne1*ne0 + (p1 + i1)*ne0 + p0;
const int idx = p3 * ne2 * ne1 * ne0 + p2 * ne1 * ne0 + (p1 + i1) * ne0 + p0;
for (int i0 = 0; i0 < blck0 && p0 + i0 < ne0; i0++) {
data_f32[idx + i0] = inf ? -INFINITY : 0.0f;
@@ -146,17 +143,15 @@ static void fa_init_kq_mask(ggml_tensor * t, std::mt19937 & rng, float min, floa
}
}
ggml_fp32_to_fp16_row(data_f32.data(), data_f16.data(), ne0*ne1*ne2*ne3);
ggml_fp32_to_fp16_row(data_f32.data(), data_f16.data(), ne0 * ne1 * ne2 * ne3);
ggml_backend_tensor_set(t, data_f16.data(), 0, data_f16.size()*sizeof(ggml_fp16_t));
ggml_backend_tensor_set(t, data_f16.data(), 0, data_f16.size() * sizeof(ggml_fp16_t));
}
// per-cell deterministic seed: the shape decides it, so a cell is reproducible
// regardless of what else the sweep visited before it
static unsigned fa_cell_seed(const fa_shape & s, unsigned base) {
unsigned h = base;
for (int v : { s.dk, s.dv, s.ne01, s.ne11, (int) s.type_kv }) {
h = h*1000003u + (unsigned) v; // small prime, standard multiplicative hash mixing
h = h * 1000003u + (unsigned) v;
}
return h;
}
@@ -178,9 +173,9 @@ static void fa_init_tensors(ggml_context * ctx, const fa_shape & s, unsigned bas
using set_override_t = void (*)(int, int);
using clear_override_t = void (*)(void);
using bucket_t = int (*)(int64_t);
using baseline_ne_t = int (*)(int, int);
using device_token_t = const char * (*)(ggml_backend_dev_t);
using bucket_t = int (*)(int64_t);
using baseline_ne_t = int (*)(int, int);
using device_token_t = const char * (*) (ggml_backend_dev_t);
struct fa_procs {
set_override_t set_ov = nullptr;
@@ -190,37 +185,25 @@ struct fa_procs {
baseline_ne_t baseline_ne = nullptr;
device_token_t dev_token = nullptr;
bool ok() const {
return set_ov && clr_ov && ne11_bucket && ne01_bucket && baseline_ne && dev_token;
}
bool ok() const { return set_ov && clr_ov && ne11_bucket && ne01_bucket && baseline_ne && dev_token; }
};
static fa_procs fa_resolve_procs(ggml_backend_dev_t dev) {
ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);
fa_procs p;
p.set_ov = (set_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_set_fa_vec_override");
p.clr_ov = (clear_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_clear_fa_vec_override");
p.ne11_bucket = (bucket_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_ne11_bucket");
p.ne01_bucket = (bucket_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_ne01_bucket");
p.baseline_ne = (baseline_ne_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_baseline_ne");
p.dev_token = (device_token_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_device_token");
p.set_ov = (set_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_set_fa_vec_override");
p.clr_ov =
(clear_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_clear_fa_vec_override");
p.ne11_bucket = (bucket_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_ne11_bucket");
p.ne01_bucket = (bucket_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_ne01_bucket");
p.baseline_ne =
(baseline_ne_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_fa_vec_baseline_ne");
p.dev_token = (device_token_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_device_token");
return p;
}
static const char * fa_type_token(ggml_type t) {
switch (t) {
case GGML_TYPE_Q4_0: return "GGML_TYPE_Q4_0";
case GGML_TYPE_Q4_1: return "GGML_TYPE_Q4_1";
case GGML_TYPE_Q5_0: return "GGML_TYPE_Q5_0";
case GGML_TYPE_Q5_1: return "GGML_TYPE_Q5_1";
case GGML_TYPE_Q8_0: return "GGML_TYPE_Q8_0";
default: GGML_ABORT("unhandled KV type in fa_type_token: %d", (int) t);
}
}
// "f16,q4_0" -> does it contain ggml_type_name(t)? null filter accepts everything
static bool fa_filter_has(const char * filter, const char * name) {
if (!filter) {
return true;
@@ -231,15 +214,16 @@ static bool fa_filter_has(const char * filter, const char * name) {
return f.find(std::string(",") + name + ",") != std::string::npos;
}
struct fa_cand { int Q, NE; };
struct fa_point { // one swept grid point with its candidate times
int dk, dv, ne11, ne01;
std::vector<double> t; // indexed like the shape's candidate list
struct fa_cand {
int Q, NE;
};
// candidate list for one shape, identical for every grid point of it. base_i is the index of
// the (Q=1, baseline NE) candidate: the anchor config, and what the tuning gates compare to.
struct fa_point {
int dk, dv, ne11, ne01;
std::vector<double> t;
};
// base_i identifies the (Q=1, baseline NE) anchor configuration.
static std::vector<fa_cand> fa_build_cands(const fa_procs & procs, int dk, int dv, int & base_i) {
const int base_ne = procs.baseline_ne(dk, dv);
@@ -267,34 +251,56 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune
const char * dev_token = procs.dev_token(dev);
struct shape_t { int dk, dv; };
const shape_t shapes[] = { { 32, 32 }, { 64, 64 }, { 96, 96 }, { 128, 128 }, { 192, 192 },
{ 192, 128 }, { 256, 256 }, { 320, 256 }, { 512, 512 }, { 576, 512 } };
struct shape_t {
int dk, dv;
};
const shape_t shapes[] = {
{ 32, 32 },
{ 64, 64 },
{ 96, 96 },
{ 128, 128 },
{ 192, 192 },
{ 192, 128 },
{ 256, 256 },
{ 320, 256 },
{ 512, 512 },
{ 576, 512 }
};
const int ne11_rep[] = { 512, 2048, 8192, 32768 }; // ne11 bucket representatives
const int ne01_rep[] = { 1, 2, 3, 4, 5, 6, 7, 8, 16 }; // point buckets (1-4) + tail mod-4 cycle + anchor
const ggml_type types[] = { GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1,
GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0 };
struct dtype_t {
ggml_type type;
const char * token;
};
const dtype_t dtypes[] = {
{ GGML_TYPE_F16, "GGML_TYPE_F16" },
{ GGML_TYPE_Q4_0, "GGML_TYPE_Q4_0" },
{ GGML_TYPE_Q4_1, "GGML_TYPE_Q4_1" },
{ GGML_TYPE_Q5_0, "GGML_TYPE_Q5_0" },
{ GGML_TYPE_Q5_1, "GGML_TYPE_Q5_1" },
{ GGML_TYPE_Q8_0, "GGML_TYPE_Q8_0" },
};
const double TUNE_TAU = 0.05; // max POINTWISE regret to ride a domain default
const double TUNE_THETA = 1.05; // min AGGREGATE bucket speedup vs baseline to tune at all
cooldown_opts cool;
cool.enabled = opts.cooldown;
cool.drift = opts.cool_drift;
cool.eps = opts.cool_eps;
cool.max_wait = opts.cool_max_wait;
cool.max_retry = opts.cool_max_retry;
const cooldown_opts cool = {
opts.cooldown, opts.cool_drift, opts.cool_eps, opts.cool_max_wait, opts.cool_max_retry,
};
fprintf(stderr, "seed=%u reps=%d cooldown=%s (drift=%.2f eps=%.2f max_wait=%ds max_retry=%d)\n",
opts.seed, opts.reps, cool.enabled ? "on" : "off",
cool.drift, cool.eps, cool.max_wait, cool.max_retry);
fprintf(stderr, "seed=%u reps=%d cooldown=%s (drift=%.2f eps=%.2f max_wait=%ds max_retry=%d)\n", opts.seed,
opts.reps, cool.enabled ? "on" : "off", cool.drift, cool.eps, cool.max_wait, cool.max_retry);
fprintf(stderr, "device token: %s\n", dev_token);
int n_untrusted = 0;
printf("// ==== BEGIN fa_vec_tuned_table rows (%s) ====\n", dev_token);
for (ggml_type type_kv : types) {
for (const auto & dtype : dtypes) {
const ggml_type type_kv = dtype.type;
if (!fa_filter_has(opts.dtype_filter, ggml_type_name(type_kv))) {
continue;
}
@@ -308,19 +314,19 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune
continue;
}
int base_i = 0;
std::vector<fa_cand> cands = fa_build_cands(procs, s.dk, s.dv, base_i);
int base_i = 0;
std::vector<fa_cand> cands = fa_build_cands(procs, s.dk, s.dv, base_i);
for (int ne11 : ne11_rep) {
for (int ne01 : ne01_rep) {
const fa_shape sh = { s.dk, s.dv, ne01, ne11, type_kv };
perf_cell cell = build_perf_cell(backend,
[&](ggml_context * ctx) { return fa_build_graph(ctx, sh); },
perf_cell cell = build_perf_cell(
backend, [&](ggml_context * ctx) { return fa_build_graph(ctx, sh); },
[&](ggml_context * ctx) { fa_init_tensors(ctx, sh, opts.seed); },
[&](ggml_tensor *) { return fa_op_flops(sh); });
if (!cell.ok) {
if (cell.gf == nullptr) {
continue;
}
@@ -334,22 +340,18 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune
char label[128];
snprintf(label, sizeof(label), "dk=%d ne11=%d", s.dk, ne11);
cell_result r = measure_cell(backend, cell, opts.reps,
(int) cands.size(), order,
[&](int i) { procs.set_ov(cands[i].Q, cands[i].NE); },
[&]() { procs.clr_ov(); },
base_i, cool, label);
cell_result r = measure_cell(
backend, cell, opts.reps, order, [&](int i) { procs.set_ov(cands[i].Q, cands[i].NE); },
[&]() { procs.clr_ov(); }, base_i, cool, label);
// per-cell noise floor: spread of the repeated same-config anchor
if (r.anchor_min > 0.0) {
fprintf(stderr, "# noise dk=%d dv=%d ne11=%d ne01=%d spread=%.1f%%\n",
s.dk, s.dv, ne11, ne01, 100.0*(r.anchor_max - r.anchor_min)/r.anchor_min);
fprintf(stderr, "# noise dk=%d dv=%d ne11=%d ne01=%d spread=%.1f%%\n", s.dk, s.dv, ne11, ne01,
100.0 * (r.anchor_max - r.anchor_min) / r.anchor_min);
}
if (!r.trusted) {
n_untrusted++;
fprintf(stderr, "# DROP untrusted cell dk=%d dv=%d ne11=%d ne01=%d\n",
s.dk, s.dv, ne11, ne01);
fprintf(stderr, "# DROP untrusted cell dk=%d dv=%d ne11=%d ne01=%d\n", s.dk, s.dv, ne11, ne01);
continue;
}
@@ -360,17 +362,17 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune
}
}
const double base_t = r.t[base_i];
const bool keep = best_i >= 0 && base_t > 0.0 && r.t[best_i] < base_t*0.98;
const bool keep = best_i >= 0 && base_t > 0.0 && r.t[best_i] < base_t * 0.98;
fprintf(stderr, "# dtype=%s dk=%d dv=%d ne11=%d ne01=%d:",
ggml_type_name(type_kv), s.dk, s.dv, ne11, ne01);
fprintf(stderr, "# dtype=%s dk=%d dv=%d ne11=%d ne01=%d:", ggml_type_name(type_kv), s.dk, s.dv,
ne11, ne01);
for (size_t i = 0; i < cands.size(); ++i) {
fprintf(stderr, " Q%dNE%d=%.1f%s", cands[i].Q, cands[i].NE, r.t[i],
(int) i == best_i ? "*" : "");
}
if (keep) {
fprintf(stderr, " => Q%d,NE%d %.2fx\n",
cands[best_i].Q, cands[best_i].NE, base_t/r.t[best_i]);
fprintf(stderr, " => Q%d,NE%d %.2fx\n", cands[best_i].Q, cands[best_i].NE,
base_t / r.t[best_i]);
} else {
fprintf(stderr, " => baseline\n");
}
@@ -385,49 +387,52 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune
// the default's pointwise regret vs the bucket target, or its aggregate slowdown vs
// baseline, exceeds TUNE_TAU.
std::vector<std::string> rows_out;
char rbuf[192];
char rbuf[192];
for (auto s : shapes) {
if (!fa_filter_has(opts.dk_filter, std::to_string(s.dk).c_str())) {
continue;
}
int base_i = 0;
std::vector<fa_cand> cands = fa_build_cands(procs, s.dk, s.dv, base_i);
int base_i = 0;
std::vector<fa_cand> cands = fa_build_cands(procs, s.dk, s.dv, base_i);
struct bkt_t {
int b11, b01, Ti;
std::vector<double> agg;
double base_agg;
int b11, b01, Ti;
std::vector<double> agg;
std::vector<const fa_point *> bp;
};
// bucket the grid points with the runtime's bucketers, so keys match fa_vec_pick.
// short-KV points (ne11 bucket 0) are dropped: the runtime serves those from baseline.
std::set<std::pair<int, int>> seen;
for (const auto & p : pts) {
if (p.dk != s.dk || p.dv != s.dv) {
continue;
}
const int b11 = procs.ne11_bucket(p.ne11);
std::set<std::pair<int, int>> buckets;
for (int ne11 : ne11_rep) {
const int b11 = procs.ne11_bucket(ne11);
if (b11 == 0) {
continue;
}
seen.insert({ b11, procs.ne01_bucket(p.ne01) });
for (int ne01 : ne01_rep) {
buckets.insert({ b11, procs.ne01_bucket(ne01) });
}
}
std::vector<bkt_t> bks;
for (const auto & bb : seen) {
for (const auto & bb : buckets) {
const int b11 = bb.first, b01 = bb.second;
std::vector<const fa_point *> bp;
for (const auto & p : pts) {
if (p.dk == s.dk && p.dv == s.dv &&
procs.ne11_bucket(p.ne11) == b11 && procs.ne01_bucket(p.ne01) == b01) {
if (p.dk == s.dk && p.dv == s.dv && procs.ne11_bucket(p.ne11) == b11 &&
procs.ne01_bucket(p.ne01) == b01) {
bp.push_back(&p);
}
}
fprintf(stderr, "# bucket dk=%d dv=%d ne11_b=%d ne01_b=%d samples=%zu\n", s.dk, s.dv, b11, b01,
bp.size());
if (bp.empty()) {
fprintf(stderr, "# WARN empty bucket dk=%d dv=%d ne11_b=%d ne01_b=%d\n", s.dk, s.dv, b11, b01);
continue;
}
std::vector<double> agg(cands.size(), 0.0), worst(cands.size(), 0.0);
for (const auto * p : bp) {
double bestt = 0.0;
@@ -439,35 +444,24 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune
for (size_t i = 0; i < cands.size(); ++i) {
agg[i] += p->t[i];
if (p->t[i] > 0.0 && bestt > 0.0) {
worst[i] = std::max(worst[i], p->t[i]/bestt);
worst[i] = std::max(worst[i], p->t[i] / bestt);
}
}
}
int robust = 0;
for (size_t i = 1; i < cands.size(); ++i) {
if (worst[i] < worst[robust] ||
(worst[i] == worst[robust] && (cands[i].Q < cands[robust].Q ||
(cands[i].Q == cands[robust].Q && cands[i].NE < cands[robust].NE)))) {
if (worst[i] < worst[robust] || (worst[i] == worst[robust] && (cands[i].Q < cands[robust].Q ||
(cands[i].Q == cands[robust].Q &&
cands[i].NE < cands[robust].NE)))) {
robust = (int) i;
}
}
const bool tune = robust != base_i && agg[base_i] > 0.0 && agg[robust] > 0.0 &&
agg[base_i]/agg[robust] >= TUNE_THETA;
agg[base_i] / agg[robust] >= TUNE_THETA;
bks.push_back({ b11, b01, tune ? robust : base_i, agg, agg[base_i], bp });
}
// bucket coverage: a hardcoded sampling grid can't produce a wrong key, only miss
// a bucket, so report what each bucket actually got
for (const auto & b : bks) {
fprintf(stderr, "# bucket dk=%d dv=%d ne11_b=%d ne01_b=%d samples=%zu\n",
s.dk, s.dv, b.b11, b.b01, b.bp.size());
if (b.bp.empty()) {
fprintf(stderr, "# WARN empty bucket dk=%d dv=%d ne11_b=%d ne01_b=%d\n",
s.dk, s.dv, b.b11, b.b01);
}
bks.push_back({ b11, b01, tune ? robust : base_i, agg, bp });
}
// pointwise regret of default cfg d vs the bucket target: a ratio-of-sums lets a
@@ -477,7 +471,7 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune
for (const auto * p : b->bp) {
const double td = p->t[d], tT = p->t[b->Ti];
if (td > 0.0 && tT > 0.0) {
r = std::max(r, td/tT - 1.0);
r = std::max(r, td / tT - 1.0);
}
}
return r;
@@ -495,14 +489,15 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune
}
// default cfg = the one minimizing (#rows, total achieved time, Q, NE)
int bestD = -1, bestRows = 1 << 30;
int bestD = -1, bestRows = 1 << 30;
double bestTot = 0.0;
for (size_t d = 0; d < cands.size(); ++d) {
int rows = ((int) d != base_i) ? 1 : 0;
double tot = 0.0;
int rows = ((int) d != base_i) ? 1 : 0;
double tot = 0.0;
for (const auto * b : db) {
const double reg = reg_pointwise(b, (int) d);
const double slow = b->base_agg > 0.0 ? b->agg[d]/b->base_agg - 1.0 : 0.0;
const double base_agg = b->agg[base_i];
const double reg = reg_pointwise(b, (int) d);
const double slow = base_agg > 0.0 ? b->agg[d] / base_agg - 1.0 : 0.0;
if (reg > TUNE_TAU || slow > TUNE_TAU) {
rows++;
tot += b->agg[b->Ti];
@@ -510,33 +505,33 @@ bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tune
tot += b->agg[d];
}
}
const bool better = bestD < 0 || rows < bestRows ||
(rows == bestRows && (tot < bestTot ||
(tot == bestTot && (cands[d].Q < cands[bestD].Q ||
(cands[d].Q == cands[bestD].Q && cands[d].NE < cands[bestD].NE)))));
const bool better =
bestD < 0 || rows < bestRows ||
(rows == bestRows &&
(tot < bestTot ||
(tot == bestTot && (cands[d].Q < cands[bestD].Q ||
(cands[d].Q == cands[bestD].Q && cands[d].NE < cands[bestD].NE)))));
if (better) {
bestD = (int) d;
bestD = (int) d;
bestRows = rows;
bestTot = tot;
bestTot = tot;
}
}
const int dom_id = (dom == 0) ? 0 : 1; // FA_VEC_DOMAIN_DECODE / FA_VEC_DOMAIN_BATCH
if (bestD != base_i) {
snprintf(rbuf, sizeof(rbuf), " { { %s, %s, %d, %d, -1, %d }, { %d, %d } },",
dev_token, fa_type_token(type_kv), s.dk, s.dv, dom_id,
cands[bestD].Q, cands[bestD].NE);
snprintf(rbuf, sizeof(rbuf), " { { %s, %s, %d, %d, -1, %d }, { %d, %d } },", dev_token,
dtype.token, s.dk, s.dv, dom, cands[bestD].Q, cands[bestD].NE);
rows_out.emplace_back(rbuf);
}
for (const auto * b : db) {
const double reg = reg_pointwise(b, bestD);
const double slow = b->base_agg > 0.0 ? b->agg[bestD]/b->base_agg - 1.0 : 0.0;
const double base_agg = b->agg[base_i];
const double reg = reg_pointwise(b, bestD);
const double slow = base_agg > 0.0 ? b->agg[bestD] / base_agg - 1.0 : 0.0;
if (reg <= TUNE_TAU && slow <= TUNE_TAU) {
continue; // rides the default / baseline
continue;
}
snprintf(rbuf, sizeof(rbuf), " { { %s, %s, %d, %d, %d, %d }, { %d, %d } },",
dev_token, fa_type_token(type_kv), s.dk, s.dv, b->b11, b->b01,
cands[b->Ti].Q, cands[b->Ti].NE);
snprintf(rbuf, sizeof(rbuf), " { { %s, %s, %d, %d, %d, %d }, { %d, %d } },", dev_token,
dtype.token, s.dk, s.dv, b->b11, b->b01, cands[b->Ti].Q, cands[b->Ti].NE);
rows_out.emplace_back(rbuf);
}
}
tools/tuning/fa-vec.h
+9 -11
View File
@@ -2,19 +2,17 @@
#include "ggml-backend.h"
// options shared by all tuners; parsed in main.cpp
struct tuner_opts {
const char * dtype_filter = nullptr; // comma-separated, e.g. "f16,q4_0"; null = all
const char * dk_filter = nullptr; // comma-separated dk values, e.g. "128,192"; null = all
int reps = 7;
unsigned seed = 1234;
bool cooldown = true;
double cool_drift = 0.10;
double cool_eps = 0.03;
int cool_max_wait = 120;
const char * dtype_filter = nullptr; // comma-separated, e.g. "f16,q4_0"; null = all
const char * dk_filter = nullptr; // comma-separated dk values, e.g. "128,192"; null = all
int reps = 7;
unsigned seed = 1234;
bool cooldown = true;
double cool_drift = 0.10;
double cool_eps = 0.03;
int cool_max_wait = 120;
int cool_max_retry = 2;
};
// runs the FA-vec (Q,NE) sweep and prints a pasteable table block on stdout.
// returns false only on environment failure (missing procs), never on perf results.
// Returns false only when the required Metal proc bridges are unavailable.
bool tuner_fa_vec_run(ggml_backend_t backend, ggml_backend_dev_t dev, const tuner_opts & opts);
tools/tuning/main.cpp
+3 -4
View File
@@ -1,7 +1,6 @@
#include "fa-vec.h"
#include "ggml.h"
#include "ggml-backend.h"
#include "ggml.h"
#include <cstdio>
#include <cstdlib>
@@ -42,8 +41,8 @@ static void usage(const char * argv0) {
}
int main(int argc, char ** argv) {
const char * tuner = nullptr;
const char * bname = nullptr;
const char * tuner = nullptr;
const char * bname = nullptr;
tuner_opts opts;
for (int i = 1; i < argc; i++) {