mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-08-12 22:31:11 +04:00
kleidiai: Add runtime feature detection mechanism for aarch64/kleidiai (#26076)
* Add runtime feature detection mechanism for aarch64/kleidiai Signed-off-by: Jonathan Clohessy <Jonathan.Clohessy@arm.com> * Address Review Comments Signed-off-by: Jonathan Clohessy <Jonathan.Clohessy@arm.com> * Add log warning for NSMC reserved value Signed-off-by: Jonathan Clohessy <Jonathan.Clohessy@arm.com> * Address review comments Signed-off-by: Jonathan Clohessy <Jonathan.Clohessy@arm.com> * Fix Rebase, move code from cpu-feats to ggml-feats Signed-off-by: Jonathan Clohessy <Jonathan.Clohessy@arm.com> * Address naming of runtime feature struct Signed-off-by: Jonathan Clohessy <Jonathan.Clohessy@arm.com> --------- Signed-off-by: Jonathan Clohessy <Jonathan.Clohessy@arm.com>
This commit is contained in:
Jonathan Clohessy
@@ -1,90 +1,11 @@
|
||||
#include "ggml-backend-impl.h"
|
||||
#include "ggml-feats.h"
|
||||
|
||||
#if defined(__aarch64__)
|
||||
|
||||
#if defined(__linux__)
|
||||
#include <sys/auxv.h>
|
||||
#elif defined(__APPLE__)
|
||||
#include <sys/sysctl.h>
|
||||
#endif
|
||||
|
||||
#if !defined(HWCAP_FPHP)
|
||||
#define HWCAP_FPHP (1 << 9)
|
||||
#endif
|
||||
|
||||
#if !defined(HWCAP_ASIMDHP)
|
||||
#define HWCAP_ASIMDHP (1 << 10)
|
||||
#endif
|
||||
|
||||
#if !defined(HWCAP_ASIMDDP)
|
||||
#define HWCAP_ASIMDDP (1 << 20)
|
||||
#endif
|
||||
|
||||
#if !defined(HWCAP_SVE)
|
||||
#define HWCAP_SVE (1 << 22)
|
||||
#endif
|
||||
|
||||
#if !defined(HWCAP2_SVE2)
|
||||
#define HWCAP2_SVE2 (1 << 1)
|
||||
#endif
|
||||
|
||||
#if !defined(HWCAP2_I8MM)
|
||||
#define HWCAP2_I8MM (1 << 13)
|
||||
#endif
|
||||
|
||||
#if !defined(HWCAP2_SME)
|
||||
#define HWCAP2_SME (1 << 23)
|
||||
#endif
|
||||
|
||||
struct aarch64_features {
|
||||
// has_neon not needed, aarch64 has NEON guaranteed
|
||||
bool has_dotprod = false;
|
||||
bool has_fp16 = false;
|
||||
bool has_sve = false;
|
||||
bool has_sve2 = false;
|
||||
bool has_i8mm = false;
|
||||
bool has_sme = false;
|
||||
bool has_sme2 = false;
|
||||
|
||||
aarch64_features() {
|
||||
#if defined(__linux__)
|
||||
uint32_t hwcap = getauxval(AT_HWCAP);
|
||||
uint32_t hwcap2 = getauxval(AT_HWCAP2);
|
||||
|
||||
has_dotprod = !!(hwcap & HWCAP_ASIMDDP);
|
||||
has_fp16 = !!(hwcap & HWCAP_FPHP) && !!(hwcap & HWCAP_ASIMDHP);
|
||||
has_sve = !!(hwcap & HWCAP_SVE);
|
||||
has_sve2 = !!(hwcap2 & HWCAP2_SVE2);
|
||||
has_i8mm = !!(hwcap2 & HWCAP2_I8MM);
|
||||
has_sme = !!(hwcap2 & HWCAP2_SME);
|
||||
#elif defined(__APPLE__)
|
||||
int oldp = 0;
|
||||
size_t size = sizeof(oldp);
|
||||
|
||||
if (sysctlbyname("hw.optional.arm.FEAT_DotProd", &oldp, &size, NULL, 0) == 0) {
|
||||
has_dotprod = static_cast<bool>(oldp);
|
||||
}
|
||||
|
||||
if (sysctlbyname("hw.optional.arm.FEAT_I8MM", &oldp, &size, NULL, 0) == 0) {
|
||||
has_i8mm = static_cast<bool>(oldp);
|
||||
}
|
||||
|
||||
if (sysctlbyname("hw.optional.arm.FEAT_SME", &oldp, &size, NULL, 0) == 0) {
|
||||
has_sme = static_cast<bool>(oldp);
|
||||
}
|
||||
|
||||
if (sysctlbyname("hw.optional.arm.FEAT_SME2", &oldp, &size, NULL, 0) == 0) {
|
||||
has_sme2 = static_cast<bool>(oldp);
|
||||
}
|
||||
|
||||
// Apple apparently does not implement SVE yet
|
||||
#endif
|
||||
}
|
||||
};
|
||||
#if defined(__aarch64__) || defined(_M_ARM64)
|
||||
|
||||
static int ggml_backend_cpu_aarch64_score() {
|
||||
int score = 1;
|
||||
aarch64_features af;
|
||||
ggml_feats_arch64_runtime_t af = ggml_get_aarch64_runtime_features();
|
||||
|
||||
#ifdef GGML_USE_DOTPROD
|
||||
if (!af.has_dotprod) { return 0; }
|
||||
@@ -116,4 +37,4 @@ static int ggml_backend_cpu_aarch64_score() {
|
||||
|
||||
GGML_BACKEND_DL_SCORE_IMPL(ggml_backend_cpu_aarch64_score)
|
||||
|
||||
# endif // defined(__aarch64__)
|
||||
# endif // defined(__aarch64__) || defined(_M_ARM64)
|
||||
|
||||
@@ -2,10 +2,12 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
//
|
||||
#include <arm_neon.h>
|
||||
#include <assert.h>
|
||||
#include <stdio.h>
|
||||
#include <cassert>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <atomic>
|
||||
#include <cfloat>
|
||||
#include <cctype>
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <stdexcept>
|
||||
@@ -17,25 +19,21 @@
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <fstream>
|
||||
#include <set>
|
||||
#include <map>
|
||||
#include <iostream>
|
||||
#include <climits>
|
||||
#include <charconv>
|
||||
#include <system_error>
|
||||
#if defined(__linux__)
|
||||
#include <asm/hwcap.h>
|
||||
#include <dirent.h>
|
||||
#include <sys/auxv.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
#ifndef HWCAP2_SME2
|
||||
#define HWCAP2_SME2 (1UL << 37)
|
||||
#endif
|
||||
#elif defined(__APPLE__)
|
||||
#include <string_view>
|
||||
#include <sys/sysctl.h>
|
||||
#include <sys/types.h>
|
||||
#elif defined(_WIN32)
|
||||
#include <windows.h>
|
||||
#include <excpt.h>
|
||||
#endif
|
||||
|
||||
#include "kleidiai.h"
|
||||
@@ -43,6 +41,7 @@
|
||||
#include "ggml-cpu.h"
|
||||
#include "ggml-cpu-impl.h"
|
||||
#include "ggml-impl.h"
|
||||
#include "ggml-feats.h"
|
||||
#include "ggml-backend-impl.h"
|
||||
#include "ggml-threading.h"
|
||||
#include "traits.h"
|
||||
@@ -64,8 +63,8 @@ struct ggml_kleidiai_context {
|
||||
ggml_kleidiai_kernels * kernels_q4;
|
||||
ggml_kleidiai_kernels * kernels_q8;
|
||||
ggml_kleidiai_kernels * kernels_f32;
|
||||
int sme_thread_cap; // <= 0 means “SME disabled/unknown”;
|
||||
int thread_hint; // <= 0 means “no hint”
|
||||
int sme_thread_cap; // <= 0 means "SME disabled/unknown"
|
||||
int thread_hint; // <= 0 means "no hint"
|
||||
int chunk_multiplier;
|
||||
} static ctx = { CPU_FEATURE_NONE, nullptr, nullptr, nullptr, 0, -1, 4 };
|
||||
|
||||
@@ -93,24 +92,117 @@ static const char* cpu_feature_to_string(cpu_feature f) {
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(__linux__) && defined(__aarch64__)
|
||||
static bool parse_cpu_dir_name(const char* name, size_t* cpu) {
|
||||
if (strncmp(name, "cpu", 3) != 0 ||
|
||||
name[3] < '0' || name[3] > '9') {
|
||||
return false;
|
||||
}
|
||||
|
||||
const char* first = name + 3;
|
||||
const char* last = name + strlen(name);
|
||||
|
||||
size_t value = 0;
|
||||
const auto [end, ec] = std::from_chars(first, last, value, 10);
|
||||
|
||||
if (ec != std::errc{} || end != last) {
|
||||
return false;
|
||||
}
|
||||
|
||||
*cpu = value;
|
||||
return true;
|
||||
}
|
||||
|
||||
static std::vector<size_t> detect_cpu_ids() {
|
||||
std::vector<size_t> cpus;
|
||||
|
||||
DIR * dir = opendir("/sys/devices/system/cpu");
|
||||
if (dir == nullptr) {
|
||||
return cpus;
|
||||
}
|
||||
|
||||
while (dirent * entry = readdir(dir)) {
|
||||
size_t cpu = 0;
|
||||
if (parse_cpu_dir_name(entry->d_name, &cpu)) {
|
||||
cpus.push_back(cpu);
|
||||
}
|
||||
}
|
||||
closedir(dir);
|
||||
|
||||
std::sort(cpus.begin(), cpus.end());
|
||||
cpus.erase(std::unique(cpus.begin(), cpus.end()), cpus.end());
|
||||
return cpus;
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(__APPLE__) && defined(__aarch64__)
|
||||
static bool apple_sme_counted_perf_level(std::string name) {
|
||||
for (std::string::size_type i = 0; i < name.size(); ++i) {
|
||||
name[i] = (char) std::tolower((unsigned char) name[i]);
|
||||
}
|
||||
|
||||
// Conservative ceiling: only count perf-level names observed to provide full SME throughput.
|
||||
// Future names should be calibrated here before they raise the automatic SME thread cap.
|
||||
return name.find("super") != std::string::npos ||
|
||||
name.find("performance") != std::string::npos;
|
||||
}
|
||||
#endif
|
||||
|
||||
static void add_smcus_from_smidr(uint64_t smidr, size_t & num_private, std::map<uint32_t, size_t> & shared_counts) {
|
||||
// Arm ARM: SMIDR_EL1. SH==0 is implementation-defined; keep the existing
|
||||
// conservative policy and only treat zero affinity as private.
|
||||
const uint32_t sh = (uint32_t)((smidr >> 13) & 0x3);
|
||||
const uint32_t nsmc = (uint32_t)((smidr >> 56) & 0xF);
|
||||
const size_t shared_count = nsmc == 0xF ? 1 : (size_t)nsmc + 1;
|
||||
const uint32_t affinity = (uint32_t)(smidr & 0xFFFu);
|
||||
const uint32_t affinity2 = (uint32_t)((smidr >> 32) & 0xFFFFFu);
|
||||
const uint32_t id = (affinity2 << 12) | affinity;
|
||||
|
||||
if (nsmc == 0xF) {
|
||||
GGML_LOG_WARN("kleidiai: NSMC detected as 0xF indicating reseved value, setting min safe shared SMCU count to 1");
|
||||
}
|
||||
|
||||
switch (sh) {
|
||||
case 2: // private SMCU
|
||||
++num_private;
|
||||
break;
|
||||
case 3: // shared SMCU
|
||||
if (shared_counts[id] < shared_count) {
|
||||
shared_counts[id] = shared_count;
|
||||
}
|
||||
break;
|
||||
case 0:
|
||||
if (id == 0) {
|
||||
++num_private;
|
||||
} else if (shared_counts[id] < shared_count) {
|
||||
shared_counts[id] = shared_count;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
static size_t detect_num_smcus() {
|
||||
if (!ggml_cpu_has_sme()) {
|
||||
auto runtime_feat = ggml_get_aarch64_runtime_features();
|
||||
if (!runtime_feat.has_sme) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
#if defined(__linux__) && defined(__aarch64__)
|
||||
// Linux/aarch64: Best-effort count of Streaming Mode Compute Units (SMCUs) via SMIDR_EL1 sysfs.
|
||||
size_t num_private = 0;
|
||||
std::set<uint32_t> shared_ids;
|
||||
std::map<uint32_t, size_t> shared_counts;
|
||||
|
||||
for (size_t cpu = 0;; ++cpu) {
|
||||
const std::vector<size_t> cpus = detect_cpu_ids();
|
||||
for (const size_t cpu : cpus) {
|
||||
const std::string path =
|
||||
"/sys/devices/system/cpu/cpu" + std::to_string(cpu) +
|
||||
"/regs/identification/smidr_el1";
|
||||
|
||||
std::ifstream file(path);
|
||||
if (!file.is_open()) {
|
||||
break;
|
||||
continue;
|
||||
}
|
||||
|
||||
uint64_t smidr = 0;
|
||||
@@ -118,54 +210,69 @@ static size_t detect_num_smcus() {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Arm ARM: SMIDR_EL1
|
||||
const uint32_t sh = (uint32_t)((smidr >> 13) & 0x3);
|
||||
// Build an "affinity-like" identifier for shared SMCUs.
|
||||
// Keep the original packing logic, but isolate it here.
|
||||
const uint32_t id = (uint32_t)((smidr & 0xFFFu) | ((smidr >> 20) & 0xFFFFF000u));
|
||||
|
||||
switch (sh) {
|
||||
case 0b10: // private SMCU
|
||||
++num_private;
|
||||
break;
|
||||
case 0b11: // shared SMCU
|
||||
shared_ids.emplace(id);
|
||||
break;
|
||||
case 0b00:
|
||||
// Ambiguous / implementation-defined. Be conservative:
|
||||
// treat id==0 as private, otherwise as shared.
|
||||
if (id == 0) ++num_private;
|
||||
else shared_ids.emplace(id);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
add_smcus_from_smidr(smidr, num_private, shared_counts);
|
||||
}
|
||||
|
||||
return num_private + shared_ids.size();
|
||||
size_t total = num_private;
|
||||
for (const auto & entry : shared_counts) {
|
||||
total += entry.second;
|
||||
}
|
||||
return total;
|
||||
|
||||
#elif defined(__APPLE__) && defined(__aarch64__)
|
||||
// table for known M4 variants. Users can override via GGML_KLEIDIAI_SME=<n>.
|
||||
char chip_name[256] = {};
|
||||
size_t size = sizeof(chip_name);
|
||||
int perf_levels = 0;
|
||||
size_t size = sizeof(perf_levels);
|
||||
if (sysctlbyname("hw.nperflevels", &perf_levels, &size, nullptr, 0) != 0 ||
|
||||
size != sizeof(perf_levels) || perf_levels <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (sysctlbyname("machdep.cpu.brand_string", chip_name, &size, nullptr, 0) == 0) {
|
||||
const std::string brand(chip_name);
|
||||
size_t units = 0;
|
||||
for (int i = 0; i < perf_levels; ++i) {
|
||||
char key[64] = {};
|
||||
int physical_cpus = 0;
|
||||
int cpus_per_l2 = 0;
|
||||
|
||||
struct ModelSMCU { const char *match; size_t smcus; };
|
||||
static const ModelSMCU table[] = {
|
||||
{ "M4 Ultra", 2 },
|
||||
{ "M4 Max", 2 },
|
||||
{ "M4 Pro", 2 },
|
||||
{ "M4", 1 },
|
||||
};
|
||||
snprintf(key, sizeof(key), "hw.perflevel%d.physicalcpu", i);
|
||||
size = sizeof(physical_cpus);
|
||||
if (sysctlbyname(key, &physical_cpus, &size, nullptr, 0) != 0 ||
|
||||
size != sizeof(physical_cpus) || physical_cpus <= 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const auto &e : table) {
|
||||
if (brand.find(e.match) != std::string::npos) {
|
||||
return e.smcus;
|
||||
}
|
||||
snprintf(key, sizeof(key), "hw.perflevel%d.cpusperl2", i);
|
||||
size = sizeof(cpus_per_l2);
|
||||
if (sysctlbyname(key, &cpus_per_l2, &size, nullptr, 0) != 0 ||
|
||||
size != sizeof(cpus_per_l2) || cpus_per_l2 <= 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
snprintf(key, sizeof(key), "hw.perflevel%d.name", i);
|
||||
size = 0;
|
||||
if (sysctlbyname(key, nullptr, &size, nullptr, 0) != 0 || size == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
std::string name(size, '\0');
|
||||
if (sysctlbyname(key, &name[0], &size, nullptr, 0) != 0) {
|
||||
continue;
|
||||
}
|
||||
name.resize(size);
|
||||
while (!name.empty() && name.back() == '\0') {
|
||||
name.pop_back();
|
||||
}
|
||||
|
||||
if (apple_sme_counted_perf_level(name)) {
|
||||
units += (size_t) ((physical_cpus + cpus_per_l2 - 1) / cpus_per_l2);
|
||||
}
|
||||
}
|
||||
|
||||
return units;
|
||||
|
||||
#elif defined(_WIN32) && (defined(_M_ARM64) || defined(__aarch64__))
|
||||
// No verified Windows arm64 SMCU detection path yet. Return unknown and use
|
||||
// GGML_KLEIDIAI_SME=N as a diagnostics/debug override for SME thread cap
|
||||
// calibration until a detection mechanism is verified on real hardware.
|
||||
return 0;
|
||||
|
||||
#else
|
||||
@@ -198,15 +305,18 @@ static void init_kleidiai_context(void) {
|
||||
if (!initialized) {
|
||||
initialized = true;
|
||||
|
||||
// Optional diagnostics/debug overrides; production defaults come from runtime detection.
|
||||
const char *env_sme = getenv("GGML_KLEIDIAI_SME");
|
||||
const char *env_threads = getenv("GGML_TOTAL_THREADS");
|
||||
const char *env_chunk_mult = getenv("GGML_KLEIDIAI_CHUNK_MULTIPLIER");
|
||||
|
||||
auto runtime_feat = ggml_get_aarch64_runtime_features();
|
||||
|
||||
size_t detected_smcus = 0;
|
||||
|
||||
ctx.features = (ggml_cpu_has_dotprod() ? CPU_FEATURE_DOTPROD : CPU_FEATURE_NONE) |
|
||||
(ggml_cpu_has_matmul_int8() ? CPU_FEATURE_I8MM : CPU_FEATURE_NONE) |
|
||||
((ggml_cpu_has_sve() && ggml_cpu_get_sve_cnt() == QK8_0) ? CPU_FEATURE_SVE : CPU_FEATURE_NONE);
|
||||
ctx.features = (runtime_feat.has_dotprod ? CPU_FEATURE_DOTPROD : CPU_FEATURE_NONE) |
|
||||
(runtime_feat.has_i8mm ? CPU_FEATURE_I8MM : CPU_FEATURE_NONE) |
|
||||
(runtime_feat.sve_cnt == QK8_0 ? CPU_FEATURE_SVE : CPU_FEATURE_NONE);
|
||||
|
||||
if (env_threads) {
|
||||
bool ok = false;
|
||||
@@ -224,54 +334,54 @@ static void init_kleidiai_context(void) {
|
||||
}
|
||||
}
|
||||
|
||||
// SME policy:
|
||||
// - env unset => auto-detect SMCUs; enable SME only if detected > 0.
|
||||
// - env=0 => force off.
|
||||
// - env>0 => force N cores, if the binary was built with SME.
|
||||
int sme_cores = 0;
|
||||
bool sme_env_ok = false;
|
||||
bool sme_env_set = (env_sme != nullptr);
|
||||
|
||||
const bool has_supported_sme_family = runtime_feat.has_sme;
|
||||
bool sme_cap_detected = false;
|
||||
|
||||
if (has_supported_sme_family) {
|
||||
detected_smcus = detect_num_smcus();
|
||||
sme_cap_detected = detected_smcus > 0;
|
||||
// Some platforms expose SME without exposing a calibrated SMCU count.
|
||||
// Use one SME thread as the conservative default; add platform SMCU detection to raise it.
|
||||
sme_cores = sme_cap_detected ? (int)detected_smcus : 1;
|
||||
|
||||
if (!sme_env_set && !sme_cap_detected) {
|
||||
GGML_LOG_INFO("kleidiai: SME detected; SMCU count unavailable, using conservative SME thread cap=1\n");
|
||||
}
|
||||
}
|
||||
|
||||
// Runtime-detect SME support and available SMCUs first. The detected SMCU
|
||||
// count is used as the SME thread cap, and GGML_KLEIDIAI_SME can debug-override that:
|
||||
// - unset: use runtime detection.
|
||||
// - 0: disable SME-family kernels.
|
||||
// - N > 0: use N as the SME thread cap, if an SME-family kernel is selectable.
|
||||
if (sme_env_set) {
|
||||
bool ok = false;
|
||||
int v = parse_uint_env(env_sme, "GGML_KLEIDIAI_SME", &ok);
|
||||
sme_env_ok = ok;
|
||||
|
||||
if (!ok) {
|
||||
GGML_LOG_WARN("kleidiai: GGML_KLEIDIAI_SME set but parsing failed; falling back to runtime SME-core detection\n");
|
||||
detected_smcus = detect_num_smcus();
|
||||
sme_cores = detected_smcus > 0 ? (int)detected_smcus : 0;
|
||||
} else if (v == 0) {
|
||||
sme_cores = 0;
|
||||
} else if (!ggml_cpu_has_sme()) {
|
||||
GGML_LOG_WARN("kleidiai: GGML_KLEIDIAI_SME=%d but the binary was not built with SME; disabling SME\n", v);
|
||||
sme_cores = 0;
|
||||
if (ok) {
|
||||
if (has_supported_sme_family) {
|
||||
sme_cores = v;
|
||||
} else {
|
||||
if (v > 0) {
|
||||
GGML_LOG_WARN("kleidiai: GGML_KLEIDIAI_SME=%d but SME is not supported on this CPU; disabling SME-family kernels\n", v);
|
||||
}
|
||||
sme_cores = 0;
|
||||
}
|
||||
} else {
|
||||
sme_cores = v;
|
||||
GGML_LOG_WARN("kleidiai: GGML_KLEIDIAI_SME set but parsing failed; using automatic SME thread cap\n");
|
||||
}
|
||||
} else {
|
||||
detected_smcus = detect_num_smcus();
|
||||
sme_cores = detected_smcus > 0 ? (int)detected_smcus : 0;
|
||||
}
|
||||
|
||||
if (!sme_env_set && ggml_cpu_has_sme() && sme_cores == 0) {
|
||||
GGML_LOG_WARN("kleidiai: runtime SME-core detection returned 0; falling back to NEON\n");
|
||||
}
|
||||
|
||||
if (sme_cores > 0) {
|
||||
if (sme_cores > 0 && has_supported_sme_family) {
|
||||
ctx.features |= CPU_FEATURE_SME;
|
||||
#if defined(__aarch64__) && defined(__linux__)
|
||||
// ARM guarantees SME2 implies SME, so only check SME2 when SME is enabled.
|
||||
if (getauxval(AT_HWCAP2) & HWCAP2_SME2) {
|
||||
if (runtime_feat.has_sme2) {
|
||||
ctx.features |= CPU_FEATURE_SME2;
|
||||
}
|
||||
#elif defined(__aarch64__) && defined(__APPLE__)
|
||||
int feat_sme2 = 0;
|
||||
size_t size = sizeof(feat_sme2);
|
||||
if (sysctlbyname("hw.optional.arm.FEAT_SME2", &feat_sme2, &size, NULL, 0) == 0 && feat_sme2) {
|
||||
ctx.features |= CPU_FEATURE_SME2;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
// Kernel selection
|
||||
@@ -297,16 +407,19 @@ static void init_kleidiai_context(void) {
|
||||
GGML_LOG_INFO("kleidiai: primary f32 kernel feature %s\n", cpu_feature_to_string(ctx.kernels_f32->required_cpu));
|
||||
}
|
||||
|
||||
ctx.sme_thread_cap = (ctx.features & CPU_FEATURE_SME) ? sme_cores : 0;
|
||||
const bool has_selected_sme_family_kernel =
|
||||
(ctx.kernels_q4 && is_sme_family(ctx.kernels_q4->required_cpu)) ||
|
||||
(ctx.kernels_q8 && is_sme_family(ctx.kernels_q8->required_cpu)) ||
|
||||
(ctx.kernels_f32 && is_sme_family(ctx.kernels_f32->required_cpu));
|
||||
ctx.sme_thread_cap = has_selected_sme_family_kernel ? sme_cores : 0;
|
||||
|
||||
if (ctx.features & CPU_FEATURE_SME) {
|
||||
const bool has_sme2 = (ctx.features & CPU_FEATURE_SME2) != CPU_FEATURE_NONE;
|
||||
if (has_selected_sme_family_kernel) {
|
||||
if (sme_env_set && sme_env_ok && sme_cores > 0) {
|
||||
GGML_LOG_INFO("kleidiai: SME%s enabled (GGML_KLEIDIAI_SME=%d override)\n",
|
||||
has_sme2 ? "2" : "", sme_cores);
|
||||
GGML_LOG_INFO("kleidiai: SME enabled (GGML_KLEIDIAI_SME=%d debug override)\n", sme_cores);
|
||||
} else if (sme_cap_detected) {
|
||||
GGML_LOG_INFO("kleidiai: SME enabled (runtime-detected SME thread cap=%d)\n", sme_cores);
|
||||
} else {
|
||||
GGML_LOG_INFO("kleidiai: SME%s enabled (runtime-detected SME cores=%d)\n",
|
||||
has_sme2 ? "2" : "", sme_cores);
|
||||
GGML_LOG_INFO("kleidiai: SME enabled (runtime SME detected, conservative thread cap=%d)\n", sme_cores);
|
||||
}
|
||||
} else {
|
||||
GGML_LOG_INFO("kleidiai: SME disabled\n");
|
||||
@@ -467,7 +580,7 @@ static int kleidiai_collect_kernel_chain_common(
|
||||
}
|
||||
|
||||
if (is_sme_family(primary->required_cpu)) {
|
||||
const cpu_feature fallback_mask = static_cast<cpu_feature>(features & ~CPU_FEATURE_SME & ~CPU_FEATURE_SME2);
|
||||
const cpu_feature fallback_mask = static_cast<cpu_feature>(features & ~(CPU_FEATURE_SME | CPU_FEATURE_SME2));
|
||||
if (fallback_mask != CPU_FEATURE_NONE) {
|
||||
ggml_kleidiai_kernels * fallback = select_fallback(fallback_mask);
|
||||
if (fallback && fallback != primary &&
|
||||
@@ -1077,13 +1190,14 @@ class tensor_traits : public ggml::cpu::tensor_traits {
|
||||
const int ith_total = params->ith;
|
||||
|
||||
int sme_slot = -1;
|
||||
int non_sme_slot = -1;
|
||||
for (int i = 0; i < runtime_count; ++i) {
|
||||
if (is_sme_family(runtime[i].kernels->required_cpu)) {
|
||||
sme_slot = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
int non_sme_slot = -1;
|
||||
|
||||
for (int i = 0; i < runtime_count; ++i) {
|
||||
if (!is_sme_family(runtime[i].kernels->required_cpu)) {
|
||||
non_sme_slot = i;
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
#pragma once
|
||||
|
||||
#if defined(__aarch64__) || defined(_M_ARM64)
|
||||
|
||||
#if defined(__linux__)
|
||||
#include <sys/auxv.h>
|
||||
#include <sys/prctl.h>
|
||||
|
||||
#if !defined(HWCAP2_SVE2)
|
||||
#define HWCAP2_SVE2 (1ULL << 1)
|
||||
#endif
|
||||
|
||||
#if !defined(HWCAP_FPHP)
|
||||
#define HWCAP_FPHP (1 << 9)
|
||||
#endif
|
||||
|
||||
#if !defined(HWCAP_ASIMDHP)
|
||||
#define HWCAP_ASIMDHP (1 << 10)
|
||||
#endif
|
||||
|
||||
#if !defined(HWCAP2_I8MM)
|
||||
#define HWCAP2_I8MM (1ULL << 13)
|
||||
#endif
|
||||
|
||||
#if !defined(HWCAP_ASIMDDP)
|
||||
#define HWCAP_ASIMDDP (1 << 20)
|
||||
#endif
|
||||
|
||||
#if !defined(HWCAP_SVE)
|
||||
#define HWCAP_SVE (1 << 22)
|
||||
#endif
|
||||
|
||||
#if !defined(HWCAP2_SME)
|
||||
#define HWCAP2_SME (1ULL << 23)
|
||||
#endif
|
||||
|
||||
#if !defined(HWCAP2_SME2)
|
||||
#define HWCAP2_SME2 (1ULL << 37)
|
||||
#endif
|
||||
|
||||
#if !defined(PR_SVE_GET_VL)
|
||||
#define PR_SVE_GET_VL 51
|
||||
#endif
|
||||
|
||||
#if !defined(PR_SVE_VL_LEN_MASK)
|
||||
#define PR_SVE_VL_LEN_MASK 0xffff
|
||||
#endif
|
||||
|
||||
#elif defined(__APPLE__)
|
||||
#include <sys/sysctl.h>
|
||||
#elif defined(_WIN32)
|
||||
#include <windows.h>
|
||||
|
||||
#if !defined(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE)
|
||||
#define PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE 43
|
||||
#endif
|
||||
|
||||
#if !defined(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE)
|
||||
#define PF_ARM_SVE_INSTRUCTIONS_AVAILABLE 46
|
||||
#endif
|
||||
|
||||
#if !defined(PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE)
|
||||
#define PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE 47
|
||||
#endif
|
||||
|
||||
#if !defined(PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE)
|
||||
#define PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE 66
|
||||
#endif
|
||||
|
||||
#if !defined(PF_ARM_V82_FP16_INSTRUCTIONS_AVAILABLE)
|
||||
#define PF_ARM_V82_FP16_INSTRUCTIONS_AVAILABLE 67
|
||||
#endif
|
||||
|
||||
#if !defined(PF_ARM_SME_INSTRUCTIONS_AVAILABLE)
|
||||
#define PF_ARM_SME_INSTRUCTIONS_AVAILABLE 70
|
||||
#endif
|
||||
|
||||
#if !defined(PF_ARM_SME2_INSTRUCTIONS_AVAILABLE)
|
||||
#define PF_ARM_SME2_INSTRUCTIONS_AVAILABLE 71
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
typedef struct ggml_feats_arch64_runtime {
|
||||
bool has_dotprod;
|
||||
bool has_fp16;
|
||||
bool has_sve;
|
||||
bool has_sve2;
|
||||
bool has_i8mm;
|
||||
bool has_sme;
|
||||
bool has_sme2;
|
||||
int sve_cnt;
|
||||
} ggml_feats_arch64_runtime_t;
|
||||
|
||||
static inline ggml_feats_arch64_runtime_t ggml_get_aarch64_runtime_features(void) {
|
||||
ggml_feats_arch64_runtime_t runtime_feat = {};
|
||||
|
||||
#if defined(__linux__)
|
||||
const unsigned long hwcap = getauxval(AT_HWCAP);
|
||||
const unsigned long hwcap2 = getauxval(AT_HWCAP2);
|
||||
|
||||
runtime_feat.has_dotprod = !!(hwcap & HWCAP_ASIMDDP);
|
||||
runtime_feat.has_fp16 = !!(hwcap & HWCAP_FPHP) && !!(hwcap & HWCAP_ASIMDHP);;
|
||||
runtime_feat.has_sve = !!(hwcap & HWCAP_SVE);
|
||||
runtime_feat.has_sve2 = !!(hwcap2 & HWCAP2_SVE2);
|
||||
runtime_feat.has_i8mm = !!(hwcap2 & HWCAP2_I8MM);
|
||||
runtime_feat.has_sme = !!(hwcap2 & HWCAP2_SME);
|
||||
runtime_feat.has_sme2 = !!(hwcap2 & HWCAP2_SME2);
|
||||
|
||||
if (runtime_feat.has_sve) {
|
||||
const int vl = prctl(PR_SVE_GET_VL);
|
||||
if (vl >= 0) {
|
||||
runtime_feat.sve_cnt = vl & PR_SVE_VL_LEN_MASK;
|
||||
}
|
||||
}
|
||||
#elif defined(__APPLE__)
|
||||
int oldp = 0;
|
||||
size_t size = sizeof(oldp);
|
||||
|
||||
if (sysctlbyname("hw.optional.arm.FEAT_DotProd", &oldp, &size, nullptr, 0) == 0) {
|
||||
runtime_feat.has_dotprod = static_cast<bool>(oldp);
|
||||
}
|
||||
|
||||
if (sysctlbyname("hw.optional.arm.FEAT_FP16", &oldp, &size, nullptr, 0) == 0) {
|
||||
runtime_feat.has_fp16 = static_cast<bool>(oldp);
|
||||
}
|
||||
|
||||
if (sysctlbyname("hw.optional.arm.FEAT_SVE", &oldp, &size, nullptr, 0) == 0) {
|
||||
runtime_feat.has_sve = static_cast<bool>(oldp);
|
||||
}
|
||||
|
||||
if (sysctlbyname("hw.optional.arm.FEAT_SVE2", &oldp, &size, nullptr, 0) == 0) {
|
||||
runtime_feat.has_sve2 = static_cast<bool>(oldp);
|
||||
}
|
||||
|
||||
if (sysctlbyname("hw.optional.arm.FEAT_I8MM", &oldp, &size, nullptr, 0) == 0) {
|
||||
runtime_feat.has_i8mm = static_cast<bool>(oldp);
|
||||
}
|
||||
|
||||
if (sysctlbyname("hw.optional.arm.FEAT_SME", &oldp, &size, nullptr, 0) == 0) {
|
||||
runtime_feat.has_sme = static_cast<bool>(oldp);
|
||||
}
|
||||
|
||||
if (sysctlbyname("hw.optional.arm.FEAT_SME2", &oldp, &size, nullptr, 0) == 0) {
|
||||
runtime_feat.has_sme2 = static_cast<bool>(oldp);
|
||||
}
|
||||
|
||||
// Apple does not support userspace non-streaming SVE; keep SVE vector length unknown.
|
||||
runtime_feat.sve_cnt = 0;
|
||||
#elif defined (_WIN32)
|
||||
runtime_feat.has_dotprod = IsProcessorFeaturePresent(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE) != 0;
|
||||
runtime_feat.has_fp16 = IsProcessorFeaturePresent(PF_ARM_V82_FP16_INSTRUCTIONS_AVAILABLE) != 0;
|
||||
runtime_feat.has_sve = IsProcessorFeaturePresent(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE) != 0;
|
||||
runtime_feat.has_sve2 = IsProcessorFeaturePresent(PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE) != 0;
|
||||
runtime_feat.has_i8mm = IsProcessorFeaturePresent(PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE) != 0;
|
||||
runtime_feat.has_sme = IsProcessorFeaturePresent(PF_ARM_SME_INSTRUCTIONS_AVAILABLE) != 0;
|
||||
runtime_feat.has_sme2 = IsProcessorFeaturePresent(PF_ARM_SME2_INSTRUCTIONS_AVAILABLE) != 0;
|
||||
|
||||
// Windows exposes SVE feature presence, but not the runtime SVE vector length here.
|
||||
runtime_feat.sve_cnt = 0;
|
||||
#endif
|
||||
|
||||
return runtime_feat;
|
||||
}
|
||||
|
||||
#endif // defined(__aarch64__) || defined(_M_ARM64)
|
||||
Reference in New Issue
Block a user