From 132753bf4e52b1a8bda8f6ec33f1785bd80470da Mon Sep 17 00:00:00 2001 From: Jonathan Clohessy Date: Wed, 12 Aug 2026 12:49:11 +0100 Subject: [PATCH] kleidiai: Add runtime feature detection mechanism for aarch64/kleidiai (#26076) * Add runtime feature detection mechanism for aarch64/kleidiai Signed-off-by: Jonathan Clohessy * Address Review Comments Signed-off-by: Jonathan Clohessy * Add log warning for NSMC reserved value Signed-off-by: Jonathan Clohessy * Address review comments Signed-off-by: Jonathan Clohessy * Fix Rebase, move code from cpu-feats to ggml-feats Signed-off-by: Jonathan Clohessy * Address naming of runtime feature struct Signed-off-by: Jonathan Clohessy --------- Signed-off-by: Jonathan Clohessy --- ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp | 87 +------ ggml/src/ggml-cpu/kleidiai/kleidiai.cpp | 312 ++++++++++++++++------- ggml/src/ggml-feats.h | 166 ++++++++++++ 3 files changed, 383 insertions(+), 182 deletions(-) create mode 100644 ggml/src/ggml-feats.h diff --git a/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp b/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp index a64ad7a3c7..1394256089 100644 --- a/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +++ b/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp @@ -1,90 +1,11 @@ #include "ggml-backend-impl.h" +#include "ggml-feats.h" -#if defined(__aarch64__) - -#if defined(__linux__) -#include -#elif defined(__APPLE__) -#include -#endif - -#if !defined(HWCAP_FPHP) -#define HWCAP_FPHP (1 << 9) -#endif - -#if !defined(HWCAP_ASIMDHP) -#define HWCAP_ASIMDHP (1 << 10) -#endif - -#if !defined(HWCAP_ASIMDDP) -#define HWCAP_ASIMDDP (1 << 20) -#endif - -#if !defined(HWCAP_SVE) -#define HWCAP_SVE (1 << 22) -#endif - -#if !defined(HWCAP2_SVE2) -#define HWCAP2_SVE2 (1 << 1) -#endif - -#if !defined(HWCAP2_I8MM) -#define HWCAP2_I8MM (1 << 13) -#endif - -#if !defined(HWCAP2_SME) -#define HWCAP2_SME (1 << 23) -#endif - -struct aarch64_features { - // has_neon not needed, aarch64 has NEON guaranteed - bool has_dotprod = false; - bool has_fp16 = false; - bool has_sve = false; - bool has_sve2 = false; - bool has_i8mm = false; - bool has_sme = false; - bool has_sme2 = false; - - aarch64_features() { -#if defined(__linux__) - uint32_t hwcap = getauxval(AT_HWCAP); - uint32_t hwcap2 = getauxval(AT_HWCAP2); - - has_dotprod = !!(hwcap & HWCAP_ASIMDDP); - has_fp16 = !!(hwcap & HWCAP_FPHP) && !!(hwcap & HWCAP_ASIMDHP); - has_sve = !!(hwcap & HWCAP_SVE); - has_sve2 = !!(hwcap2 & HWCAP2_SVE2); - has_i8mm = !!(hwcap2 & HWCAP2_I8MM); - has_sme = !!(hwcap2 & HWCAP2_SME); -#elif defined(__APPLE__) - int oldp = 0; - size_t size = sizeof(oldp); - - if (sysctlbyname("hw.optional.arm.FEAT_DotProd", &oldp, &size, NULL, 0) == 0) { - has_dotprod = static_cast(oldp); - } - - if (sysctlbyname("hw.optional.arm.FEAT_I8MM", &oldp, &size, NULL, 0) == 0) { - has_i8mm = static_cast(oldp); - } - - if (sysctlbyname("hw.optional.arm.FEAT_SME", &oldp, &size, NULL, 0) == 0) { - has_sme = static_cast(oldp); - } - - if (sysctlbyname("hw.optional.arm.FEAT_SME2", &oldp, &size, NULL, 0) == 0) { - has_sme2 = static_cast(oldp); - } - - // Apple apparently does not implement SVE yet -#endif - } -}; +#if defined(__aarch64__) || defined(_M_ARM64) static int ggml_backend_cpu_aarch64_score() { int score = 1; - aarch64_features af; + ggml_feats_arch64_runtime_t af = ggml_get_aarch64_runtime_features(); #ifdef GGML_USE_DOTPROD if (!af.has_dotprod) { return 0; } @@ -116,4 +37,4 @@ static int ggml_backend_cpu_aarch64_score() { GGML_BACKEND_DL_SCORE_IMPL(ggml_backend_cpu_aarch64_score) -# endif // defined(__aarch64__) +# endif // defined(__aarch64__) || defined(_M_ARM64) diff --git a/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp b/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp index 1c5a459f21..c1cc33db27 100644 --- a/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +++ b/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp @@ -2,10 +2,12 @@ // SPDX-License-Identifier: MIT // #include -#include -#include +#include +#include +#include #include #include +#include #include #include #include @@ -17,25 +19,21 @@ #include #include #include -#include +#include #include #include +#include +#include #if defined(__linux__) #include +#include #include #include #include #include -#ifndef HWCAP2_SME2 -#define HWCAP2_SME2 (1UL << 37) -#endif #elif defined(__APPLE__) -#include #include #include -#elif defined(_WIN32) -#include -#include #endif #include "kleidiai.h" @@ -43,6 +41,7 @@ #include "ggml-cpu.h" #include "ggml-cpu-impl.h" #include "ggml-impl.h" +#include "ggml-feats.h" #include "ggml-backend-impl.h" #include "ggml-threading.h" #include "traits.h" @@ -64,8 +63,8 @@ struct ggml_kleidiai_context { ggml_kleidiai_kernels * kernels_q4; ggml_kleidiai_kernels * kernels_q8; ggml_kleidiai_kernels * kernels_f32; - int sme_thread_cap; // <= 0 means “SME disabled/unknown”; - int thread_hint; // <= 0 means “no hint” + int sme_thread_cap; // <= 0 means "SME disabled/unknown" + int thread_hint; // <= 0 means "no hint" int chunk_multiplier; } static ctx = { CPU_FEATURE_NONE, nullptr, nullptr, nullptr, 0, -1, 4 }; @@ -93,24 +92,117 @@ static const char* cpu_feature_to_string(cpu_feature f) { } } +#if defined(__linux__) && defined(__aarch64__) +static bool parse_cpu_dir_name(const char* name, size_t* cpu) { + if (strncmp(name, "cpu", 3) != 0 || + name[3] < '0' || name[3] > '9') { + return false; + } + + const char* first = name + 3; + const char* last = name + strlen(name); + + size_t value = 0; + const auto [end, ec] = std::from_chars(first, last, value, 10); + + if (ec != std::errc{} || end != last) { + return false; + } + + *cpu = value; + return true; +} + +static std::vector detect_cpu_ids() { + std::vector cpus; + + DIR * dir = opendir("/sys/devices/system/cpu"); + if (dir == nullptr) { + return cpus; + } + + while (dirent * entry = readdir(dir)) { + size_t cpu = 0; + if (parse_cpu_dir_name(entry->d_name, &cpu)) { + cpus.push_back(cpu); + } + } + closedir(dir); + + std::sort(cpus.begin(), cpus.end()); + cpus.erase(std::unique(cpus.begin(), cpus.end()), cpus.end()); + return cpus; +} +#endif + +#if defined(__APPLE__) && defined(__aarch64__) +static bool apple_sme_counted_perf_level(std::string name) { + for (std::string::size_type i = 0; i < name.size(); ++i) { + name[i] = (char) std::tolower((unsigned char) name[i]); + } + + // Conservative ceiling: only count perf-level names observed to provide full SME throughput. + // Future names should be calibrated here before they raise the automatic SME thread cap. + return name.find("super") != std::string::npos || + name.find("performance") != std::string::npos; +} +#endif + +static void add_smcus_from_smidr(uint64_t smidr, size_t & num_private, std::map & shared_counts) { + // Arm ARM: SMIDR_EL1. SH==0 is implementation-defined; keep the existing + // conservative policy and only treat zero affinity as private. + const uint32_t sh = (uint32_t)((smidr >> 13) & 0x3); + const uint32_t nsmc = (uint32_t)((smidr >> 56) & 0xF); + const size_t shared_count = nsmc == 0xF ? 1 : (size_t)nsmc + 1; + const uint32_t affinity = (uint32_t)(smidr & 0xFFFu); + const uint32_t affinity2 = (uint32_t)((smidr >> 32) & 0xFFFFFu); + const uint32_t id = (affinity2 << 12) | affinity; + + if (nsmc == 0xF) { + GGML_LOG_WARN("kleidiai: NSMC detected as 0xF indicating reseved value, setting min safe shared SMCU count to 1"); + } + + switch (sh) { + case 2: // private SMCU + ++num_private; + break; + case 3: // shared SMCU + if (shared_counts[id] < shared_count) { + shared_counts[id] = shared_count; + } + break; + case 0: + if (id == 0) { + ++num_private; + } else if (shared_counts[id] < shared_count) { + shared_counts[id] = shared_count; + } + break; + default: + break; + } +} + static size_t detect_num_smcus() { - if (!ggml_cpu_has_sme()) { + auto runtime_feat = ggml_get_aarch64_runtime_features(); + if (!runtime_feat.has_sme) { return 0; } #if defined(__linux__) && defined(__aarch64__) // Linux/aarch64: Best-effort count of Streaming Mode Compute Units (SMCUs) via SMIDR_EL1 sysfs. size_t num_private = 0; - std::set shared_ids; + std::map shared_counts; - for (size_t cpu = 0;; ++cpu) { + const std::vector cpus = detect_cpu_ids(); + for (const size_t cpu : cpus) { const std::string path = "/sys/devices/system/cpu/cpu" + std::to_string(cpu) + "/regs/identification/smidr_el1"; std::ifstream file(path); if (!file.is_open()) { - break; + continue; } uint64_t smidr = 0; @@ -118,54 +210,69 @@ static size_t detect_num_smcus() { continue; } - // Arm ARM: SMIDR_EL1 - const uint32_t sh = (uint32_t)((smidr >> 13) & 0x3); - // Build an "affinity-like" identifier for shared SMCUs. - // Keep the original packing logic, but isolate it here. - const uint32_t id = (uint32_t)((smidr & 0xFFFu) | ((smidr >> 20) & 0xFFFFF000u)); - - switch (sh) { - case 0b10: // private SMCU - ++num_private; - break; - case 0b11: // shared SMCU - shared_ids.emplace(id); - break; - case 0b00: - // Ambiguous / implementation-defined. Be conservative: - // treat id==0 as private, otherwise as shared. - if (id == 0) ++num_private; - else shared_ids.emplace(id); - break; - default: - break; - } + add_smcus_from_smidr(smidr, num_private, shared_counts); } - return num_private + shared_ids.size(); + size_t total = num_private; + for (const auto & entry : shared_counts) { + total += entry.second; + } + return total; #elif defined(__APPLE__) && defined(__aarch64__) - // table for known M4 variants. Users can override via GGML_KLEIDIAI_SME=. - char chip_name[256] = {}; - size_t size = sizeof(chip_name); + int perf_levels = 0; + size_t size = sizeof(perf_levels); + if (sysctlbyname("hw.nperflevels", &perf_levels, &size, nullptr, 0) != 0 || + size != sizeof(perf_levels) || perf_levels <= 0) { + return 0; + } - if (sysctlbyname("machdep.cpu.brand_string", chip_name, &size, nullptr, 0) == 0) { - const std::string brand(chip_name); + size_t units = 0; + for (int i = 0; i < perf_levels; ++i) { + char key[64] = {}; + int physical_cpus = 0; + int cpus_per_l2 = 0; - struct ModelSMCU { const char *match; size_t smcus; }; - static const ModelSMCU table[] = { - { "M4 Ultra", 2 }, - { "M4 Max", 2 }, - { "M4 Pro", 2 }, - { "M4", 1 }, - }; + snprintf(key, sizeof(key), "hw.perflevel%d.physicalcpu", i); + size = sizeof(physical_cpus); + if (sysctlbyname(key, &physical_cpus, &size, nullptr, 0) != 0 || + size != sizeof(physical_cpus) || physical_cpus <= 0) { + continue; + } - for (const auto &e : table) { - if (brand.find(e.match) != std::string::npos) { - return e.smcus; - } + snprintf(key, sizeof(key), "hw.perflevel%d.cpusperl2", i); + size = sizeof(cpus_per_l2); + if (sysctlbyname(key, &cpus_per_l2, &size, nullptr, 0) != 0 || + size != sizeof(cpus_per_l2) || cpus_per_l2 <= 0) { + continue; + } + + snprintf(key, sizeof(key), "hw.perflevel%d.name", i); + size = 0; + if (sysctlbyname(key, nullptr, &size, nullptr, 0) != 0 || size == 0) { + continue; + } + + std::string name(size, '\0'); + if (sysctlbyname(key, &name[0], &size, nullptr, 0) != 0) { + continue; + } + name.resize(size); + while (!name.empty() && name.back() == '\0') { + name.pop_back(); + } + + if (apple_sme_counted_perf_level(name)) { + units += (size_t) ((physical_cpus + cpus_per_l2 - 1) / cpus_per_l2); } } + + return units; + +#elif defined(_WIN32) && (defined(_M_ARM64) || defined(__aarch64__)) + // No verified Windows arm64 SMCU detection path yet. Return unknown and use + // GGML_KLEIDIAI_SME=N as a diagnostics/debug override for SME thread cap + // calibration until a detection mechanism is verified on real hardware. return 0; #else @@ -198,15 +305,18 @@ static void init_kleidiai_context(void) { if (!initialized) { initialized = true; + // Optional diagnostics/debug overrides; production defaults come from runtime detection. const char *env_sme = getenv("GGML_KLEIDIAI_SME"); const char *env_threads = getenv("GGML_TOTAL_THREADS"); const char *env_chunk_mult = getenv("GGML_KLEIDIAI_CHUNK_MULTIPLIER"); + auto runtime_feat = ggml_get_aarch64_runtime_features(); + size_t detected_smcus = 0; - ctx.features = (ggml_cpu_has_dotprod() ? CPU_FEATURE_DOTPROD : CPU_FEATURE_NONE) | - (ggml_cpu_has_matmul_int8() ? CPU_FEATURE_I8MM : CPU_FEATURE_NONE) | - ((ggml_cpu_has_sve() && ggml_cpu_get_sve_cnt() == QK8_0) ? CPU_FEATURE_SVE : CPU_FEATURE_NONE); + ctx.features = (runtime_feat.has_dotprod ? CPU_FEATURE_DOTPROD : CPU_FEATURE_NONE) | + (runtime_feat.has_i8mm ? CPU_FEATURE_I8MM : CPU_FEATURE_NONE) | + (runtime_feat.sve_cnt == QK8_0 ? CPU_FEATURE_SVE : CPU_FEATURE_NONE); if (env_threads) { bool ok = false; @@ -224,54 +334,54 @@ static void init_kleidiai_context(void) { } } - // SME policy: - // - env unset => auto-detect SMCUs; enable SME only if detected > 0. - // - env=0 => force off. - // - env>0 => force N cores, if the binary was built with SME. int sme_cores = 0; bool sme_env_ok = false; bool sme_env_set = (env_sme != nullptr); + const bool has_supported_sme_family = runtime_feat.has_sme; + bool sme_cap_detected = false; + + if (has_supported_sme_family) { + detected_smcus = detect_num_smcus(); + sme_cap_detected = detected_smcus > 0; + // Some platforms expose SME without exposing a calibrated SMCU count. + // Use one SME thread as the conservative default; add platform SMCU detection to raise it. + sme_cores = sme_cap_detected ? (int)detected_smcus : 1; + + if (!sme_env_set && !sme_cap_detected) { + GGML_LOG_INFO("kleidiai: SME detected; SMCU count unavailable, using conservative SME thread cap=1\n"); + } + } + + // Runtime-detect SME support and available SMCUs first. The detected SMCU + // count is used as the SME thread cap, and GGML_KLEIDIAI_SME can debug-override that: + // - unset: use runtime detection. + // - 0: disable SME-family kernels. + // - N > 0: use N as the SME thread cap, if an SME-family kernel is selectable. if (sme_env_set) { bool ok = false; int v = parse_uint_env(env_sme, "GGML_KLEIDIAI_SME", &ok); sme_env_ok = ok; - if (!ok) { - GGML_LOG_WARN("kleidiai: GGML_KLEIDIAI_SME set but parsing failed; falling back to runtime SME-core detection\n"); - detected_smcus = detect_num_smcus(); - sme_cores = detected_smcus > 0 ? (int)detected_smcus : 0; - } else if (v == 0) { - sme_cores = 0; - } else if (!ggml_cpu_has_sme()) { - GGML_LOG_WARN("kleidiai: GGML_KLEIDIAI_SME=%d but the binary was not built with SME; disabling SME\n", v); - sme_cores = 0; + if (ok) { + if (has_supported_sme_family) { + sme_cores = v; + } else { + if (v > 0) { + GGML_LOG_WARN("kleidiai: GGML_KLEIDIAI_SME=%d but SME is not supported on this CPU; disabling SME-family kernels\n", v); + } + sme_cores = 0; + } } else { - sme_cores = v; + GGML_LOG_WARN("kleidiai: GGML_KLEIDIAI_SME set but parsing failed; using automatic SME thread cap\n"); } - } else { - detected_smcus = detect_num_smcus(); - sme_cores = detected_smcus > 0 ? (int)detected_smcus : 0; } - if (!sme_env_set && ggml_cpu_has_sme() && sme_cores == 0) { - GGML_LOG_WARN("kleidiai: runtime SME-core detection returned 0; falling back to NEON\n"); - } - - if (sme_cores > 0) { + if (sme_cores > 0 && has_supported_sme_family) { ctx.features |= CPU_FEATURE_SME; -#if defined(__aarch64__) && defined(__linux__) - // ARM guarantees SME2 implies SME, so only check SME2 when SME is enabled. - if (getauxval(AT_HWCAP2) & HWCAP2_SME2) { + if (runtime_feat.has_sme2) { ctx.features |= CPU_FEATURE_SME2; } -#elif defined(__aarch64__) && defined(__APPLE__) - int feat_sme2 = 0; - size_t size = sizeof(feat_sme2); - if (sysctlbyname("hw.optional.arm.FEAT_SME2", &feat_sme2, &size, NULL, 0) == 0 && feat_sme2) { - ctx.features |= CPU_FEATURE_SME2; - } -#endif } // Kernel selection @@ -297,16 +407,19 @@ static void init_kleidiai_context(void) { GGML_LOG_INFO("kleidiai: primary f32 kernel feature %s\n", cpu_feature_to_string(ctx.kernels_f32->required_cpu)); } - ctx.sme_thread_cap = (ctx.features & CPU_FEATURE_SME) ? sme_cores : 0; + const bool has_selected_sme_family_kernel = + (ctx.kernels_q4 && is_sme_family(ctx.kernels_q4->required_cpu)) || + (ctx.kernels_q8 && is_sme_family(ctx.kernels_q8->required_cpu)) || + (ctx.kernels_f32 && is_sme_family(ctx.kernels_f32->required_cpu)); + ctx.sme_thread_cap = has_selected_sme_family_kernel ? sme_cores : 0; - if (ctx.features & CPU_FEATURE_SME) { - const bool has_sme2 = (ctx.features & CPU_FEATURE_SME2) != CPU_FEATURE_NONE; + if (has_selected_sme_family_kernel) { if (sme_env_set && sme_env_ok && sme_cores > 0) { - GGML_LOG_INFO("kleidiai: SME%s enabled (GGML_KLEIDIAI_SME=%d override)\n", - has_sme2 ? "2" : "", sme_cores); + GGML_LOG_INFO("kleidiai: SME enabled (GGML_KLEIDIAI_SME=%d debug override)\n", sme_cores); + } else if (sme_cap_detected) { + GGML_LOG_INFO("kleidiai: SME enabled (runtime-detected SME thread cap=%d)\n", sme_cores); } else { - GGML_LOG_INFO("kleidiai: SME%s enabled (runtime-detected SME cores=%d)\n", - has_sme2 ? "2" : "", sme_cores); + GGML_LOG_INFO("kleidiai: SME enabled (runtime SME detected, conservative thread cap=%d)\n", sme_cores); } } else { GGML_LOG_INFO("kleidiai: SME disabled\n"); @@ -467,7 +580,7 @@ static int kleidiai_collect_kernel_chain_common( } if (is_sme_family(primary->required_cpu)) { - const cpu_feature fallback_mask = static_cast(features & ~CPU_FEATURE_SME & ~CPU_FEATURE_SME2); + const cpu_feature fallback_mask = static_cast(features & ~(CPU_FEATURE_SME | CPU_FEATURE_SME2)); if (fallback_mask != CPU_FEATURE_NONE) { ggml_kleidiai_kernels * fallback = select_fallback(fallback_mask); if (fallback && fallback != primary && @@ -1077,13 +1190,14 @@ class tensor_traits : public ggml::cpu::tensor_traits { const int ith_total = params->ith; int sme_slot = -1; + int non_sme_slot = -1; for (int i = 0; i < runtime_count; ++i) { if (is_sme_family(runtime[i].kernels->required_cpu)) { sme_slot = i; break; } } - int non_sme_slot = -1; + for (int i = 0; i < runtime_count; ++i) { if (!is_sme_family(runtime[i].kernels->required_cpu)) { non_sme_slot = i; diff --git a/ggml/src/ggml-feats.h b/ggml/src/ggml-feats.h new file mode 100644 index 0000000000..d4a8c83a7f --- /dev/null +++ b/ggml/src/ggml-feats.h @@ -0,0 +1,166 @@ +#pragma once + +#if defined(__aarch64__) || defined(_M_ARM64) + +#if defined(__linux__) +#include +#include + +#if !defined(HWCAP2_SVE2) +#define HWCAP2_SVE2 (1ULL << 1) +#endif + +#if !defined(HWCAP_FPHP) +#define HWCAP_FPHP (1 << 9) +#endif + +#if !defined(HWCAP_ASIMDHP) +#define HWCAP_ASIMDHP (1 << 10) +#endif + +#if !defined(HWCAP2_I8MM) +#define HWCAP2_I8MM (1ULL << 13) +#endif + +#if !defined(HWCAP_ASIMDDP) +#define HWCAP_ASIMDDP (1 << 20) +#endif + +#if !defined(HWCAP_SVE) +#define HWCAP_SVE (1 << 22) +#endif + +#if !defined(HWCAP2_SME) +#define HWCAP2_SME (1ULL << 23) +#endif + +#if !defined(HWCAP2_SME2) +#define HWCAP2_SME2 (1ULL << 37) +#endif + +#if !defined(PR_SVE_GET_VL) +#define PR_SVE_GET_VL 51 +#endif + +#if !defined(PR_SVE_VL_LEN_MASK) +#define PR_SVE_VL_LEN_MASK 0xffff +#endif + +#elif defined(__APPLE__) +#include +#elif defined(_WIN32) +#include + +#if !defined(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE) +#define PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE 43 +#endif + +#if !defined(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE) +#define PF_ARM_SVE_INSTRUCTIONS_AVAILABLE 46 +#endif + +#if !defined(PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE) +#define PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE 47 +#endif + +#if !defined(PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE) +#define PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE 66 +#endif + +#if !defined(PF_ARM_V82_FP16_INSTRUCTIONS_AVAILABLE) +#define PF_ARM_V82_FP16_INSTRUCTIONS_AVAILABLE 67 +#endif + +#if !defined(PF_ARM_SME_INSTRUCTIONS_AVAILABLE) +#define PF_ARM_SME_INSTRUCTIONS_AVAILABLE 70 +#endif + +#if !defined(PF_ARM_SME2_INSTRUCTIONS_AVAILABLE) +#define PF_ARM_SME2_INSTRUCTIONS_AVAILABLE 71 +#endif + +#endif + +typedef struct ggml_feats_arch64_runtime { + bool has_dotprod; + bool has_fp16; + bool has_sve; + bool has_sve2; + bool has_i8mm; + bool has_sme; + bool has_sme2; + int sve_cnt; +} ggml_feats_arch64_runtime_t; + +static inline ggml_feats_arch64_runtime_t ggml_get_aarch64_runtime_features(void) { + ggml_feats_arch64_runtime_t runtime_feat = {}; + +#if defined(__linux__) + const unsigned long hwcap = getauxval(AT_HWCAP); + const unsigned long hwcap2 = getauxval(AT_HWCAP2); + + runtime_feat.has_dotprod = !!(hwcap & HWCAP_ASIMDDP); + runtime_feat.has_fp16 = !!(hwcap & HWCAP_FPHP) && !!(hwcap & HWCAP_ASIMDHP);; + runtime_feat.has_sve = !!(hwcap & HWCAP_SVE); + runtime_feat.has_sve2 = !!(hwcap2 & HWCAP2_SVE2); + runtime_feat.has_i8mm = !!(hwcap2 & HWCAP2_I8MM); + runtime_feat.has_sme = !!(hwcap2 & HWCAP2_SME); + runtime_feat.has_sme2 = !!(hwcap2 & HWCAP2_SME2); + + if (runtime_feat.has_sve) { + const int vl = prctl(PR_SVE_GET_VL); + if (vl >= 0) { + runtime_feat.sve_cnt = vl & PR_SVE_VL_LEN_MASK; + } + } +#elif defined(__APPLE__) + int oldp = 0; + size_t size = sizeof(oldp); + + if (sysctlbyname("hw.optional.arm.FEAT_DotProd", &oldp, &size, nullptr, 0) == 0) { + runtime_feat.has_dotprod = static_cast(oldp); + } + + if (sysctlbyname("hw.optional.arm.FEAT_FP16", &oldp, &size, nullptr, 0) == 0) { + runtime_feat.has_fp16 = static_cast(oldp); + } + + if (sysctlbyname("hw.optional.arm.FEAT_SVE", &oldp, &size, nullptr, 0) == 0) { + runtime_feat.has_sve = static_cast(oldp); + } + + if (sysctlbyname("hw.optional.arm.FEAT_SVE2", &oldp, &size, nullptr, 0) == 0) { + runtime_feat.has_sve2 = static_cast(oldp); + } + + if (sysctlbyname("hw.optional.arm.FEAT_I8MM", &oldp, &size, nullptr, 0) == 0) { + runtime_feat.has_i8mm = static_cast(oldp); + } + + if (sysctlbyname("hw.optional.arm.FEAT_SME", &oldp, &size, nullptr, 0) == 0) { + runtime_feat.has_sme = static_cast(oldp); + } + + if (sysctlbyname("hw.optional.arm.FEAT_SME2", &oldp, &size, nullptr, 0) == 0) { + runtime_feat.has_sme2 = static_cast(oldp); + } + + // Apple does not support userspace non-streaming SVE; keep SVE vector length unknown. + runtime_feat.sve_cnt = 0; +#elif defined (_WIN32) + runtime_feat.has_dotprod = IsProcessorFeaturePresent(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE) != 0; + runtime_feat.has_fp16 = IsProcessorFeaturePresent(PF_ARM_V82_FP16_INSTRUCTIONS_AVAILABLE) != 0; + runtime_feat.has_sve = IsProcessorFeaturePresent(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE) != 0; + runtime_feat.has_sve2 = IsProcessorFeaturePresent(PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE) != 0; + runtime_feat.has_i8mm = IsProcessorFeaturePresent(PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE) != 0; + runtime_feat.has_sme = IsProcessorFeaturePresent(PF_ARM_SME_INSTRUCTIONS_AVAILABLE) != 0; + runtime_feat.has_sme2 = IsProcessorFeaturePresent(PF_ARM_SME2_INSTRUCTIONS_AVAILABLE) != 0; + + // Windows exposes SVE feature presence, but not the runtime SVE vector length here. + runtime_feat.sve_cnt = 0; +#endif + + return runtime_feat; +} + +#endif // defined(__aarch64__) || defined(_M_ARM64)