From 008125b5c19cba92de667e9fb489bf94b03c2f86 Mon Sep 17 00:00:00 2001 From: Adam Caldwell <2320451+accaldwell@users.noreply.github.com> Date: Tue, 17 Mar 2026 02:44:55 -0700 Subject: [PATCH] Enable AVX-VNNI 256-bit path for Q4_K and Q5_K R4 matmul (#1446) Add new CPU macro HAVE_VNNI256 for CPUs with 256-bit VNNI (AVX-VNNI) support or better (AVX512-VNNI+VL), separate from HAVE_FANCY_SIMD which requires the full AVX-512 set. Relax four #ifdef guards in mul_mat_q4_k_r4_q8_k and mul_mat_q5_k_r4_q8_k to use HAVE_VNNI256 instead of HAVE_FANCY_SIMD, enabling vpdpbusd and cvtepi8_epi32 on Alder Lake, Raptor Lake, and similar CPUs. Co-authored-by: Adam Caldwell --- ggml/src/iqk/iqk_config.h | 6 ++++++ ggml/src/iqk/iqk_gemm_kquants.cpp | 8 ++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/ggml/src/iqk/iqk_config.h b/ggml/src/iqk/iqk_config.h index 3d8238d7d..78e1d6fc4 100644 --- a/ggml/src/iqk/iqk_config.h +++ b/ggml/src/iqk/iqk_config.h @@ -46,5 +46,11 @@ #if defined(__AVX512F__) && defined(__AVX512VNNI__) && defined(__AVX512VL__) && defined(__AVX512BW__) && defined(__AVX512DQ__) #define HAVE_FANCY_SIMD #endif +#if defined HAVE_VNNI256 + #undef HAVE_VNNI256 +#endif +#if defined(__AVXVNNI__) || (defined(__AVX512VNNI__) && defined(__AVX512VL__)) + #define HAVE_VNNI256 +#endif #endif diff --git a/ggml/src/iqk/iqk_gemm_kquants.cpp b/ggml/src/iqk/iqk_gemm_kquants.cpp index 05baea149..ff1121d75 100644 --- a/ggml/src/iqk/iqk_gemm_kquants.cpp +++ b/ggml/src/iqk/iqk_gemm_kquants.cpp @@ -1536,7 +1536,7 @@ static void mul_mat_q4_k_r4_q8_k(int n, const void * vx, size_t bx, const DataIn auto mins = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(lbits, 4), mf), _mm256_and_si256(_mm256_srli_epi16(hbits, 2), m3)); process_min_r4_b32(ibl, m4, mins, q8, acc); for (int ib = 0; ib < QK_K/32; ++ib) { -#ifdef HAVE_FANCY_SIMD +#ifdef HAVE_VNNI256 auto scales_d = _mm256_cvtepi8_epi32(_mm_set1_epi32(hd.val[ib])); #else auto aux = _mm_set1_epi32(hd.val[ib]); @@ -1551,7 +1551,7 @@ static void mul_mat_q4_k_r4_q8_k(int n, const void * vx, size_t bx, const DataIn qx[3] = _mm256_and_si256(_mm256_srli_epi16(bits2, 4), mf); for (int iy = 0; iy < nrc_y; ++iy) { auto y = _mm256_loadu_si256((const __m256i*)q8.y[iy][ibl].qs+ib); -#ifdef HAVE_FANCY_SIMD +#ifdef HAVE_VNNI256 auto sumi = _mm256_setzero_si256(); sumi = _mm256_dpbusd_epi32(sumi, qx[0], _mm256_shuffle_epi32(y, 0x00)); sumi = _mm256_dpbusd_epi32(sumi, qx[1], _mm256_shuffle_epi32(y, 0x55)); @@ -1605,7 +1605,7 @@ static void mul_mat_q5_k_r4_q8_k(int n, const void * vx, size_t bx, const DataIn auto mins = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(lbits, 4), mf), _mm256_and_si256(_mm256_srli_epi16(hbits, 2), m30)); process_min_r4_b32(ibl, m4, mins, q8, acc); for (int ib = 0; ib < QK_K/32; ++ib) { -#ifdef HAVE_FANCY_SIMD +#ifdef HAVE_VNNI256 auto scales_d = _mm256_cvtepi8_epi32(_mm_set1_epi32(hd.val[ib])); #else auto aux = _mm_set1_epi32(hd.val[ib]); @@ -1622,7 +1622,7 @@ static void mul_mat_q5_k_r4_q8_k(int n, const void * vx, size_t bx, const DataIn qx[3] = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(lbits2, 4), mf), _mm256_and_si256(m10, _mm256_srli_epi16(hbits, 3))); for (int iy = 0; iy < nrc_y; ++iy) { auto y = _mm256_loadu_si256((const __m256i*)q8.y[iy][ibl].qs+ib); -#ifdef HAVE_FANCY_SIMD +#ifdef HAVE_VNNI256 auto sumi = _mm256_setzero_si256(); sumi = _mm256_dpbusd_epi32(sumi, qx[0], _mm256_shuffle_epi32(y, 0x00)); sumi = _mm256_dpbusd_epi32(sumi, qx[1], _mm256_shuffle_epi32(y, 0x55));