LLM-Inference/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-08-12 22:29:39 +04:00
Code Issues Packages Projects Releases Wiki Activity

Allow concatenating quantized tensors (#2232)

Browse Source 
* Allow concatenating quantized tensors

* Missed this assert

* Allow K to be f32 in ggml_cuda_op_indexer_topk
This commit is contained in:
Kawrakow
2026-08-03 08:01:57 +03:00
committed by GitHub
parent c3b075f069
commit bee545824a
2 changed files with 8 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-cuda/indexer_topk.cu
+2 -1
View File
@@ -69,7 +69,8 @@ void ggml_cuda_op_indexer_topk(ggml_backend_cuda_context & ctx, ggml_tensor * ds
int n_top_k = dst->ne[0];
int n_kv = k->ne[1];
int n_head = q->ne[1];
GGML_ASSERT(k->type == GGML_TYPE_F16 || ggml_is_quantized(k->type));
//if (k->type != GGML_TYPE_F16 && !ggml_is_quantized(k->type)) printf("%s: K is %s?\n", __func__, ggml_type_name(k->type));
GGML_ASSERT(k->type == GGML_TYPE_F16 || k->type == GGML_TYPE_F32 || ggml_is_quantized(k->type));
GGML_ASSERT(k->ne[2] == 1 || k->ne[3] == 1);
GGML_ASSERT(k->ne[1] > n_top_k);
GGML_ASSERT(k->ne[1] == m->ne[0]);
ggml/src/ggml.c
+6 -3
View File
@@ -15469,10 +15469,13 @@ static bool ggml_compute_forward_concat_any_opt(
const struct ggml_tensor * src0 = dst->src[0];
const struct ggml_tensor * src1 = dst->src[1];
if (ggml_is_quantized(src0->type)) return false;
if (ggml_is_quantized(src0->type)) {
size_t row_meta = type_traits[src0->type].row_meta_size;
if (row_meta > 0) return false; // We cannot concatenate quants that has per row meta data
}
//if (ggml_is_quantized(src0->type)) return false;
GGML_ASSERT(src0->type == src1->type && src0->type == dst->type);
GGML_ASSERT(!ggml_is_quantized(src0->type));
const int ith = params->ith;
const int nth = params->nth;
@@ -15521,7 +15524,7 @@ static bool ggml_compute_forward_concat_any_opt(
if (d > 0) nrows *= dst->ne[d];
}
size_t row_size = ggml_row_size(dst->type, dst->ne[0]);
if (src0->nb[1] == row_size && src1->nb[1] == row_size) {
if (src0->nb[1] >= row_size && src1->nb[1] >= row_size) {
int npt = (nrows + nth - 1)/nth;
int first = ith*npt;
int last = MIN(first + npt, nrows);