mirror of
https://github.com/ggml-org/whisper.cpp.git
synced 2026-08-12 22:29:04 +04:00
vulkan: Support quantized concat (llama/25684)
This commit is contained in:
Jeff Bolz
committed by
Georgi Gerganov
Georgi Gerganov
parent
1e64098c9e
commit
23feefc7c7
@@ -1481,6 +1481,11 @@ struct vk_op_binary_push_constants {
|
||||
float param1; float param2; int32_t param3;
|
||||
};
|
||||
|
||||
// Distinct type with the same layout so concat can overload tensor offset initialization.
|
||||
struct vk_op_concat_push_constants : vk_op_binary_push_constants {};
|
||||
static_assert(sizeof(vk_op_concat_push_constants) == sizeof(vk_op_binary_push_constants));
|
||||
static_assert(std::is_standard_layout_v<vk_op_concat_push_constants>);
|
||||
|
||||
struct vk_op_multi_add_push_constants {
|
||||
// shape for dst
|
||||
uint32_t ne20; uint32_t ne21; uint32_t ne22; uint32_t ne23;
|
||||
@@ -2246,6 +2251,40 @@ static uint32_t get_misalign_bytes(const ggml_backend_vk_context * ctx, const gg
|
||||
return ((vk_tensor_offset(t) + t->view_offs) & (ctx->device->properties.limits.minStorageBufferOffsetAlignment - 1));;
|
||||
}
|
||||
|
||||
static uint32_t ggml_vk_concat_unit_size(ggml_type type) {
|
||||
const uint32_t type_size = ggml_type_size(type);
|
||||
|
||||
if (!ggml_is_quantized(type)) {
|
||||
return type_size;
|
||||
}
|
||||
|
||||
// Use the widest existing concat shader that evenly divides a quant block.
|
||||
if (type_size % 8 == 0) {
|
||||
return 8;
|
||||
}
|
||||
if (type_size % 4 == 0) {
|
||||
return 4;
|
||||
}
|
||||
if (type_size % 2 == 0) {
|
||||
return 2;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
static bool ggml_vk_concat_supported(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst) {
|
||||
if (src0->type != src1->type || src0->type != dst->type) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!ggml_is_quantized(src0->type)) {
|
||||
const size_t type_size = ggml_type_size(src0->type);
|
||||
return type_size == 1 || type_size == 2 || type_size == 4 || type_size == 8;
|
||||
}
|
||||
|
||||
// Quantized tensor rows are block-aligned when created.
|
||||
return ggml_is_contiguous_rows(src0) && ggml_is_contiguous_rows(src1) && ggml_is_contiguous_rows(dst);
|
||||
}
|
||||
|
||||
template <typename T> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, T &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
|
||||
GGML_UNUSED(p);
|
||||
GGML_UNUSED(src0);
|
||||
@@ -10896,14 +10935,10 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
|
||||
}
|
||||
return nullptr;
|
||||
case GGML_OP_CONCAT: {
|
||||
if (src0->type != src1->type || src0->type != dst->type) {
|
||||
if (!ggml_vk_concat_supported(src0, src1, dst)) {
|
||||
return nullptr;
|
||||
}
|
||||
if (ggml_blck_size(src0->type) != 1) {
|
||||
return nullptr;
|
||||
}
|
||||
const size_t type_size = ggml_type_size(src0->type);
|
||||
switch (type_size) {
|
||||
switch (ggml_vk_concat_unit_size(src0->type)) {
|
||||
case 1:
|
||||
return ctx->device->pipeline_concat_i8;
|
||||
case 2:
|
||||
@@ -11595,6 +11630,18 @@ template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk
|
||||
GGML_UNUSED(src3);
|
||||
}
|
||||
|
||||
template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_concat_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
|
||||
const uint32_t unit_size = ggml_vk_concat_unit_size(dst->type);
|
||||
const uint32_t a_offset = get_misalign_bytes(ctx, src0) / unit_size;
|
||||
const uint32_t b_offset = get_misalign_bytes(ctx, src1) / unit_size;
|
||||
const uint32_t d_offset = get_misalign_bytes(ctx, dst) / unit_size;
|
||||
|
||||
p.misalign_offsets = (a_offset << 16) | (b_offset << 8) | d_offset;
|
||||
|
||||
GGML_UNUSED(src2);
|
||||
GGML_UNUSED(src3);
|
||||
}
|
||||
|
||||
template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_upscale_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {
|
||||
const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);
|
||||
const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);
|
||||
@@ -11630,7 +11677,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co
|
||||
}
|
||||
std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3];
|
||||
std::cerr << "), " << ggml_op_name(op) << ")");
|
||||
GGML_ASSERT(op == GGML_OP_GET_ROWS || op == GGML_OP_CPY || (!ggml_is_quantized(src0->type) && (src1 == nullptr || !ggml_is_quantized(src1->type)))); // NOLINT
|
||||
GGML_ASSERT(op == GGML_OP_GET_ROWS || op == GGML_OP_CPY || op == GGML_OP_CONCAT || (!ggml_is_quantized(src0->type) && (src1 == nullptr || !ggml_is_quantized(src1->type)))); // NOLINT
|
||||
GGML_ASSERT(dst->buffer != nullptr);
|
||||
const uint64_t ne00 = src0->ne[0];
|
||||
const uint64_t ne01 = src0->ne[1];
|
||||
@@ -11885,6 +11932,9 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co
|
||||
ne *= ggml_type_size(src0->type) / 2;
|
||||
}
|
||||
}
|
||||
if (op == GGML_OP_CONCAT && ggml_is_quantized(dst->type)) {
|
||||
ne = ne / ggml_blck_size(dst->type) * ggml_type_size(dst->type) / ggml_vk_concat_unit_size(dst->type);
|
||||
}
|
||||
// copy_to_quant has block size of 32, and each thread does QUANT_K elements.
|
||||
// Splitting into 512x512xZ wouldn't work well since each workgroup does 1024 elements.
|
||||
// So divide by block size here before splitting into 512x512 groups.
|
||||
@@ -12525,18 +12575,28 @@ static void ggml_vk_opt_step_sgd(ggml_backend_vk_context * ctx, vk_context& subc
|
||||
static void ggml_vk_concat(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
|
||||
int * op_params = (int *)dst->op_params;
|
||||
|
||||
const uint32_t src0_type_size = ggml_type_size(src0->type);
|
||||
const uint32_t src1_type_size = ggml_type_size(src1->type);
|
||||
const uint32_t dst_type_size = ggml_type_size(dst->type);
|
||||
const uint32_t unit_size = ggml_vk_concat_unit_size(dst->type);
|
||||
const uint32_t units_per_block = ggml_type_size(dst->type) / unit_size;
|
||||
const uint32_t block_size = ggml_blck_size(dst->type);
|
||||
const bool quantized = ggml_is_quantized(dst->type);
|
||||
|
||||
ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_CONCAT, {
|
||||
(uint32_t)ggml_nelements(dst),
|
||||
(uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size,
|
||||
(uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size,
|
||||
(uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size,
|
||||
// Address dimension 0 in packed storage units; higher strides may be noncontiguous.
|
||||
const uint32_t ne00 = src0->ne[0] / block_size * units_per_block;
|
||||
const uint32_t ne10 = src1->ne[0] / block_size * units_per_block;
|
||||
const uint32_t ne20 = dst->ne[0] / block_size * units_per_block;
|
||||
const uint32_t nb00 = quantized ? 1 : src0->nb[0] / unit_size;
|
||||
const uint32_t nb10 = quantized ? 1 : src1->nb[0] / unit_size;
|
||||
const uint32_t nb20 = quantized ? 1 : dst->nb[0] / unit_size;
|
||||
|
||||
vk_op_concat_push_constants pc {{
|
||||
ne20 * (uint32_t)dst->ne[1] * (uint32_t)dst->ne[2] * (uint32_t)dst->ne[3],
|
||||
ne00, (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], nb00, (uint32_t)src0->nb[1] / unit_size, (uint32_t)src0->nb[2] / unit_size, (uint32_t)src0->nb[3] / unit_size,
|
||||
ne10, (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], nb10, (uint32_t)src1->nb[1] / unit_size, (uint32_t)src1->nb[2] / unit_size, (uint32_t)src1->nb[3] / unit_size,
|
||||
ne20, (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], nb20, (uint32_t) dst->nb[1] / unit_size, (uint32_t) dst->nb[2] / unit_size, (uint32_t) dst->nb[3] / unit_size,
|
||||
0,
|
||||
0.0f, 0.0f, op_params[0],
|
||||
});
|
||||
}};
|
||||
ggml_vk_op_f32<vk_op_concat_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_CONCAT, std::move(pc));
|
||||
}
|
||||
|
||||
static void ggml_vk_upscale(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
|
||||
@@ -17872,12 +17932,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
|
||||
return op->src[0]->type == op->src[1]->type && op->src[0]->type == op->type &&
|
||||
(op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_I32);
|
||||
case GGML_OP_CONCAT: {
|
||||
if (op->src[0]->type != op->src[1]->type || op->src[0]->type != op->type) {
|
||||
return false;
|
||||
}
|
||||
const size_t type_size = ggml_type_size(op->type);
|
||||
return ggml_blck_size(op->type) == 1 &&
|
||||
(type_size == 1 || type_size == 2 || type_size == 4 || type_size == 8);
|
||||
return ggml_vk_concat_supported(op->src[0], op->src[1], op);
|
||||
}
|
||||
case GGML_OP_ADD1:
|
||||
return (op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32)
|
||||
|
||||
Reference in New Issue
Block a user