From 592feef04a1802b18cbeffd0fd0eb5d02570c2ec Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 7 Aug 2026 17:12:59 +0300 Subject: [PATCH] talk-llama : sync llama.cpp --- examples/talk-llama/llama-arch.cpp | 2 + examples/talk-llama/llama-arch.h | 1 + examples/talk-llama/llama-ext.h | 6 +++ examples/talk-llama/llama-grammar.cpp | 6 ++- examples/talk-llama/llama-model-loader.cpp | 18 +++++++- examples/talk-llama/llama-model.cpp | 53 ++++++++++++++++++++++ examples/talk-llama/llama-sampler.cpp | 20 ++++---- examples/talk-llama/llama-sampler.h | 1 - examples/talk-llama/llama-vocab.cpp | 19 +++++--- examples/talk-llama/llama.h | 5 +- examples/talk-llama/models/dflash.cpp | 2 +- examples/talk-llama/models/models.h | 5 ++ examples/talk-llama/models/qwen3tts.cpp | 3 ++ examples/talk-llama/models/qwen3vl.cpp | 25 +++++++++- 14 files changed, 137 insertions(+), 29 deletions(-) create mode 100644 examples/talk-llama/models/qwen3tts.cpp diff --git a/examples/talk-llama/llama-arch.cpp b/examples/talk-llama/llama-arch.cpp index ea0ddd114..836cfade2 100644 --- a/examples/talk-llama/llama-arch.cpp +++ b/examples/talk-llama/llama-arch.cpp @@ -144,6 +144,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_TALKIE, "talkie" }, { LLM_ARCH_MELLUM, "mellum" }, { LLM_ARCH_NANBEIGE, "nanbeige" }, + { LLM_ARCH_QWEN3TTS, "qwen3tts" }, { LLM_ARCH_UNKNOWN, "(unknown)" }, }; @@ -1026,6 +1027,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) { case LLM_ARCH_MINIMAX_M3: case LLM_ARCH_MISTRAL4: case LLM_ARCH_KIMI_LINEAR: + case LLM_ARCH_QWEN3TTS: return false; default: return true; diff --git a/examples/talk-llama/llama-arch.h b/examples/talk-llama/llama-arch.h index cbc97085e..49c2a6ac3 100644 --- a/examples/talk-llama/llama-arch.h +++ b/examples/talk-llama/llama-arch.h @@ -149,6 +149,7 @@ enum llm_arch { LLM_ARCH_MINIMAX_M3, LLM_ARCH_DFLASH, LLM_ARCH_NANBEIGE, + LLM_ARCH_QWEN3TTS, LLM_ARCH_UNKNOWN, }; diff --git a/examples/talk-llama/llama-ext.h b/examples/talk-llama/llama-ext.h index 348bbae95..35d6e58ad 100644 --- a/examples/talk-llama/llama-ext.h +++ b/examples/talk-llama/llama-ext.h @@ -124,3 +124,9 @@ LLAMA_API llama_context * llama_get_ctx_other(struct llama_context * ctx); LLAMA_API const int32_t * llama_model_target_layer_ids (const struct llama_model * model); // returns the number of extracted layers from target model LLAMA_API uint32_t llama_model_target_layer_ids_n(const struct llama_model * model); + +// retrieves the whole token embedding matrix in F32 format (n_embd * n_vocab) +// returns total number of elements or 0 on error +// if out is nullptr, returns the number of tokens without writing to out +// caller must allocate enough memory for out before calling +LLAMA_API uint32_t llama_model_get_tok_embd(const struct llama_model * model, float * out); diff --git a/examples/talk-llama/llama-grammar.cpp b/examples/talk-llama/llama-grammar.cpp index 363644464..c685346b6 100644 --- a/examples/talk-llama/llama-grammar.cpp +++ b/examples/talk-llama/llama-grammar.cpp @@ -648,10 +648,12 @@ const char * llama_grammar_parser::parse_sequence( } else { throw std::runtime_error(std::string("expecting ',' at ") + pos); } - bool has_max = max_times != UINT64_MAX; - if (min_times > MAX_REPETITION_THRESHOLD || (has_max && max_times > MAX_REPETITION_THRESHOLD)) { + if (min_times > MAX_REPETITION_THRESHOLD) { throw std::runtime_error(std::string("number of repetitions exceeds sane defaults, please reduce the number of repetitions")); } + if (max_times != UINT64_MAX && max_times > MAX_REPETITION_THRESHOLD) { + max_times = UINT64_MAX; + } handle_repetitions(min_times, max_times); } else { break; diff --git a/examples/talk-llama/llama-model-loader.cpp b/examples/talk-llama/llama-model-loader.cpp index b31e92e2d..71bc9f7ef 100644 --- a/examples/talk-llama/llama-model-loader.cpp +++ b/examples/talk-llama/llama-model-loader.cpp @@ -1249,7 +1249,13 @@ struct ggml_tensor * llama_model_loader::create_tensor( for (size_t dim = 0; dim < GGML_MAX_DIMS; dim++) { t_meta.ne[dim] = dim < ne.size() ? ne.begin()[dim] : 1; GGML_ASSERT(t_meta.ne[dim] >= 1); - t_meta.nb[dim] = dim == 0 ? ggml_type_size(type) : t_meta.ne[dim-1]*t_meta.nb[dim-1]; + if (dim == 0) { + t_meta.nb[dim] = ggml_type_size(type); + } else if (dim == 1) { + t_meta.nb[dim] = ggml_row_size(type, t_meta.ne[dim-1]); + } else { + t_meta.nb[dim] = t_meta.nb[dim-1]*t_meta.ne[dim-1]; + } GGML_ASSERT(t_meta.nb[dim] >= 1); } ggml_set_name(&t_meta, tn.str().c_str()); @@ -1272,10 +1278,18 @@ struct ggml_tensor * llama_model_loader::create_tensor( if (flags & TENSOR_ALLOW_RESHAPE) { for (size_t dim = 0; dim < GGML_MAX_DIMS; dim++) { t_meta.ne[dim] = dim < ne.size() ? ne.begin()[dim] : 1; - t_meta.nb[dim] = dim == 0 ? ggml_type_size(t_meta.type) : t_meta.ne[dim-1]*t_meta.nb[dim-1]; + if (dim == 0) { + t_meta.nb[dim] = ggml_type_size(t_meta.type); + } else if (dim == 1) { + t_meta.nb[dim] = ggml_row_size(t_meta.type, t_meta.ne[dim-1]); + } else { + t_meta.nb[dim] = t_meta.ne[dim-1]*t_meta.nb[dim-1]; + } } } + GGML_ASSERT(ggml_nbytes(&t_meta) == ggml_nbytes(cur)); + ggml_backend_buffer_type_t buft = buft_for_tensor(&t_meta); if (buft == nullptr) { return nullptr; diff --git a/examples/talk-llama/llama-model.cpp b/examples/talk-llama/llama-model.cpp index 333f506de..4cc1c0a1c 100644 --- a/examples/talk-llama/llama-model.cpp +++ b/examples/talk-llama/llama-model.cpp @@ -112,6 +112,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_qwen3vl(params); case LLM_ARCH_QWEN3VLMOE: return new llama_model_qwen3vlmoe(params); + case LLM_ARCH_QWEN3TTS: + return new llama_model_qwen3tts(params); case LLM_ARCH_PHI2: return new llama_model_phi2(params); case LLM_ARCH_PHI3: @@ -2693,6 +2695,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_QWEN3VLMOE: case LLM_ARCH_QWEN35: case LLM_ARCH_QWEN35MOE: + case LLM_ARCH_QWEN3TTS: return LLAMA_ROPE_TYPE_IMROPE; case LLM_ARCH_GLM4: @@ -2887,6 +2890,21 @@ void llama_model_base::create_tensor_qkv(llama_layer & layer, int bid, int64_t n_embd_, int64_t n_embd_q_, int64_t n_embd_k_, int64_t n_embd_v_, int flags) { const int64_t n_embd_qkv = n_embd_q_ + n_embd_k_ + n_embd_v_; + + if (flags & TENSOR_SKIP) { + const int skip = TENSOR_NOT_REQUIRED | TENSOR_SKIP; + + create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", bid), {n_embd_, n_embd_qkv}, skip | TENSOR_SKIP_IF_VIRTUAL); + create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", bid), {n_embd_qkv}, skip | TENSOR_SKIP_IF_VIRTUAL); + create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", bid), {n_embd_, n_embd_q_}, skip); + create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", bid), {n_embd_, n_embd_k_}, skip); + create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", bid), {n_embd_, n_embd_v_}, skip); + create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", bid), {n_embd_q_}, skip); + create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", bid), {n_embd_k_}, skip); + create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", bid), {n_embd_v_}, skip); + return; + } + layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", bid), {n_embd_, n_embd_qkv}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL); if (layer.wqkv) { layer.wqkv_b = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", bid), {n_embd_qkv}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL); @@ -2908,3 +2926,38 @@ const int32_t * llama_model_target_layer_ids(const struct llama_model * model) { uint32_t llama_model_target_layer_ids_n(const struct llama_model * model) { return (uint32_t) model->target_layer_ids.size(); } + +uint32_t llama_model_get_tok_embd(const struct llama_model * model, float * out) { + if (model->vocab.n_tokens() == 0 || model->tok_embd == nullptr) { + return 0; + } + + const ggml_tensor * tensor = model->tok_embd; + const size_t nelements = ggml_nelements(tensor); + GGML_ASSERT(nelements <= UINT32_MAX); // for the return type + + if (out == nullptr) { + return (uint32_t) nelements; + } + + if (tensor->type == GGML_TYPE_F32) { + ggml_backend_tensor_get(tensor, out, 0, nelements * sizeof(float)); + return (uint32_t) nelements; + } + + std::vector buf(ggml_nbytes(tensor)); + ggml_backend_tensor_get(tensor, buf.data(), 0, buf.size()); + + const ggml_type_traits * traits = ggml_get_type_traits(tensor->type); + if (tensor->type == GGML_TYPE_F16) { + ggml_fp16_to_fp32_row((const ggml_fp16_t *) buf.data(), out, nelements); + } else if (tensor->type == GGML_TYPE_BF16) { + ggml_bf16_to_fp32_row((const ggml_bf16_t *) buf.data(), out, nelements); + } else if (ggml_is_quantized(tensor->type) && traits->to_float != nullptr) { + traits->to_float(buf.data(), out, nelements); + } else { + GGML_ABORT("unsupported tensor type for dequantization: %s", ggml_type_name(tensor->type)); + } + + return (uint32_t) nelements; +} diff --git a/examples/talk-llama/llama-sampler.cpp b/examples/talk-llama/llama-sampler.cpp index 6cf2d27cf..e550fbe4a 100644 --- a/examples/talk-llama/llama-sampler.cpp +++ b/examples/talk-llama/llama-sampler.cpp @@ -3078,8 +3078,6 @@ struct llama_sampler * llama_sampler_init_top_n_sigma(float n) { // DRY struct llama_sampler_dry { - int32_t total_context_size; - const float dry_multiplier; const float dry_base; const int32_t dry_allowed_length; @@ -3155,8 +3153,7 @@ static void llama_sampler_dry_apply(struct llama_sampler * smpl, llama_token_dat return; } - int32_t effective_dry_penalty_last_n = (ctx->dry_penalty_last_n == -1) ? ctx->total_context_size : std::max(ctx->dry_penalty_last_n, 0); - int last_n_repeat = std::min(std::min((int)ctx->last_tokens.size(), effective_dry_penalty_last_n), ctx->total_context_size); + int last_n_repeat = std::min((int) ctx->last_tokens.size(), ctx->dry_penalty_last_n); if (last_n_repeat <= ctx->dry_allowed_length) { return; @@ -3369,7 +3366,7 @@ static struct llama_sampler * llama_sampler_dry_clone(const struct llama_sampler llama_vocab dummy_vocab; // dummy vocab is passed because it is only needed for raw sequence breaker processing, which we have already done and will simply be copying - auto * result = llama_sampler_init_dry(&dummy_vocab, ctx->total_context_size, ctx->dry_multiplier, ctx->dry_base, ctx->dry_allowed_length, ctx->dry_penalty_last_n, NULL, 0); + auto * result = llama_sampler_init_dry(&dummy_vocab, ctx->dry_multiplier, ctx->dry_base, ctx->dry_allowed_length, ctx->dry_penalty_last_n, NULL, 0); // Copy the state, including the processed breakers { @@ -3400,8 +3397,8 @@ static struct llama_sampler_i llama_sampler_dry_i = { /* .backend_set_input = */ nullptr, }; -struct llama_sampler * llama_sampler_init_dry(const struct llama_vocab * vocab, int32_t n_ctx_train, float dry_multiplier, float dry_base, int32_t dry_allowed_length, int32_t dry_penalty_last_n, const char** seq_breakers, size_t num_breakers) { - int32_t effective_dry_penalty_last_n = (dry_penalty_last_n == -1) ? n_ctx_train : std::max(dry_penalty_last_n, 0); +struct llama_sampler * llama_sampler_init_dry(const struct llama_vocab * vocab, float dry_multiplier, float dry_base, int32_t dry_allowed_length, int32_t dry_penalty_last_n, const char** seq_breakers, size_t num_breakers) { + dry_penalty_last_n = std::max(dry_penalty_last_n, 0); std::unordered_multimap> processed_breakers; const int MAX_CHAR_LEN = 40; const int MAX_SEQ_LEN = 20; @@ -3438,23 +3435,22 @@ struct llama_sampler * llama_sampler_init_dry(const struct llama_vocab * vocab, return llama_sampler_init( /* .iface = */ &llama_sampler_dry_i, /* .ctx = */ new llama_sampler_dry { - /* .total_context_size = */ n_ctx_train, /* .dry_multiplier = */ dry_multiplier, /* .dry_base = */ dry_base, /* .dry_allowed_length = */ dry_allowed_length, /* .dry_penalty_last_n = */ dry_penalty_last_n, /* .dry_processed_breakers = */ std::move(processed_breakers), - /* .dry_repeat_count = */ dry_enabled ? std::vector(effective_dry_penalty_last_n, 0) : std::vector{}, + /* .dry_repeat_count = */ dry_enabled ? std::vector(dry_penalty_last_n, 0) : std::vector{}, /* .dry_max_token_repeat = */ {}, - /* .last_tokens = */ dry_enabled ? ring_buffer(effective_dry_penalty_last_n) : ring_buffer(0), + /* .last_tokens = */ dry_enabled ? ring_buffer(dry_penalty_last_n) : ring_buffer(0), } ); } // wrapper for test-sampling.cpp -struct llama_sampler * llama_sampler_init_dry_testing(int32_t context_size, float dry_multiplier, float dry_base, int32_t dry_allowed_length, int32_t dry_penalty_last_n, const std::vector>& seq_breakers) { +struct llama_sampler * llama_sampler_init_dry_testing(float dry_multiplier, float dry_base, int32_t dry_allowed_length, int32_t dry_penalty_last_n, const std::vector>& seq_breakers) { llama_vocab dummy_vocab; - auto * result = llama_sampler_init_dry(&dummy_vocab, context_size, dry_multiplier, dry_base, dry_allowed_length, dry_penalty_last_n, NULL, 0); + auto * result = llama_sampler_init_dry(&dummy_vocab, dry_multiplier, dry_base, dry_allowed_length, dry_penalty_last_n, NULL, 0); auto * ctx = (llama_sampler_dry *) result->ctx; // Process the token-based sequence breakers diff --git a/examples/talk-llama/llama-sampler.h b/examples/talk-llama/llama-sampler.h index b9bfc20d2..929207514 100644 --- a/examples/talk-llama/llama-sampler.h +++ b/examples/talk-llama/llama-sampler.h @@ -34,7 +34,6 @@ struct llama_sampler_chain { }; struct llama_sampler * llama_sampler_init_dry_testing( - int32_t context_size, float dry_multiplier, float dry_base, int32_t dry_allowed_length, diff --git a/examples/talk-llama/llama-vocab.cpp b/examples/talk-llama/llama-vocab.cpp index 10032a8c6..4a01dfd4c 100644 --- a/examples/talk-llama/llama-vocab.cpp +++ b/examples/talk-llama/llama-vocab.cpp @@ -1373,8 +1373,10 @@ struct llm_tokenizer_plamo2 : llm_tokenizer { if (vocab.is_byte(token_id)) { if (entry.text.length() == 6 && entry.text.substr(0, 3) == "<0x" && entry.text.back() == '>') { std::string hex_str = entry.text.substr(3, 2); - int byte_val = std::stoi(hex_str, nullptr, 16); - bytes_[byte_val] = static_cast(token_id); + if (std::isxdigit(static_cast(hex_str[0])) && std::isxdigit(static_cast(hex_str[1]))) { + int byte_val = std::stoi(hex_str, nullptr, 16); + bytes_[byte_val] = static_cast(token_id); + } } continue; } @@ -3625,12 +3627,15 @@ int32_t llama_vocab::impl::token_to_piece(llama_token token, char * buf, int32_t if (vocab.is_byte(token)) { // Handle byte tokens like <0xXX> if (token_text.length() == 6 && token_text.substr(0, 3) == "<0x" && token_text.back() == '>') { - int hex_val = std::stoi(token_text.substr(3, 2), nullptr, 16); - if (length < 1) { - return -1; + std::string hex_str = token_text.substr(3, 2); + if (std::isxdigit(static_cast(hex_str[0])) && std::isxdigit(static_cast(hex_str[1]))) { + int hex_val = std::stoi(hex_str, nullptr, 16); + if (length < 1) { + return -1; + } + buf[0] = static_cast(hex_val); + return 1; } - buf[0] = static_cast(hex_val); - return 1; } } diff --git a/examples/talk-llama/llama.h b/examples/talk-llama/llama.h index fb2ca38ce..a14498925 100644 --- a/examples/talk-llama/llama.h +++ b/examples/talk-llama/llama.h @@ -1425,7 +1425,7 @@ extern "C" { /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first. LLAMA_API struct llama_sampler * llama_sampler_init_penalties( int32_t n_vocab, - int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size) + int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty) float penalty_repeat, // must be > 0.0, 1.0 = disabled float penalty_freq, // must be finite, 0.0 = disabled float penalty_present); // must be finite, 0.0 = disabled @@ -1433,11 +1433,10 @@ extern "C" { /// @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982 LLAMA_API struct llama_sampler * llama_sampler_init_dry( const struct llama_vocab * vocab, - int32_t n_ctx_train, float dry_multiplier, float dry_base, int32_t dry_allowed_length, - int32_t dry_penalty_last_n, + int32_t dry_penalty_last_n, // last n tokens to penalize (0 = disable penalty) const char ** seq_breakers, size_t num_breakers); diff --git a/examples/talk-llama/models/dflash.cpp b/examples/talk-llama/models/dflash.cpp index 6c82ab3da..daff6e78f 100644 --- a/examples/talk-llama/models/dflash.cpp +++ b/examples/talk-llama/models/dflash.cpp @@ -125,7 +125,7 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) { layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head * n_embd_head}, 0); layer.wkv = create_tensor(tn(LLM_TENSOR_ATTN_KV, "weight", i), {n_embd, n_embd_head}, 0); layer.attn_kv_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_NORM, "weight", i), {n_embd_head}, 0); - layer.wo_a = create_tensor(tn(LLM_TENSOR_ATTN_OUT_A, "weight", i), {n_head * n_embd_head / o_groups, o_lora_rank * o_groups}, 0); + layer.wo_a = create_tensor(tn(LLM_TENSOR_ATTN_OUT_A, "weight", i), {n_head * n_embd_head / o_groups, o_lora_rank, o_groups}, TENSOR_ALLOW_RESHAPE); layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT_B, "weight", i), {o_groups * o_lora_rank, n_embd}, 0); layer.hc_attn_fn = create_tensor(tn(LLM_TENSOR_HC_ATTN_FN, "weight", i), {hc_dim, hc_mix_dim}, 0); diff --git a/examples/talk-llama/models/models.h b/examples/talk-llama/models/models.h index 5f206621d..ad3dadaf3 100644 --- a/examples/talk-llama/models/models.h +++ b/examples/talk-llama/models/models.h @@ -596,6 +596,11 @@ struct llama_model_qwen3vlmoe : public llama_model_base { }; +struct llama_model_qwen3tts : public llama_model_qwen3vl { + llama_model_qwen3tts(const struct llama_model_params & params) : llama_model_qwen3vl(params) {} +}; + + struct llama_model_phi2 : public llama_model_base { llama_model_phi2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; diff --git a/examples/talk-llama/models/qwen3tts.cpp b/examples/talk-llama/models/qwen3tts.cpp new file mode 100644 index 000000000..3604f844c --- /dev/null +++ b/examples/talk-llama/models/qwen3tts.cpp @@ -0,0 +1,3 @@ +#include "models.h" + +// llama_model_qwen3tts reuses llama_model_qwen3vl's hparams/tensors/graph logic diff --git a/examples/talk-llama/models/qwen3vl.cpp b/examples/talk-llama/models/qwen3vl.cpp index 724d6140d..5596620f0 100644 --- a/examples/talk-llama/models/qwen3vl.cpp +++ b/examples/talk-llama/models/qwen3vl.cpp @@ -16,11 +16,16 @@ void llama_model_qwen3vl::load_arch_hparams(llama_model_loader & ml) { void llama_model_qwen3vl::load_arch_tensors(llama_model_loader &) { LLAMA_LOAD_LOCALS; + int64_t n_vocab_out = n_vocab; + if (arch == LLM_ARCH_QWEN3TTS) { + n_vocab_out = 3072; + } + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); // output output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); - output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab_out}, TENSOR_NOT_REQUIRED); // if output is NULL, init from the input tok embed if (output == NULL) { output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); @@ -166,6 +171,24 @@ llama_model_qwen3vl::graph::graph(const llama_model & model, const llm_graph_par // lm_head cur = build_lora_mm(model.output, cur, model.output_s); + int64_t n_vocab_in = model.tok_embd->ne[1]; + int64_t n_vocab_out = model.output->ne[1]; + if (n_vocab_in > n_vocab_out) { + // case: Qwen3TTS model with codec_head as output + GGML_ASSERT(model.output_norm); + int64_t pad = n_vocab_in - n_vocab_out; + + // using this trick to get a scalar -inf tensor to pad the output + ggml_tensor * neg_inf = ggml_scale_bias(ctx0, + ggml_view_1d(ctx0, model.output_norm, 1, 0), + 0.0f, -INFINITY); + neg_inf = ggml_repeat_4d(ctx0, neg_inf, pad, cur->ne[1], 1, 1); + cur = ggml_concat(ctx0, neg_inf, cur, 0); // [padded .. n_vocab_out, n_stream] + + } else if (n_vocab_in < n_vocab_out) { + GGML_ABORT("invalid case"); + } + cb(cur, "result_output", -1); res->t_logits = cur;