LLM-Inference/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-08-12 22:31:11 +04:00
Code Issues Packages Projects Releases Wiki Activity

clean up comments

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-08-04 15:18:19 +02:00
parent 2f40d38c7c
commit 201e50cc20
13 changed files with 81 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conversion/qwen3tts.py
+10 -23
View File
@@ -216,8 +216,7 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
if hparams is None:
hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)
hparams["text_config"] = {"hidden_size": hparams["talker_config"]["hidden_size"]}
# ECAPA-TDNN has a fixed 4-stage backbone, not a configurable transformer depth;
# MmprojModel.__init__ still needs one of the n_block_keys to build its tensor map
# ECAPA-TDNN has a fixed 4-stage backbone, but MmprojModel.__init__ needs a n_block_keys
hparams["speaker_encoder_config"]["n_layers"] = 4
super().__init__(dir_model, *args, hparams=hparams, **kwargs)
self._wav_config_cache = None
@@ -234,7 +233,7 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
# mel_spectrogram() front-end: sr=24000, n_fft=1024, hop=256, n_mels=128, fmin=0, fmax=12000 (=sr/2, the clip.cpp default)
self.gguf_writer.add_audio_num_mel_bins(128)
# the 3 SE-Res2Net stages (blocks 1-3); the stem conv, mfa, asp and fc are singletons, not part of this count
# 3 SE-Res2Net stages; the stem conv, mfa, asp and fc are not counted here
self.gguf_writer.add_audio_block_count(3)
# ECAPA-TDNN has no attention/FFN, these are dummy to allow clip.cpp to load it
self.gguf_writer.add_audio_embedding_length(1536)
@@ -256,8 +255,7 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
# note: code2wav hparams are hardcoded on the mtmd/clip.cpp side for now, not written here
def _wav_decoder_config(self) -> dict[str, Any] | None:
# code2wav (RVQ codes -> raw PCM) lives in its own checkpoint dir, sibling to
# the main safetensors, with its own config.json
# code2wav has its own config.json, inside the speech_tokenizer dir
if self._wav_config_cache is None:
path = self.dir_model / "speech_tokenizer" / "config.json"
with open(path, "r", encoding="utf-8") as f:
@@ -266,18 +264,14 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
return self._wav_config_cache
def tensor_force_quant(self, name, new_name, bid, n_dims):
# regular (non-transpose) conv1d/conv1d_dw weights must be F16, never BF16:
# ggml_conv_1d(_dw) pairs the kernel as mul_mat's src1 against an F32 im2col
# src0, and the CPU backend only accepts src1 in F32 -- BF16 kernels can't be
# scheduled.
# conv1d/conv1d_dw kernels must be F16, ggml_conv_1d(_dw) has no BF16 path
if new_name.endswith(".weight") and (
new_name in ("a.gen.wav.pre_conv.weight", "a.gen.wav.dac.entry.weight", "a.gen.wav.dac.post_conv.weight")
or (".up.blk." in new_name and new_name.endswith(".dwconv.weight"))
or (".dac.blk." in new_name and (new_name.endswith(".conv1.weight") or new_name.endswith(".conv2.weight")))
):
return gguf.GGMLQuantizationType.F16
# causal ConvTranspose1d weights: ggml_compute_forward_conv_transpose_1d
# only implements F16/F32 kernels, never BF16
# ConvTranspose1d kernels: only F16/F32 are implemented, no BF16
if new_name.endswith(".conv.weight") and (".up.blk." in new_name or ".dac.blk." in new_name):
return gguf.GGMLQuantizationType.F32
return super().tensor_force_quant(name, new_name, bid, n_dims)
@@ -296,16 +290,12 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
return super().filter_tensors((name, gen))
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
# code2wav tensors come from generate_extra_tensors() already renamed to their
# final gguf name (bypassing tensor_mapping.py, same as the code_predictor
# tensors below); prepare_tensors() re-runs modify_tensors() on them too, so
# pass them through untouched instead of falling into map_tensor_name()
# code2wav tensors are already named by generate_extra_tensors(), pass them through
if name.startswith("a.gen.wav."):
yield (name, data_torch)
return
# codebook-0 embedding: fed back into the talker backbone once a codec token is
# generated, the counterpart of code_predictor's codec_embedding.{0..14} for codebooks 1-15
# codebook-0 embedding, fed back to the talker backbone (codebooks 1-15 live in code_predictor)
if name == "talker.model.codec_embedding.weight":
yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_GEN_CODE_OUT_EMBD), data_torch)
return
@@ -360,8 +350,7 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
yield from self._generate_code2wav_tensors()
def _generate_code2wav_tensors(self) -> Iterable[tuple[str, Tensor]]:
# code2wav lives in its own checkpoint dir (speech_tokenizer/model.safetensors),
# not the main safetensors this ModelBase was constructed from
# code2wav weights live in speech_tokenizer/model.safetensors, not the main safetensors
from safetensors.torch import load_file
wav_config = self._wav_decoder_config()
@@ -371,13 +360,11 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
return state_dict[name]
def snake_fold(alpha: Tensor, beta: Tensor) -> tuple[Tensor, Tensor]:
# SnakeBeta activation folds its exp()/reciprocal at conversion time, so the
# runtime graph is only mul -> sin -> sqr -> mul -> add
# fold SnakeBeta's exp()/reciprocal here, so the graph is only mul/sin/sqr/mul/add
return torch.exp(alpha), 1.0 / (torch.exp(beta) + 1e-9)
def rvq_codebook(prefix: str, n_layers: int) -> Tensor:
# the checkpoint stores EMA training accumulators, not a ready embedding
# table: codebook[i] = embedding_sum[i] / cluster_usage[i]
# checkpoint has EMA accumulators, so codebook[i] = embedding_sum[i] / cluster_usage[i]
books = []
for i in range(n_layers):
embedding_sum = get(f"{prefix}.vq.layers.{i}._codebook.embedding_sum")
gguf-py/gguf/constants.py
+2 -3
View File
@@ -976,7 +976,7 @@ class MODEL_TENSOR(IntEnum):
A_ENC_SE_CONV2 = auto() # qwen3tts
A_ENC_ASP_ATTN = auto() # qwen3tts
A_ENC_ASP_TDNN = auto() # qwen3tts
# qwen3tts code_predictor: autoregressively predicts the remaining RVQ codebooks
# qwen3tts code_predictor: predicts the remaining RVQ codebooks
A_GEN_CODE_PROJ_IN = auto() # small_to_mtp_projection
A_GEN_CODE_EMBD = auto() # per-codebook embedding table, merged 3D [n_codebooks, vocab, dim]
A_GEN_CODE_HEAD = auto() # per-codebook output head, merged 3D [n_codebooks, vocab, dim]
@@ -993,8 +993,7 @@ class MODEL_TENSOR(IntEnum):
A_GEN_CODE_FFN_UP = auto()
A_GEN_CODE_FFN_DOWN = auto()
A_GEN_CODE_OUTPUT_NORM = auto()
# qwen3tts code2wav: RVQ codes -> raw PCM (quantizer decode + pre_conv +
# pre_transformer + ConvNeXt upsample + DAC decoder)
# qwen3tts code2wav: RVQ codes -> raw PCM
A_GEN_WAV_QUANT_FIRST_IN = auto() # semantic RVQ, in_proj (1x1 conv, loaded as 2D)
A_GEN_WAV_QUANT_FIRST_OUT = auto() # semantic RVQ, out_proj
A_GEN_WAV_QUANT_FIRST_CB = auto() # semantic RVQ codebook (1 layer), folded from embedding_sum/cluster_usage
tools/mtmd/clip-impl.h
+1 -1
View File
@@ -219,7 +219,7 @@
#define TN_A_GEN_CODE_NORM "a.gen.code.output_norm.%s"
// qwen3tts code2wav (RVQ codes -> raw PCM)
// pre_transformer per-layer tensors are loaded through the generic TN_ATTN_*/TN_FFN_*/TN_LN_*/TN_LS_* macros with prefix "a.gen.wav.tfm"
// pre_transformer layers use the generic TN_ATTN_*/TN_FFN_*/TN_LN_*/TN_LS_* macros, prefix "a.gen.wav.tfm"
#define TN_A_GEN_WAV_QUANT_FIRST_IN "a.gen.wav.quant.first.in_proj.%s"
#define TN_A_GEN_WAV_QUANT_FIRST_OUT "a.gen.wav.quant.first.out_proj.%s"
#define TN_A_GEN_WAV_QUANT_FIRST_CB "a.gen.wav.quant.first.codebook.%s"
tools/mtmd/clip-model.h
+5 -9
View File
@@ -299,8 +299,7 @@ struct clip_layer {
ggml_tensor * cross_attn_norm_w = nullptr;
ggml_tensor * cross_attn_norm_b = nullptr;
// qwen3tts speaker encoder: SE-Res2Net block (conv_pw1_w/b and conv_pw2_w/b
// above are reused for this block's tdnn1/tdnn2 bottleneck convs)
// qwen3tts speaker encoder: SE-Res2Net block, tdnn1/tdnn2 reuse conv_pw1_w/b and conv_pw2_w/b above
ggml_tensor * se_conv1_w = nullptr;
ggml_tensor * se_conv1_b = nullptr;
ggml_tensor * se_conv2_w = nullptr;
@@ -389,8 +388,7 @@ struct qf_block {
// qwen3tts code2wav: RVQ codes -> raw PCM
struct clip_code2wav {
// one ConvNeXt block + its preceding causal ConvTranspose1d
// (the "upsample" stage between pre_transformer and the DAC decoder)
// "upsample" stage: one ConvNeXt block plus the causal ConvTranspose1d before it
struct upsample_block {
ggml_tensor * conv_w = nullptr; // causal ConvTranspose1d, 2x
ggml_tensor * conv_b = nullptr;
@@ -405,8 +403,7 @@ struct clip_code2wav {
ggml_tensor * gamma = nullptr; // layer scale
};
// one DAC residual unit
// (SnakeBeta -> dilated causal conv -> SnakeBeta -> pointwise causal conv)
// one DAC residual unit: SnakeBeta -> dilated causal conv -> SnakeBeta -> pointwise causal conv
struct dac_res {
ggml_tensor * act1_alpha = nullptr;
ggml_tensor * act1_beta = nullptr;
@@ -668,9 +665,8 @@ struct clip_model {
ggml_tensor * conv2d_3_w = nullptr;
ggml_tensor * conv2d_3_b = nullptr;
// qwen3tts speaker encoder (ECAPA-TDNN): the stem conv (block 0) reuses
// conv1d_1_w/b, the multi-layer feature aggregation conv reuses conv_out_w/b,
// and the final speaker embedding projection reuses mm_fc_w/b
// qwen3tts speaker encoder (ECAPA-TDNN)
// reused tensors: stem conv is conv1d_1_w/b, feature aggregation is conv_out_w/b, output proj is mm_fc_w/b
ggml_tensor * spk_asp_attn_w = nullptr;
ggml_tensor * spk_asp_attn_b = nullptr;
ggml_tensor * spk_asp_tdnn_w = nullptr;
tools/mtmd/clip.cpp
+10 -20
View File
@@ -1701,8 +1701,7 @@ struct clip_model_loader {
} break;
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
{
// ECAPA-TDNN speaker/voice encoder; mel_spectrogram() front-end
// matches the Slaney mel default (fmin=0, fmax=sample_rate/2)
// ECAPA-TDNN speaker encoder, mel front-end uses the Slaney default (fmin=0, fmax=sr/2)
hparams.audio_sample_rate = 24000;
hparams.audio_n_fft = 1024;
hparams.audio_window_len = 1024;
@@ -1710,7 +1709,6 @@ struct clip_model_loader {
} break;
case PROJECTOR_TYPE_QWEN3TTS_GEN:
{
// discrete-token autoregressive predictor, no mel-frontend needed
// TODO: hardcoded for now, read from code_predictor_config instead
hparams.rope_theta = 1000000.0f;
@@ -2754,9 +2752,7 @@ struct clip_model_loader {
c2w.tfm_out_proj_b = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "bias"));
c2w.tfm_output_norm_w = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_NORM, "weight"));
// pre_transformer layers: own prefix/layer-count, so loaded manually
// rather than through the generic model.layers loop (already claimed
// by code_predictor's 5 layers)
// loaded manually, the generic model.layers loop is taken by code_predictor
c2w.tfm_layers.resize(hparams.wav_tfm_n_layer);
for (int il = 0; il < hparams.wav_tfm_n_layer; il++) {
auto & layer = c2w.tfm_layers[il];
@@ -4020,8 +4016,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
} break;
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
{
// attentive statistics pooling collapses the whole clip into
// a single speaker embedding vector, regardless of its length
// pooling gives one speaker embedding, whatever the clip length is
n_patches = 1;
} break;
case PROJECTOR_TYPE_QWEN3TTS_GEN:
@@ -4173,7 +4168,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
set_input_f32("inp_raw", inp_raw);
} else if (!(ctx->proj_type() == PROJECTOR_TYPE_QWEN3TTS_GEN && params->gen_process == CLIP_GEN_PROCESS_GEN_WAV)) {
// audio input (code2wav has no hidden-state/raw input at all, its only input is the "inp_codes" tensor handled in the switch below)
// audio input, code2wav is not here: its only input is "inp_codes", set in the switch below
GGML_ASSERT(imgs.entries.size() == 1);
const auto & mel_inp = imgs.entries[0];
@@ -4738,15 +4733,13 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) {
GGML_ASSERT(params->codes != nullptr);
// reorder frame-major input to the group-major layout the graph wants,
// padding the rear with code 0 up to one window (tail trimmed off below)
// frame-major input to group-major, rear-padded with code 0 up to one window
const int64_t n_codes = model.gen_code_head_w->ne[2] + 1;
const int64_t n_frames_w = hparams.wav_tfm_swa;
const int64_t n_frames = (int64_t) params->codes->size() / n_codes;
GGML_ASSERT(n_frames > 0 && n_frames <= n_frames_w);
// bound each code against its codebook's vocab before it becomes
// a ggml_get_rows index into the codebook tensor
// codes are used as ggml_get_rows indices, so check them against the codebook vocab
const int64_t vocab_first = model.c2w.quant_first_cb_w->ne[1];
const int64_t vocab_rest = model.c2w.quant_rest_cb_w->ne[1];
for (int64_t f = 0; f < n_frames; f++) {
@@ -4769,8 +4762,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
}
set_input_i32("inp_codes", codes);
// upload the state carried over from the previous call, or
// zero-fill on a cold start (no previous state, or wrong size)
// upload the state from the previous call, or zero-fill on a cold start
size_t offset = 0;
for (const auto & slot : list_c2w_state_slots(hparams, model)) {
ggml_tensor * t = get_inp_tensor(("state_in_" + slot.name).c_str());
@@ -4793,8 +4785,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
std::vector<int32_t> code0 = { params->code0 };
set_input_i32("inp_code0", code0);
// one uniform(0,1) draw per codebook, consumed by do_sampling()'s
// inverse-CDF token selection (inp_rand_0 .. inp_rand_{n_acoustic-1})
// one uniform(0,1) draw per codebook, used by do_sampling()
static std::mt19937 rng{ std::random_device{}() };
std::uniform_real_distribution<float> dist(0.0f, 1.0f);
const int64_t n_acoustic = model.gen_code_head_w->ne[2];
@@ -5219,7 +5210,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
return false;
}
// the last node is the embedding tensor (not produced by the code2wav sub-graph, which has no out_embd at all)
// the last node is the embedding tensor, code2wav has no out_embd
ggml_tensor * embeddings = params->out_embd ? ggml_graph_node(gf, -1) : nullptr;
if (embeddings != nullptr) {
@@ -5270,8 +5261,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
out_audio.resize(ggml_nelements(audio));
ggml_backend_tensor_get(audio, out_audio.data(), 0, ggml_nbytes(audio));
// a short (rear-padded) batch only has real audio for its real frames;
// the tail generated from the code-0 padding is discarded
// drop the tail audio that comes from the code-0 rear padding
const int64_t n_codes = model.gen_code_head_w->ne[2] + 1;
const int64_t n_frames_w = hparams.wav_tfm_swa;
const int64_t n_frames = (int64_t) params->codes->size() / n_codes;
tools/mtmd/clip.h
+9 -15
View File
@@ -96,26 +96,20 @@ struct clip_encode_params {
const clip_image_f32_batch * imgs = nullptr;
std::vector<float> * out_embd = nullptr;
// for audio gen, imgs has exactly one entry (unused content for GEN_WAV,
// for GEN_CODE it holds the hidden state from backbone, size (n_text_embd, 1))
// for audio gen, imgs has exactly one entry: hidden state from backbone (GEN_CODE) or unused (GEN_WAV)
clip_gen_process_type gen_process = CLIP_GEN_PROCESS_GEN_UNKNOWN;
// GEN_CODE: code0 is the sampled semantic code from backbone, out_codes
// receives this frame's 16 sampled codes, out_embd receives the embd to
// be fed back to the backbone for the next frame
int32_t code0 = 0;
// GEN_CODE: out_embd receives the embd to feed back to the backbone
int32_t code0 = 0; // semantic code sampled by the backbone
int32_t top_k = 50;
float top_p = 1.0f;
std::vector<int32_t> * out_codes = nullptr;
std::vector<int32_t> * out_codes = nullptr; // this frame's 16 sampled codes
// GEN_WAV: codes holds this frame's 16 RVQ codes, out_audio receives the
// decoded PCM samples (F32). state_in is the state from the previous
// call (null or wrong size means cold start, state is zero-filled).
// state_out receives the state to pass into the next call.
const std::vector<int32_t> * codes = nullptr;
std::vector<float> * out_audio = nullptr;
const std::vector<uint8_t> * state_in = nullptr;
std::vector<uint8_t> * state_out = nullptr;
// GEN_WAV
const std::vector<int32_t> * codes = nullptr; // this frame's 16 RVQ codes
std::vector<float> * out_audio = nullptr; // decoded PCM samples, F32
const std::vector<uint8_t> * state_in = nullptr; // state from previous call, null or wrong size means cold start
std::vector<uint8_t> * state_out = nullptr; // state for the next call
};
bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params);
tools/mtmd/models/models.h
+3 -5
View File
@@ -244,8 +244,8 @@ struct clip_graph_qwen3tts_gen : clip_graph {
float top_p;
//
// code_gen: backbone hidden state + sampled code0 -> 16 RVQ codes.
// MTP-style autoregressive code predictor: one token per codebook, causal KV cache.
// code_gen: backbone hidden state + sampled code0 -> 16 RVQ codes
// MTP-style code predictor, one token per codebook
//
struct code_gen : clip_graph {
code_gen(const clip_graph & parent, int top_k, float top_p)
@@ -308,7 +308,6 @@ struct clip_graph_qwen3tts_gen : clip_graph {
ggml_tensor * snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const;
ggml_tensor * quant_decode(ggml_tensor * inp_codes) const;
// il: layer index, used to look up this layer's K/V slice of the sliding-window KV cache
ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const clip_layer & layer, int il) const;
ggml_tensor * convnext_block(ggml_tensor * x, const clip_code2wav::upsample_block & blk, const std::string & state_prefix) const;
ggml_tensor * dac_res_unit(ggml_tensor * x, const clip_code2wav::dac_res & res, int dilation, const std::string & state_name) const;
@@ -318,13 +317,12 @@ struct clip_graph_qwen3tts_gen : clip_graph {
};
};
// one named, shaped (ne0, ne1) persisted state buffer used by code2wav; see qwen3tts-gen.cpp
// one persisted state buffer used by code2wav, see qwen3tts-gen.cpp
struct c2w_state_slot {
std::string name;
int64_t ne0;
int64_t ne1;
};
// enumerates code2wav's persisted state buffers; see qwen3tts-gen.cpp
std::vector<c2w_state_slot> list_c2w_state_slots(const clip_hparams & hparams, const clip_model & model);
struct clip_graph_kimik25 : clip_graph {
tools/mtmd/models/qwen3tts-gen.cpp
+17 -30
View File
@@ -332,9 +332,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv1d_dw(ggml_tensor *
return y;
}
// causal ConvTranspose1d with persisted overlap-add tail: adjacent frames'
// output windows overlap by (kernel - stride) samples; that overlap is
// carried forward as state instead of being discarded
// causal ConvTranspose1d, the (kernel - stride) overlap tail is kept as state for the next call
// x: [T, IC], w: [K, OC, IC]. state_name empty means K == stride (no overlap). returns [T * stride, OC]
ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & state_name) const {
const int K = (int) w->ne[0];
@@ -342,8 +340,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv_transpose1d(ggml_te
const int trim = K - stride;
const int64_t emit_len = x->ne[0] * stride;
// transposed conv as GEMM + scatter-add: fold w [K, OC, IC] to [IC, K*OC], contract over IC
// then col2im scatters each column to its strided output offset. y: [emit_len + trim, OC]
// transposed conv as GEMM + col2im scatter-add, y: [emit_len + trim, OC]
ggml_tensor * w2 = ggml_reshape_2d(ctx0, w, (int64_t) K * OC, w->ne[2]);
w2 = ggml_cont(ctx0, ggml_transpose(ctx0, w2));
ggml_tensor * xt = ggml_cont(ctx0, ggml_transpose(ctx0, x));
@@ -415,10 +412,8 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::quant_decode(ggml_tensor * inp_
}
// one pre_transformer layer over a batch of N = sliding_window new frames
// attention runs over [(W-1)-frame prefix from the last batch] + [N new frames]:
// the prefix gives left-context, a banded causal mask keeps each query within its W-frame window,
// and RoPE uses a persisted, ever-increasing position counter so phases line up across batches
// next batch's persisted state is just this batch's last (W-1) frames
// attention runs over [(W-1)-frame prefix from the last batch] + [N new frames]
// RoPE positions come from a persisted counter, so phases line up across batches
ggml_tensor * clip_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor * cur, const clip_layer & layer, int il) const {
const int n_head = hparams.wav_tfm_n_head;
const int n_head_kv = hparams.wav_tfm_n_head_kv;
@@ -450,7 +445,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor *
k = ggml_rope_ext(ctx0, k, pos, nullptr, (int) d_head, GGML_ROPE_TYPE_NEOX, 0,
hparams.wav_tfm_rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
// the position counter is layer-independent -- only push it once, from layer 0
// the position counter is the same for all layers, push it once from layer 0
if (il == 0) {
state_out.push_back({"tfm_pos", ggml_scale_bias(ctx0, state_in.at("tfm_pos"), 1.0f, (float) N)});
}
@@ -470,8 +465,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor *
state_out.push_back({"tfm_v_" + std::to_string(il),
ggml_cont(ctx0, ggml_view_2d(ctx0, v_full, v_full->ne[0], prefix, v_full->nb[1], (size_t) N * v_full->nb[1]))});
// banded causal mask over [total_kv keys, N queries]: key j is visible to
// query i (offset by `prefix`) iff 0 <= (prefix+i) - j < W
// banded causal mask: key j is visible to query i iff 0 <= (prefix+i) - j < W
ggml_tensor * pos_k = ggml_reshape_2d(ctx0, ggml_arange(ctx0, 0.0f, (float) total_kv, 1.0f), total_kv, 1);
ggml_tensor * pos_q = ggml_reshape_2d(ctx0, ggml_arange(ctx0, (float) prefix, (float) (prefix + N), 1.0f), 1, N);
ggml_tensor * pos_q_grid = ggml_repeat_4d(ctx0, pos_q, total_kv, N, 1, 1);
@@ -481,8 +475,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor *
ggml_tensor * in_window = ggml_step(ctx0, ggml_scale_bias(ctx0, diff, -1.0f, (float) W - 0.5f)); // diff < W
ggml_tensor * keep = ggml_mul(ctx0, causal_keep, in_window);
// clamp the cold prefix: key j holds real state only when j >= prefix - tfm_pos
// earlier keys are zero-filled cold start; attending to them would dilute the softmax
// on a cold start, key j is real state only when j >= prefix - tfm_pos, mask out the rest
ggml_tensor * warm = ggml_step(ctx0, ggml_scale_bias(ctx0, ggml_add(ctx0, pos_k, base),
1.0f, 0.5f - (float) prefix)); // j + pos > prefix - 0.5
keep = ggml_mul(ctx0, keep, warm);
@@ -559,8 +552,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::decode(ggml_tensor * inp_codes)
x = causal_conv1d(x, c2w.pre_conv_w, c2w.pre_conv_b, 1, "pre_conv"); // [N, 1024]
cb(x, "wav_pre_conv_out", -1);
// 3. pre_transformer: back to C-first [1024, N], project down to hidden_size,
// run 8 layers with a persisted sliding-window KV cache, project back up
// 3. pre_transformer: back to C-first [1024, N], project down, run the layers, project back up
ggml_tensor * cur = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [1024, N]
cur = ggml_mul_mat(ctx0, c2w.tfm_in_proj_w, cur);
cur = ggml_add(ctx0, cur, c2w.tfm_in_proj_b); // [512 (tfm hidden), N]
@@ -575,8 +567,8 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::decode(ggml_tensor * inp_codes)
cur = ggml_add(ctx0, cur, c2w.tfm_out_proj_b); // [1024, N]
cb(cur, "wav_tfm_out", -1);
// 4. upsample: 2x (causal ConvTranspose1d, stride 2 + ConvNeXt block), back to T-first.
// kernel == stride here, so there's no transpose-conv overlap, no state needed.
// 4. upsample: 2x (causal ConvTranspose1d, stride 2 + ConvNeXt block), back to T-first
// kernel == stride here, so there is no overlap tail to persist
x = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); // [N, 1024]
for (size_t il = 0; il < c2w.upsample.size(); il++) {
const auto & up = c2w.upsample[il];
@@ -612,17 +604,15 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::decode(ggml_tensor * inp_codes)
return x;
}
// enumerates code2wav's persisted state buffers: RoPE position counter, one
// K/V slot per pre_transformer layer, one left-context/tail slot per stateful conv
// pure shape lookup, no graph needed; shared by build() and clip.cpp's state (de)serialization
// code2wav's persisted state buffers: RoPE position counter, K/V per pre_transformer layer,
// left-context/tail per stateful conv. shape lookup only, no graph needed
std::vector<c2w_state_slot> list_c2w_state_slots(const clip_hparams & hparams, const clip_model & model) {
const auto & c2w = model.c2w;
std::vector<c2w_state_slot> slots;
slots.push_back({"tfm_pos", 1, 1});
// persisted prefix is (W-1) frames: the code2wav batch itself supplies the
// other N=W frames of context for its own later frames (see tfm_layer_forward)
// prefix is (W-1) frames, the batch itself gives the other N=W frames (see tfm_layer_forward)
const int64_t d_head = c2w.tfm_layers[0].q_w->ne[1] / hparams.wav_tfm_n_head;
const int64_t kv_ch = d_head * hparams.wav_tfm_n_head_kv;
const int64_t prefix = hparams.wav_tfm_swa - 1;
@@ -658,9 +648,8 @@ std::vector<c2w_state_slot> list_c2w_state_slots(const clip_hparams & hparams, c
return slots;
}
// builds both the code_gen sub-graph (h_state -> 16 RVQ codes + next embd) and the
// code2wav sub-graph (16 RVQ codes -> raw PCM) into the same cgraph every call, then
// selects which one runs via ggml_build_forward_select(), keeping topology constant
// both sub-graphs are always built, so the topology stays constant
// ggml_build_forward_select() then picks the one that actually runs
ggml_cgraph * clip_graph_qwen3tts_gen::build() {
GGML_ASSERT(n_batch == 1); // this module only ever processes one frame at a time
@@ -672,7 +661,7 @@ ggml_cgraph * clip_graph_qwen3tts_gen::build() {
}
// ---- CLIP_GEN_PROCESS_GEN_CODE: backbone hidden state -> 16 RVQ codes + next-step embd ----
// fixed-size [n_mmproj_embd] input; not build_inp_raw(), since a GEN_WAV call's `img` has no hidden-state data
// not build_inp_raw(), a GEN_WAV call's `img` has no hidden-state data
ggml_tensor * h_state = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_mmproj_embd);
ggml_set_name(h_state, "inp_raw"); // must keep this exact name, clip_encode() sets it by name
ggml_set_input(h_state);
@@ -762,9 +751,7 @@ ggml_cgraph * clip_graph_qwen3tts_gen::build() {
ggml_set_output(slot.second);
}
// select the active branch; both are always built above (constant topology), only the
// selected side's nodes actually compute. out_embd goes last so it ends up as the graph's
// last node, since clip_encode() reads it back via ggml_graph_node(gf, -1)
// out_embd goes last, clip_encode() reads it back via ggml_graph_node(gf, -1)
ggml_tensor * outs[2];
outs[0] = out_codes; outs[1] = out_audio;
ggml_build_forward_select(gf, outs, 2, idx);
tools/mtmd/models/qwen3tts-spkenc.cpp
+9 -13
View File
@@ -3,7 +3,7 @@
static constexpr int SPK_RES2NET_SCALE = 8; // enc_res2net_scale
static constexpr int SPK_DILATIONS[3] = { 2, 3, 4 }; // enc_dilations[1..3]
// Conv1d, kernel K, padding "same" (reflect), dilation d.
// conv1d, kernel K, padding "same" (reflect), dilation d
// x: [C, T] (ne[0]=C, ne[1]=T) -> [out_c, T]
ggml_tensor * clip_graph_qwen3tts_spkenc::conv1d_same(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const {
const int K = (int) w->ne[0];
@@ -11,16 +11,14 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::conv1d_same(ggml_tensor * x, ggml_tens
const int OC = (int) w->ne[2];
const int pad = ((K - 1) * dilation) / 2;
// ggml_pad_reflect_1d pads ne[0], so bring T onto ne[0] first; im2col
// below expects the same [T, IC] layout.
// ggml_pad_reflect_1d pads ne[0], so bring T onto ne[0] first, same layout as im2col wants
ggml_tensor * x_t = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [T, IC]
if (pad > 0) {
x_t = ggml_pad_reflect_1d(ctx0, x_t, pad, pad); // [T + 2*pad, IC]
}
ggml_tensor * x4d = ggml_reshape_4d(ctx0, x_t, x_t->ne[0], IC, 1, 1);
// Dummy F32 kernel: im2col only reads its shape (K, IC), never its data,
// so this avoids a type assert when w is quantized.
// dummy F32 kernel, im2col only reads its shape, so a quantized w does not assert
ggml_tensor * dummy = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, K, IC, 1, 1);
ggml_tensor * col = ggml_im2col(ctx0, dummy, x4d, 1, 1, 0, 0, dilation, 1, false, GGML_TYPE_F32);
@@ -36,8 +34,8 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::conv1d_same(ggml_tensor * x, ggml_tens
return y;
}
// Res2Net: split channel axis into `scale` chunks, chain dilated conv1d
// branches. x: [C, T] -> [C, T]
// Res2Net: split channel axis into `scale` chunks, chain dilated conv1d branches
// x: [C, T] -> [C, T]
ggml_tensor * clip_graph_qwen3tts_spkenc::res2net(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const {
const int64_t C = x->ne[0];
const int64_t T = x->ne[1];
@@ -71,7 +69,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::res2net(ggml_tensor * x, const clip_la
return acc;
}
// Squeeze-and-excitation gate. x: [C, T] -> [C, T]
// squeeze-and-excitation gate. x: [C, T] -> [C, T]
ggml_tensor * clip_graph_qwen3tts_spkenc::se_block(ggml_tensor * x, const clip_layer & layer) const {
// temporal mean, keepdim: transpose so T is on ne[0], reduce, transpose back
ggml_tensor * x_t = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [T, C]
@@ -98,7 +96,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::se_res2net_block(ggml_tensor * x, cons
return ggml_add(ctx0, h, residual);
}
// Attentive statistics pooling. x: [C, T] -> [2*C, 1]
// attentive statistics pooling. x: [C, T] -> [2*C, 1]
ggml_tensor * clip_graph_qwen3tts_spkenc::attentive_stats_pool(ggml_tensor * x) const {
const int64_t T = x->ne[1];
@@ -132,8 +130,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::attentive_stats_pool(ggml_tensor * x)
ggml_tensor * w_t = ggml_soft_max(ctx0, a_t);
ggml_tensor * w = ggml_cont(ctx0, ggml_transpose(ctx0, w_t)); // [C, T]
// weighted mean: sum(w * x) over T (w already sums to 1 over T,
// ggml_mean gives 1/T scaling so multiply back by T to undo it)
// weighted mean: sum(w * x) over T, multiply by T to undo ggml_mean's 1/T scaling
ggml_tensor * wx = ggml_mul(ctx0, w, x);
ggml_tensor * wx_t = ggml_cont(ctx0, ggml_transpose(ctx0, wx));
ggml_tensor * w_mean = ggml_mean(ctx0, wx_t);
@@ -155,8 +152,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::attentive_stats_pool(ggml_tensor * x)
}
ggml_cgraph * clip_graph_qwen3tts_spkenc::build() {
// inp_raw: [T, n_mel, 1, 1] (nx=T frames, ny=n_mel bins), from the
// preprocessor's mel_spectrogram() output (mtmd_audio_preprocessor_qwen3tts_spk)
// inp_raw: [T, n_mel, 1, 1], from mtmd_audio_preprocessor_qwen3tts_spk
ggml_tensor * inp = build_inp_raw(1);
inp = ggml_reshape_2d(ctx0, inp, inp->ne[0], inp->ne[1]);
tools/mtmd/mtmd-audio.cpp
+2 -6
View File
@@ -794,12 +794,8 @@ bool mtmd_audio_preprocessor_mimo_audio::preprocess(const float *
//
// mtmd_audio_preprocessor_qwen3tts_spk
//
// Mirrors qwen_tts.core.models.modeling_qwen3_tts.mel_spectrogram():
// pad reflect by (n_fft - hop) / 2, STFT (n_fft, hop, win=n_fft, hann
// periodic, center=False), mel = slaney_mel_basis @ |STFT|, log(max(mel, 1e-5)).
// Unlike Whisper-style encoders the whole clip is consumed in a single
// forward pass by the ECAPA-TDNN body, so there's no 30s/3000-frame chunking
// or Whisper (max-8)/4 normalization here.
// same as mel_spectrogram() in modeling_qwen3_tts.py
// ECAPA-TDNN takes the whole clip in one pass, so no Whisper-style chunking or normalization
//
void mtmd_audio_preprocessor_qwen3tts_spk::initialize() {
tools/mtmd/mtmd-helper-common.h
+1 -1
View File
@@ -56,7 +56,7 @@ struct mtmd_helper_logger {
}
};
// inline (C++17): one shared instance across every TU that includes this header
// inline, so all TUs including this header share one instance
inline mtmd_helper_logger g_logger;
#define LOG_DBG(...) g_logger.log(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__)
tools/mtmd/mtmd-helper-gen.cpp
+11 -19
View File
@@ -19,8 +19,7 @@
// Audio generation helpers
//
// maps the --tts-lang codes (see tools/tts/README.md) to the language
// names used by the codec_language special tokens
// --tts-lang codes -> language names used by the codec_language special tokens
static const std::unordered_map<std::string, std::string> tts_lang_codes = {
{ "zh", "chinese" },
{ "en", "english" },
@@ -86,9 +85,8 @@ public:
virtual int32_t set_input(const mtmd_helper_gen_audio_inp * inp) = 0;
// decodes at most n_batch prompt tokens; returns remaining count (0 = done), <0 on error
virtual int32_t step_prompt(int32_t n_batch) = 0;
// sampled may be LLAMA_TOKEN_NULL for pipelines with no discrete backbone token
// (e.g. continuous/diffusion models); such pipelines read whatever they need
// directly off h_state_in instead
// sampled can be LLAMA_TOKEN_NULL for pipelines with no discrete backbone token,
// those read what they need from h_state_in instead
virtual int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out) = 0;
virtual int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) = 0;
@@ -101,8 +99,8 @@ protected:
mtmd_gen_audio_info info;
};
// Qwen3-TTS: dual-track discrete AR (backbone codec_0 + MTP code-predictor for the
// remaining 15 codebooks) into a windowed causal conv/transformer decode (code2wav)
// Qwen3-TTS: backbone samples codec_0, code_predictor gives the other 15 codebooks,
// then code2wav decodes them to PCM
class qwen3tts_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
public:
using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
@@ -187,9 +185,7 @@ public:
n_prompt = (int) prompt.size();
// the talker rides the qwen3vl interleaved mrope: positions carry
// n_pos_per_embd sections laid out [section * n_tokens + i], all
// equal for a pure text/codec stream
// the talker uses the qwen3vl interleaved mrope, all sections are equal for a text/codec stream
mrope = llama_model_rope_type(model) == LLAMA_ROPE_TYPE_MROPE ||
llama_model_rope_type(model) == LLAMA_ROPE_TYPE_IMROPE;
const int n_pos_per_embd = mrope ? 4 : 1;
@@ -209,9 +205,8 @@ public:
top_p = inp->top_p > 0 ? inp->top_p : 1.0f;
out_type = inp->out_type;
// the text stream keeps flowing during generation: the input after
// frame k adds trailing text row k on top of the codes embedding,
// then tts_eos, then tts_pad once the utterance is spent
// the text stream keeps flowing during generation: after frame k, the input adds
// trailing text row k on top of the codes embedding, then tts_eos, then tts_pad
for (int i = 3; i < n_ids - 5; i++) overlay.push_back(row(ids[(size_t) i]));
overlay.push_back(row(tts_eos));
overlay.push_back(row(tts_pad));
@@ -356,8 +351,7 @@ private:
return true;
}
// encodes a reference wav (already loaded as a bitmap) through the mmproj's
// speaker encoder, returning the single x-vector embedding row it produces
// runs the reference wav through the speaker encoder, returns one x-vector embedding row
bool encode_speaker(mtmd_bitmap * bitmap, std::vector<float> & out) {
if (!mtmd_support_audio(mctx)) {
LOG_ERR("mtmd_helper_gen_audio: mmproj has no speaker/audio encoder\n");
@@ -390,8 +384,7 @@ private:
return ok;
}
// runs one GEN_WAV process() call on whatever is currently buffered, carrying
// the persisted state (KV cache + conv left-context) across batches
// one GEN_WAV process() call over the buffered codes, state is carried across batches
bool flush_gen_wav() {
if (codes_buf.empty()) {
return true;
@@ -427,8 +420,7 @@ private:
llama_token tts_eos = LLAMA_TOKEN_NULL;
std::vector<float> tok_embd; // whole token embedding matrix, n_vocab * n_embd
// matches hparams.wav_tfm_sliding_window hardcoded in clip.cpp; code2wav
// batches exactly this many frames per call
// must match hparams.wav_tfm_swa hardcoded in clip.cpp
size_t window_frames = 72;
// per-generation state, cleared by reset()
tools/mtmd/mtmd-helper.h
+1 -3
View File
@@ -202,7 +202,6 @@ MTMD_API int32_t mtmd_helper_gen_audio_set_input(
const struct mtmd_helper_gen_audio_inp * inp);
// processes at most n_batch prompt tokens per call
// returns 0 (no more prompt left to process)
// returns: >0 = number of prompt tokens remaining, 0 = done, <0 = error
MTMD_API int32_t mtmd_helper_gen_audio_step_prompt(
mtmd_helper_gen_audio * ctx,
@@ -217,8 +216,7 @@ MTMD_API int32_t mtmd_helper_gen_audio_step_gen(
const float ** h_state_out);
// out_data valid until next get_output() or reset() call
// out_n_samples (optional, can be NULL) receives the number of generated PCM samples,
// which combined with out_sample_rate gives the output audio duration
// out_n_samples (optional, can be NULL) receives the number of generated PCM samples
MTMD_API int32_t mtmd_helper_gen_audio_get_output(
mtmd_helper_gen_audio * ctx,
int32_t * out_sample_rate,