From 201e50cc2076a20adc460c41598593c7cd7b0813 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Tue, 4 Aug 2026 15:18:19 +0200 Subject: [PATCH] clean up comments --- conversion/qwen3tts.py | 33 ++++++------------- gguf-py/gguf/constants.py | 5 ++- tools/mtmd/clip-impl.h | 2 +- tools/mtmd/clip-model.h | 14 +++----- tools/mtmd/clip.cpp | 30 ++++++----------- tools/mtmd/clip.h | 24 +++++--------- tools/mtmd/models/models.h | 8 ++--- tools/mtmd/models/qwen3tts-gen.cpp | 47 ++++++++++----------------- tools/mtmd/models/qwen3tts-spkenc.cpp | 22 +++++-------- tools/mtmd/mtmd-audio.cpp | 8 ++--- tools/mtmd/mtmd-helper-common.h | 2 +- tools/mtmd/mtmd-helper-gen.cpp | 30 +++++++---------- tools/mtmd/mtmd-helper.h | 4 +-- 13 files changed, 81 insertions(+), 148 deletions(-) diff --git a/conversion/qwen3tts.py b/conversion/qwen3tts.py index 22462f9df1..d21a505951 100644 --- a/conversion/qwen3tts.py +++ b/conversion/qwen3tts.py @@ -216,8 +216,7 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel): if hparams is None: hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False) hparams["text_config"] = {"hidden_size": hparams["talker_config"]["hidden_size"]} - # ECAPA-TDNN has a fixed 4-stage backbone, not a configurable transformer depth; - # MmprojModel.__init__ still needs one of the n_block_keys to build its tensor map + # ECAPA-TDNN has a fixed 4-stage backbone, but MmprojModel.__init__ needs a n_block_keys hparams["speaker_encoder_config"]["n_layers"] = 4 super().__init__(dir_model, *args, hparams=hparams, **kwargs) self._wav_config_cache = None @@ -234,7 +233,7 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel): self.gguf_writer.add_audio_projection_dim(self.n_embd_text) # mel_spectrogram() front-end: sr=24000, n_fft=1024, hop=256, n_mels=128, fmin=0, fmax=12000 (=sr/2, the clip.cpp default) self.gguf_writer.add_audio_num_mel_bins(128) - # the 3 SE-Res2Net stages (blocks 1-3); the stem conv, mfa, asp and fc are singletons, not part of this count + # 3 SE-Res2Net stages; the stem conv, mfa, asp and fc are not counted here self.gguf_writer.add_audio_block_count(3) # ECAPA-TDNN has no attention/FFN, these are dummy to allow clip.cpp to load it self.gguf_writer.add_audio_embedding_length(1536) @@ -256,8 +255,7 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel): # note: code2wav hparams are hardcoded on the mtmd/clip.cpp side for now, not written here def _wav_decoder_config(self) -> dict[str, Any] | None: - # code2wav (RVQ codes -> raw PCM) lives in its own checkpoint dir, sibling to - # the main safetensors, with its own config.json + # code2wav has its own config.json, inside the speech_tokenizer dir if self._wav_config_cache is None: path = self.dir_model / "speech_tokenizer" / "config.json" with open(path, "r", encoding="utf-8") as f: @@ -266,18 +264,14 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel): return self._wav_config_cache def tensor_force_quant(self, name, new_name, bid, n_dims): - # regular (non-transpose) conv1d/conv1d_dw weights must be F16, never BF16: - # ggml_conv_1d(_dw) pairs the kernel as mul_mat's src1 against an F32 im2col - # src0, and the CPU backend only accepts src1 in F32 -- BF16 kernels can't be - # scheduled. + # conv1d/conv1d_dw kernels must be F16, ggml_conv_1d(_dw) has no BF16 path if new_name.endswith(".weight") and ( new_name in ("a.gen.wav.pre_conv.weight", "a.gen.wav.dac.entry.weight", "a.gen.wav.dac.post_conv.weight") or (".up.blk." in new_name and new_name.endswith(".dwconv.weight")) or (".dac.blk." in new_name and (new_name.endswith(".conv1.weight") or new_name.endswith(".conv2.weight"))) ): return gguf.GGMLQuantizationType.F16 - # causal ConvTranspose1d weights: ggml_compute_forward_conv_transpose_1d - # only implements F16/F32 kernels, never BF16 + # ConvTranspose1d kernels: only F16/F32 are implemented, no BF16 if new_name.endswith(".conv.weight") and (".up.blk." in new_name or ".dac.blk." in new_name): return gguf.GGMLQuantizationType.F32 return super().tensor_force_quant(name, new_name, bid, n_dims) @@ -296,16 +290,12 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel): return super().filter_tensors((name, gen)) def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: - # code2wav tensors come from generate_extra_tensors() already renamed to their - # final gguf name (bypassing tensor_mapping.py, same as the code_predictor - # tensors below); prepare_tensors() re-runs modify_tensors() on them too, so - # pass them through untouched instead of falling into map_tensor_name() + # code2wav tensors are already named by generate_extra_tensors(), pass them through if name.startswith("a.gen.wav."): yield (name, data_torch) return - # codebook-0 embedding: fed back into the talker backbone once a codec token is - # generated, the counterpart of code_predictor's codec_embedding.{0..14} for codebooks 1-15 + # codebook-0 embedding, fed back to the talker backbone (codebooks 1-15 live in code_predictor) if name == "talker.model.codec_embedding.weight": yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_GEN_CODE_OUT_EMBD), data_torch) return @@ -360,8 +350,7 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel): yield from self._generate_code2wav_tensors() def _generate_code2wav_tensors(self) -> Iterable[tuple[str, Tensor]]: - # code2wav lives in its own checkpoint dir (speech_tokenizer/model.safetensors), - # not the main safetensors this ModelBase was constructed from + # code2wav weights live in speech_tokenizer/model.safetensors, not the main safetensors from safetensors.torch import load_file wav_config = self._wav_decoder_config() @@ -371,13 +360,11 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel): return state_dict[name] def snake_fold(alpha: Tensor, beta: Tensor) -> tuple[Tensor, Tensor]: - # SnakeBeta activation folds its exp()/reciprocal at conversion time, so the - # runtime graph is only mul -> sin -> sqr -> mul -> add + # fold SnakeBeta's exp()/reciprocal here, so the graph is only mul/sin/sqr/mul/add return torch.exp(alpha), 1.0 / (torch.exp(beta) + 1e-9) def rvq_codebook(prefix: str, n_layers: int) -> Tensor: - # the checkpoint stores EMA training accumulators, not a ready embedding - # table: codebook[i] = embedding_sum[i] / cluster_usage[i] + # checkpoint has EMA accumulators, so codebook[i] = embedding_sum[i] / cluster_usage[i] books = [] for i in range(n_layers): embedding_sum = get(f"{prefix}.vq.layers.{i}._codebook.embedding_sum") diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 20f50733df..d939e66cee 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -976,7 +976,7 @@ class MODEL_TENSOR(IntEnum): A_ENC_SE_CONV2 = auto() # qwen3tts A_ENC_ASP_ATTN = auto() # qwen3tts A_ENC_ASP_TDNN = auto() # qwen3tts - # qwen3tts code_predictor: autoregressively predicts the remaining RVQ codebooks + # qwen3tts code_predictor: predicts the remaining RVQ codebooks A_GEN_CODE_PROJ_IN = auto() # small_to_mtp_projection A_GEN_CODE_EMBD = auto() # per-codebook embedding table, merged 3D [n_codebooks, vocab, dim] A_GEN_CODE_HEAD = auto() # per-codebook output head, merged 3D [n_codebooks, vocab, dim] @@ -993,8 +993,7 @@ class MODEL_TENSOR(IntEnum): A_GEN_CODE_FFN_UP = auto() A_GEN_CODE_FFN_DOWN = auto() A_GEN_CODE_OUTPUT_NORM = auto() - # qwen3tts code2wav: RVQ codes -> raw PCM (quantizer decode + pre_conv + - # pre_transformer + ConvNeXt upsample + DAC decoder) + # qwen3tts code2wav: RVQ codes -> raw PCM A_GEN_WAV_QUANT_FIRST_IN = auto() # semantic RVQ, in_proj (1x1 conv, loaded as 2D) A_GEN_WAV_QUANT_FIRST_OUT = auto() # semantic RVQ, out_proj A_GEN_WAV_QUANT_FIRST_CB = auto() # semantic RVQ codebook (1 layer), folded from embedding_sum/cluster_usage diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 7222660c77..e1567ee5ba 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -219,7 +219,7 @@ #define TN_A_GEN_CODE_NORM "a.gen.code.output_norm.%s" // qwen3tts code2wav (RVQ codes -> raw PCM) -// pre_transformer per-layer tensors are loaded through the generic TN_ATTN_*/TN_FFN_*/TN_LN_*/TN_LS_* macros with prefix "a.gen.wav.tfm" +// pre_transformer layers use the generic TN_ATTN_*/TN_FFN_*/TN_LN_*/TN_LS_* macros, prefix "a.gen.wav.tfm" #define TN_A_GEN_WAV_QUANT_FIRST_IN "a.gen.wav.quant.first.in_proj.%s" #define TN_A_GEN_WAV_QUANT_FIRST_OUT "a.gen.wav.quant.first.out_proj.%s" #define TN_A_GEN_WAV_QUANT_FIRST_CB "a.gen.wav.quant.first.codebook.%s" diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index 30014f1f50..101f49cd18 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -299,8 +299,7 @@ struct clip_layer { ggml_tensor * cross_attn_norm_w = nullptr; ggml_tensor * cross_attn_norm_b = nullptr; - // qwen3tts speaker encoder: SE-Res2Net block (conv_pw1_w/b and conv_pw2_w/b - // above are reused for this block's tdnn1/tdnn2 bottleneck convs) + // qwen3tts speaker encoder: SE-Res2Net block, tdnn1/tdnn2 reuse conv_pw1_w/b and conv_pw2_w/b above ggml_tensor * se_conv1_w = nullptr; ggml_tensor * se_conv1_b = nullptr; ggml_tensor * se_conv2_w = nullptr; @@ -389,8 +388,7 @@ struct qf_block { // qwen3tts code2wav: RVQ codes -> raw PCM struct clip_code2wav { - // one ConvNeXt block + its preceding causal ConvTranspose1d - // (the "upsample" stage between pre_transformer and the DAC decoder) + // "upsample" stage: one ConvNeXt block plus the causal ConvTranspose1d before it struct upsample_block { ggml_tensor * conv_w = nullptr; // causal ConvTranspose1d, 2x ggml_tensor * conv_b = nullptr; @@ -405,8 +403,7 @@ struct clip_code2wav { ggml_tensor * gamma = nullptr; // layer scale }; - // one DAC residual unit - // (SnakeBeta -> dilated causal conv -> SnakeBeta -> pointwise causal conv) + // one DAC residual unit: SnakeBeta -> dilated causal conv -> SnakeBeta -> pointwise causal conv struct dac_res { ggml_tensor * act1_alpha = nullptr; ggml_tensor * act1_beta = nullptr; @@ -668,9 +665,8 @@ struct clip_model { ggml_tensor * conv2d_3_w = nullptr; ggml_tensor * conv2d_3_b = nullptr; - // qwen3tts speaker encoder (ECAPA-TDNN): the stem conv (block 0) reuses - // conv1d_1_w/b, the multi-layer feature aggregation conv reuses conv_out_w/b, - // and the final speaker embedding projection reuses mm_fc_w/b + // qwen3tts speaker encoder (ECAPA-TDNN) + // reused tensors: stem conv is conv1d_1_w/b, feature aggregation is conv_out_w/b, output proj is mm_fc_w/b ggml_tensor * spk_asp_attn_w = nullptr; ggml_tensor * spk_asp_attn_b = nullptr; ggml_tensor * spk_asp_tdnn_w = nullptr; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 0e5dd95743..d6670030ff 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1701,8 +1701,7 @@ struct clip_model_loader { } break; case PROJECTOR_TYPE_QWEN3TTS_SPKENC: { - // ECAPA-TDNN speaker/voice encoder; mel_spectrogram() front-end - // matches the Slaney mel default (fmin=0, fmax=sample_rate/2) + // ECAPA-TDNN speaker encoder, mel front-end uses the Slaney default (fmin=0, fmax=sr/2) hparams.audio_sample_rate = 24000; hparams.audio_n_fft = 1024; hparams.audio_window_len = 1024; @@ -1710,7 +1709,6 @@ struct clip_model_loader { } break; case PROJECTOR_TYPE_QWEN3TTS_GEN: { - // discrete-token autoregressive predictor, no mel-frontend needed // TODO: hardcoded for now, read from code_predictor_config instead hparams.rope_theta = 1000000.0f; @@ -2754,9 +2752,7 @@ struct clip_model_loader { c2w.tfm_out_proj_b = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "bias")); c2w.tfm_output_norm_w = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_NORM, "weight")); - // pre_transformer layers: own prefix/layer-count, so loaded manually - // rather than through the generic model.layers loop (already claimed - // by code_predictor's 5 layers) + // loaded manually, the generic model.layers loop is taken by code_predictor c2w.tfm_layers.resize(hparams.wav_tfm_n_layer); for (int il = 0; il < hparams.wav_tfm_n_layer; il++) { auto & layer = c2w.tfm_layers[il]; @@ -4020,8 +4016,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { } break; case PROJECTOR_TYPE_QWEN3TTS_SPKENC: { - // attentive statistics pooling collapses the whole clip into - // a single speaker embedding vector, regardless of its length + // pooling gives one speaker embedding, whatever the clip length is n_patches = 1; } break; case PROJECTOR_TYPE_QWEN3TTS_GEN: @@ -4173,7 +4168,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { set_input_f32("inp_raw", inp_raw); } else if (!(ctx->proj_type() == PROJECTOR_TYPE_QWEN3TTS_GEN && params->gen_process == CLIP_GEN_PROCESS_GEN_WAV)) { - // audio input (code2wav has no hidden-state/raw input at all, its only input is the "inp_codes" tensor handled in the switch below) + // audio input, code2wav is not here: its only input is "inp_codes", set in the switch below GGML_ASSERT(imgs.entries.size() == 1); const auto & mel_inp = imgs.entries[0]; @@ -4738,15 +4733,13 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) { GGML_ASSERT(params->codes != nullptr); - // reorder frame-major input to the group-major layout the graph wants, - // padding the rear with code 0 up to one window (tail trimmed off below) + // frame-major input to group-major, rear-padded with code 0 up to one window const int64_t n_codes = model.gen_code_head_w->ne[2] + 1; const int64_t n_frames_w = hparams.wav_tfm_swa; const int64_t n_frames = (int64_t) params->codes->size() / n_codes; GGML_ASSERT(n_frames > 0 && n_frames <= n_frames_w); - // bound each code against its codebook's vocab before it becomes - // a ggml_get_rows index into the codebook tensor + // codes are used as ggml_get_rows indices, so check them against the codebook vocab const int64_t vocab_first = model.c2w.quant_first_cb_w->ne[1]; const int64_t vocab_rest = model.c2w.quant_rest_cb_w->ne[1]; for (int64_t f = 0; f < n_frames; f++) { @@ -4769,8 +4762,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { } set_input_i32("inp_codes", codes); - // upload the state carried over from the previous call, or - // zero-fill on a cold start (no previous state, or wrong size) + // upload the state from the previous call, or zero-fill on a cold start size_t offset = 0; for (const auto & slot : list_c2w_state_slots(hparams, model)) { ggml_tensor * t = get_inp_tensor(("state_in_" + slot.name).c_str()); @@ -4793,8 +4785,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { std::vector code0 = { params->code0 }; set_input_i32("inp_code0", code0); - // one uniform(0,1) draw per codebook, consumed by do_sampling()'s - // inverse-CDF token selection (inp_rand_0 .. inp_rand_{n_acoustic-1}) + // one uniform(0,1) draw per codebook, used by do_sampling() static std::mt19937 rng{ std::random_device{}() }; std::uniform_real_distribution dist(0.0f, 1.0f); const int64_t n_acoustic = model.gen_code_head_w->ne[2]; @@ -5219,7 +5210,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { return false; } - // the last node is the embedding tensor (not produced by the code2wav sub-graph, which has no out_embd at all) + // the last node is the embedding tensor, code2wav has no out_embd ggml_tensor * embeddings = params->out_embd ? ggml_graph_node(gf, -1) : nullptr; if (embeddings != nullptr) { @@ -5270,8 +5261,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { out_audio.resize(ggml_nelements(audio)); ggml_backend_tensor_get(audio, out_audio.data(), 0, ggml_nbytes(audio)); - // a short (rear-padded) batch only has real audio for its real frames; - // the tail generated from the code-0 padding is discarded + // drop the tail audio that comes from the code-0 rear padding const int64_t n_codes = model.gen_code_head_w->ne[2] + 1; const int64_t n_frames_w = hparams.wav_tfm_swa; const int64_t n_frames = (int64_t) params->codes->size() / n_codes; diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h index 2974b827db..7f706d976e 100644 --- a/tools/mtmd/clip.h +++ b/tools/mtmd/clip.h @@ -96,26 +96,20 @@ struct clip_encode_params { const clip_image_f32_batch * imgs = nullptr; std::vector * out_embd = nullptr; - // for audio gen, imgs has exactly one entry (unused content for GEN_WAV, - // for GEN_CODE it holds the hidden state from backbone, size (n_text_embd, 1)) + // for audio gen, imgs has exactly one entry: hidden state from backbone (GEN_CODE) or unused (GEN_WAV) clip_gen_process_type gen_process = CLIP_GEN_PROCESS_GEN_UNKNOWN; - // GEN_CODE: code0 is the sampled semantic code from backbone, out_codes - // receives this frame's 16 sampled codes, out_embd receives the embd to - // be fed back to the backbone for the next frame - int32_t code0 = 0; + // GEN_CODE: out_embd receives the embd to feed back to the backbone + int32_t code0 = 0; // semantic code sampled by the backbone int32_t top_k = 50; float top_p = 1.0f; - std::vector * out_codes = nullptr; + std::vector * out_codes = nullptr; // this frame's 16 sampled codes - // GEN_WAV: codes holds this frame's 16 RVQ codes, out_audio receives the - // decoded PCM samples (F32). state_in is the state from the previous - // call (null or wrong size means cold start, state is zero-filled). - // state_out receives the state to pass into the next call. - const std::vector * codes = nullptr; - std::vector * out_audio = nullptr; - const std::vector * state_in = nullptr; - std::vector * state_out = nullptr; + // GEN_WAV + const std::vector * codes = nullptr; // this frame's 16 RVQ codes + std::vector * out_audio = nullptr; // decoded PCM samples, F32 + const std::vector * state_in = nullptr; // state from previous call, null or wrong size means cold start + std::vector * state_out = nullptr; // state for the next call }; bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params); diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index 546c14c990..eb924972bf 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -244,8 +244,8 @@ struct clip_graph_qwen3tts_gen : clip_graph { float top_p; // - // code_gen: backbone hidden state + sampled code0 -> 16 RVQ codes. - // MTP-style autoregressive code predictor: one token per codebook, causal KV cache. + // code_gen: backbone hidden state + sampled code0 -> 16 RVQ codes + // MTP-style code predictor, one token per codebook // struct code_gen : clip_graph { code_gen(const clip_graph & parent, int top_k, float top_p) @@ -308,7 +308,6 @@ struct clip_graph_qwen3tts_gen : clip_graph { ggml_tensor * snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const; ggml_tensor * quant_decode(ggml_tensor * inp_codes) const; - // il: layer index, used to look up this layer's K/V slice of the sliding-window KV cache ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const clip_layer & layer, int il) const; ggml_tensor * convnext_block(ggml_tensor * x, const clip_code2wav::upsample_block & blk, const std::string & state_prefix) const; ggml_tensor * dac_res_unit(ggml_tensor * x, const clip_code2wav::dac_res & res, int dilation, const std::string & state_name) const; @@ -318,13 +317,12 @@ struct clip_graph_qwen3tts_gen : clip_graph { }; }; -// one named, shaped (ne0, ne1) persisted state buffer used by code2wav; see qwen3tts-gen.cpp +// one persisted state buffer used by code2wav, see qwen3tts-gen.cpp struct c2w_state_slot { std::string name; int64_t ne0; int64_t ne1; }; -// enumerates code2wav's persisted state buffers; see qwen3tts-gen.cpp std::vector list_c2w_state_slots(const clip_hparams & hparams, const clip_model & model); struct clip_graph_kimik25 : clip_graph { diff --git a/tools/mtmd/models/qwen3tts-gen.cpp b/tools/mtmd/models/qwen3tts-gen.cpp index ddba92fd37..b6c95efa94 100644 --- a/tools/mtmd/models/qwen3tts-gen.cpp +++ b/tools/mtmd/models/qwen3tts-gen.cpp @@ -332,9 +332,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv1d_dw(ggml_tensor * return y; } -// causal ConvTranspose1d with persisted overlap-add tail: adjacent frames' -// output windows overlap by (kernel - stride) samples; that overlap is -// carried forward as state instead of being discarded +// causal ConvTranspose1d, the (kernel - stride) overlap tail is kept as state for the next call // x: [T, IC], w: [K, OC, IC]. state_name empty means K == stride (no overlap). returns [T * stride, OC] ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & state_name) const { const int K = (int) w->ne[0]; @@ -342,8 +340,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv_transpose1d(ggml_te const int trim = K - stride; const int64_t emit_len = x->ne[0] * stride; - // transposed conv as GEMM + scatter-add: fold w [K, OC, IC] to [IC, K*OC], contract over IC - // then col2im scatters each column to its strided output offset. y: [emit_len + trim, OC] + // transposed conv as GEMM + col2im scatter-add, y: [emit_len + trim, OC] ggml_tensor * w2 = ggml_reshape_2d(ctx0, w, (int64_t) K * OC, w->ne[2]); w2 = ggml_cont(ctx0, ggml_transpose(ctx0, w2)); ggml_tensor * xt = ggml_cont(ctx0, ggml_transpose(ctx0, x)); @@ -415,10 +412,8 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::quant_decode(ggml_tensor * inp_ } // one pre_transformer layer over a batch of N = sliding_window new frames -// attention runs over [(W-1)-frame prefix from the last batch] + [N new frames]: -// the prefix gives left-context, a banded causal mask keeps each query within its W-frame window, -// and RoPE uses a persisted, ever-increasing position counter so phases line up across batches -// next batch's persisted state is just this batch's last (W-1) frames +// attention runs over [(W-1)-frame prefix from the last batch] + [N new frames] +// RoPE positions come from a persisted counter, so phases line up across batches ggml_tensor * clip_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor * cur, const clip_layer & layer, int il) const { const int n_head = hparams.wav_tfm_n_head; const int n_head_kv = hparams.wav_tfm_n_head_kv; @@ -450,7 +445,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor * k = ggml_rope_ext(ctx0, k, pos, nullptr, (int) d_head, GGML_ROPE_TYPE_NEOX, 0, hparams.wav_tfm_rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); - // the position counter is layer-independent -- only push it once, from layer 0 + // the position counter is the same for all layers, push it once from layer 0 if (il == 0) { state_out.push_back({"tfm_pos", ggml_scale_bias(ctx0, state_in.at("tfm_pos"), 1.0f, (float) N)}); } @@ -470,8 +465,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor * state_out.push_back({"tfm_v_" + std::to_string(il), ggml_cont(ctx0, ggml_view_2d(ctx0, v_full, v_full->ne[0], prefix, v_full->nb[1], (size_t) N * v_full->nb[1]))}); - // banded causal mask over [total_kv keys, N queries]: key j is visible to - // query i (offset by `prefix`) iff 0 <= (prefix+i) - j < W + // banded causal mask: key j is visible to query i iff 0 <= (prefix+i) - j < W ggml_tensor * pos_k = ggml_reshape_2d(ctx0, ggml_arange(ctx0, 0.0f, (float) total_kv, 1.0f), total_kv, 1); ggml_tensor * pos_q = ggml_reshape_2d(ctx0, ggml_arange(ctx0, (float) prefix, (float) (prefix + N), 1.0f), 1, N); ggml_tensor * pos_q_grid = ggml_repeat_4d(ctx0, pos_q, total_kv, N, 1, 1); @@ -481,8 +475,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor * ggml_tensor * in_window = ggml_step(ctx0, ggml_scale_bias(ctx0, diff, -1.0f, (float) W - 0.5f)); // diff < W ggml_tensor * keep = ggml_mul(ctx0, causal_keep, in_window); - // clamp the cold prefix: key j holds real state only when j >= prefix - tfm_pos - // earlier keys are zero-filled cold start; attending to them would dilute the softmax + // on a cold start, key j is real state only when j >= prefix - tfm_pos, mask out the rest ggml_tensor * warm = ggml_step(ctx0, ggml_scale_bias(ctx0, ggml_add(ctx0, pos_k, base), 1.0f, 0.5f - (float) prefix)); // j + pos > prefix - 0.5 keep = ggml_mul(ctx0, keep, warm); @@ -559,8 +552,7 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::decode(ggml_tensor * inp_codes) x = causal_conv1d(x, c2w.pre_conv_w, c2w.pre_conv_b, 1, "pre_conv"); // [N, 1024] cb(x, "wav_pre_conv_out", -1); - // 3. pre_transformer: back to C-first [1024, N], project down to hidden_size, - // run 8 layers with a persisted sliding-window KV cache, project back up + // 3. pre_transformer: back to C-first [1024, N], project down, run the layers, project back up ggml_tensor * cur = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [1024, N] cur = ggml_mul_mat(ctx0, c2w.tfm_in_proj_w, cur); cur = ggml_add(ctx0, cur, c2w.tfm_in_proj_b); // [512 (tfm hidden), N] @@ -575,8 +567,8 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::decode(ggml_tensor * inp_codes) cur = ggml_add(ctx0, cur, c2w.tfm_out_proj_b); // [1024, N] cb(cur, "wav_tfm_out", -1); - // 4. upsample: 2x (causal ConvTranspose1d, stride 2 + ConvNeXt block), back to T-first. - // kernel == stride here, so there's no transpose-conv overlap, no state needed. + // 4. upsample: 2x (causal ConvTranspose1d, stride 2 + ConvNeXt block), back to T-first + // kernel == stride here, so there is no overlap tail to persist x = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); // [N, 1024] for (size_t il = 0; il < c2w.upsample.size(); il++) { const auto & up = c2w.upsample[il]; @@ -612,17 +604,15 @@ ggml_tensor * clip_graph_qwen3tts_gen::code2wav::decode(ggml_tensor * inp_codes) return x; } -// enumerates code2wav's persisted state buffers: RoPE position counter, one -// K/V slot per pre_transformer layer, one left-context/tail slot per stateful conv -// pure shape lookup, no graph needed; shared by build() and clip.cpp's state (de)serialization +// code2wav's persisted state buffers: RoPE position counter, K/V per pre_transformer layer, +// left-context/tail per stateful conv. shape lookup only, no graph needed std::vector list_c2w_state_slots(const clip_hparams & hparams, const clip_model & model) { const auto & c2w = model.c2w; std::vector slots; slots.push_back({"tfm_pos", 1, 1}); - // persisted prefix is (W-1) frames: the code2wav batch itself supplies the - // other N=W frames of context for its own later frames (see tfm_layer_forward) + // prefix is (W-1) frames, the batch itself gives the other N=W frames (see tfm_layer_forward) const int64_t d_head = c2w.tfm_layers[0].q_w->ne[1] / hparams.wav_tfm_n_head; const int64_t kv_ch = d_head * hparams.wav_tfm_n_head_kv; const int64_t prefix = hparams.wav_tfm_swa - 1; @@ -658,9 +648,8 @@ std::vector list_c2w_state_slots(const clip_hparams & hparams, c return slots; } -// builds both the code_gen sub-graph (h_state -> 16 RVQ codes + next embd) and the -// code2wav sub-graph (16 RVQ codes -> raw PCM) into the same cgraph every call, then -// selects which one runs via ggml_build_forward_select(), keeping topology constant +// both sub-graphs are always built, so the topology stays constant +// ggml_build_forward_select() then picks the one that actually runs ggml_cgraph * clip_graph_qwen3tts_gen::build() { GGML_ASSERT(n_batch == 1); // this module only ever processes one frame at a time @@ -672,7 +661,7 @@ ggml_cgraph * clip_graph_qwen3tts_gen::build() { } // ---- CLIP_GEN_PROCESS_GEN_CODE: backbone hidden state -> 16 RVQ codes + next-step embd ---- - // fixed-size [n_mmproj_embd] input; not build_inp_raw(), since a GEN_WAV call's `img` has no hidden-state data + // not build_inp_raw(), a GEN_WAV call's `img` has no hidden-state data ggml_tensor * h_state = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_mmproj_embd); ggml_set_name(h_state, "inp_raw"); // must keep this exact name, clip_encode() sets it by name ggml_set_input(h_state); @@ -762,9 +751,7 @@ ggml_cgraph * clip_graph_qwen3tts_gen::build() { ggml_set_output(slot.second); } - // select the active branch; both are always built above (constant topology), only the - // selected side's nodes actually compute. out_embd goes last so it ends up as the graph's - // last node, since clip_encode() reads it back via ggml_graph_node(gf, -1) + // out_embd goes last, clip_encode() reads it back via ggml_graph_node(gf, -1) ggml_tensor * outs[2]; outs[0] = out_codes; outs[1] = out_audio; ggml_build_forward_select(gf, outs, 2, idx); diff --git a/tools/mtmd/models/qwen3tts-spkenc.cpp b/tools/mtmd/models/qwen3tts-spkenc.cpp index 5c3f633124..d4659fd63d 100644 --- a/tools/mtmd/models/qwen3tts-spkenc.cpp +++ b/tools/mtmd/models/qwen3tts-spkenc.cpp @@ -3,7 +3,7 @@ static constexpr int SPK_RES2NET_SCALE = 8; // enc_res2net_scale static constexpr int SPK_DILATIONS[3] = { 2, 3, 4 }; // enc_dilations[1..3] -// Conv1d, kernel K, padding "same" (reflect), dilation d. +// conv1d, kernel K, padding "same" (reflect), dilation d // x: [C, T] (ne[0]=C, ne[1]=T) -> [out_c, T] ggml_tensor * clip_graph_qwen3tts_spkenc::conv1d_same(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const { const int K = (int) w->ne[0]; @@ -11,16 +11,14 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::conv1d_same(ggml_tensor * x, ggml_tens const int OC = (int) w->ne[2]; const int pad = ((K - 1) * dilation) / 2; - // ggml_pad_reflect_1d pads ne[0], so bring T onto ne[0] first; im2col - // below expects the same [T, IC] layout. + // ggml_pad_reflect_1d pads ne[0], so bring T onto ne[0] first, same layout as im2col wants ggml_tensor * x_t = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [T, IC] if (pad > 0) { x_t = ggml_pad_reflect_1d(ctx0, x_t, pad, pad); // [T + 2*pad, IC] } ggml_tensor * x4d = ggml_reshape_4d(ctx0, x_t, x_t->ne[0], IC, 1, 1); - // Dummy F32 kernel: im2col only reads its shape (K, IC), never its data, - // so this avoids a type assert when w is quantized. + // dummy F32 kernel, im2col only reads its shape, so a quantized w does not assert ggml_tensor * dummy = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, K, IC, 1, 1); ggml_tensor * col = ggml_im2col(ctx0, dummy, x4d, 1, 1, 0, 0, dilation, 1, false, GGML_TYPE_F32); @@ -36,8 +34,8 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::conv1d_same(ggml_tensor * x, ggml_tens return y; } -// Res2Net: split channel axis into `scale` chunks, chain dilated conv1d -// branches. x: [C, T] -> [C, T] +// Res2Net: split channel axis into `scale` chunks, chain dilated conv1d branches +// x: [C, T] -> [C, T] ggml_tensor * clip_graph_qwen3tts_spkenc::res2net(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const { const int64_t C = x->ne[0]; const int64_t T = x->ne[1]; @@ -71,7 +69,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::res2net(ggml_tensor * x, const clip_la return acc; } -// Squeeze-and-excitation gate. x: [C, T] -> [C, T] +// squeeze-and-excitation gate. x: [C, T] -> [C, T] ggml_tensor * clip_graph_qwen3tts_spkenc::se_block(ggml_tensor * x, const clip_layer & layer) const { // temporal mean, keepdim: transpose so T is on ne[0], reduce, transpose back ggml_tensor * x_t = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [T, C] @@ -98,7 +96,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::se_res2net_block(ggml_tensor * x, cons return ggml_add(ctx0, h, residual); } -// Attentive statistics pooling. x: [C, T] -> [2*C, 1] +// attentive statistics pooling. x: [C, T] -> [2*C, 1] ggml_tensor * clip_graph_qwen3tts_spkenc::attentive_stats_pool(ggml_tensor * x) const { const int64_t T = x->ne[1]; @@ -132,8 +130,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::attentive_stats_pool(ggml_tensor * x) ggml_tensor * w_t = ggml_soft_max(ctx0, a_t); ggml_tensor * w = ggml_cont(ctx0, ggml_transpose(ctx0, w_t)); // [C, T] - // weighted mean: sum(w * x) over T (w already sums to 1 over T, - // ggml_mean gives 1/T scaling so multiply back by T to undo it) + // weighted mean: sum(w * x) over T, multiply by T to undo ggml_mean's 1/T scaling ggml_tensor * wx = ggml_mul(ctx0, w, x); ggml_tensor * wx_t = ggml_cont(ctx0, ggml_transpose(ctx0, wx)); ggml_tensor * w_mean = ggml_mean(ctx0, wx_t); @@ -155,8 +152,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::attentive_stats_pool(ggml_tensor * x) } ggml_cgraph * clip_graph_qwen3tts_spkenc::build() { - // inp_raw: [T, n_mel, 1, 1] (nx=T frames, ny=n_mel bins), from the - // preprocessor's mel_spectrogram() output (mtmd_audio_preprocessor_qwen3tts_spk) + // inp_raw: [T, n_mel, 1, 1], from mtmd_audio_preprocessor_qwen3tts_spk ggml_tensor * inp = build_inp_raw(1); inp = ggml_reshape_2d(ctx0, inp, inp->ne[0], inp->ne[1]); diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp index 2811d24df7..7fbc18ea93 100644 --- a/tools/mtmd/mtmd-audio.cpp +++ b/tools/mtmd/mtmd-audio.cpp @@ -794,12 +794,8 @@ bool mtmd_audio_preprocessor_mimo_audio::preprocess(const float * // // mtmd_audio_preprocessor_qwen3tts_spk // -// Mirrors qwen_tts.core.models.modeling_qwen3_tts.mel_spectrogram(): -// pad reflect by (n_fft - hop) / 2, STFT (n_fft, hop, win=n_fft, hann -// periodic, center=False), mel = slaney_mel_basis @ |STFT|, log(max(mel, 1e-5)). -// Unlike Whisper-style encoders the whole clip is consumed in a single -// forward pass by the ECAPA-TDNN body, so there's no 30s/3000-frame chunking -// or Whisper (max-8)/4 normalization here. +// same as mel_spectrogram() in modeling_qwen3_tts.py +// ECAPA-TDNN takes the whole clip in one pass, so no Whisper-style chunking or normalization // void mtmd_audio_preprocessor_qwen3tts_spk::initialize() { diff --git a/tools/mtmd/mtmd-helper-common.h b/tools/mtmd/mtmd-helper-common.h index 3083e21eea..968b4df9c8 100644 --- a/tools/mtmd/mtmd-helper-common.h +++ b/tools/mtmd/mtmd-helper-common.h @@ -56,7 +56,7 @@ struct mtmd_helper_logger { } }; -// inline (C++17): one shared instance across every TU that includes this header +// inline, so all TUs including this header share one instance inline mtmd_helper_logger g_logger; #define LOG_DBG(...) g_logger.log(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__) diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp index eb5e867637..b52dc8e5a3 100644 --- a/tools/mtmd/mtmd-helper-gen.cpp +++ b/tools/mtmd/mtmd-helper-gen.cpp @@ -19,8 +19,7 @@ // Audio generation helpers // -// maps the --tts-lang codes (see tools/tts/README.md) to the language -// names used by the codec_language special tokens +// --tts-lang codes -> language names used by the codec_language special tokens static const std::unordered_map tts_lang_codes = { { "zh", "chinese" }, { "en", "english" }, @@ -86,9 +85,8 @@ public: virtual int32_t set_input(const mtmd_helper_gen_audio_inp * inp) = 0; // decodes at most n_batch prompt tokens; returns remaining count (0 = done), <0 on error virtual int32_t step_prompt(int32_t n_batch) = 0; - // sampled may be LLAMA_TOKEN_NULL for pipelines with no discrete backbone token - // (e.g. continuous/diffusion models); such pipelines read whatever they need - // directly off h_state_in instead + // sampled can be LLAMA_TOKEN_NULL for pipelines with no discrete backbone token, + // those read what they need from h_state_in instead virtual int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out) = 0; virtual int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) = 0; @@ -101,8 +99,8 @@ protected: mtmd_gen_audio_info info; }; -// Qwen3-TTS: dual-track discrete AR (backbone codec_0 + MTP code-predictor for the -// remaining 15 codebooks) into a windowed causal conv/transformer decode (code2wav) +// Qwen3-TTS: backbone samples codec_0, code_predictor gives the other 15 codebooks, +// then code2wav decodes them to PCM class qwen3tts_gen_audio_pipeline : public mtmd_gen_audio_pipeline { public: using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline; @@ -187,9 +185,7 @@ public: n_prompt = (int) prompt.size(); - // the talker rides the qwen3vl interleaved mrope: positions carry - // n_pos_per_embd sections laid out [section * n_tokens + i], all - // equal for a pure text/codec stream + // the talker uses the qwen3vl interleaved mrope, all sections are equal for a text/codec stream mrope = llama_model_rope_type(model) == LLAMA_ROPE_TYPE_MROPE || llama_model_rope_type(model) == LLAMA_ROPE_TYPE_IMROPE; const int n_pos_per_embd = mrope ? 4 : 1; @@ -209,9 +205,8 @@ public: top_p = inp->top_p > 0 ? inp->top_p : 1.0f; out_type = inp->out_type; - // the text stream keeps flowing during generation: the input after - // frame k adds trailing text row k on top of the codes embedding, - // then tts_eos, then tts_pad once the utterance is spent + // the text stream keeps flowing during generation: after frame k, the input adds + // trailing text row k on top of the codes embedding, then tts_eos, then tts_pad for (int i = 3; i < n_ids - 5; i++) overlay.push_back(row(ids[(size_t) i])); overlay.push_back(row(tts_eos)); overlay.push_back(row(tts_pad)); @@ -356,8 +351,7 @@ private: return true; } - // encodes a reference wav (already loaded as a bitmap) through the mmproj's - // speaker encoder, returning the single x-vector embedding row it produces + // runs the reference wav through the speaker encoder, returns one x-vector embedding row bool encode_speaker(mtmd_bitmap * bitmap, std::vector & out) { if (!mtmd_support_audio(mctx)) { LOG_ERR("mtmd_helper_gen_audio: mmproj has no speaker/audio encoder\n"); @@ -390,8 +384,7 @@ private: return ok; } - // runs one GEN_WAV process() call on whatever is currently buffered, carrying - // the persisted state (KV cache + conv left-context) across batches + // one GEN_WAV process() call over the buffered codes, state is carried across batches bool flush_gen_wav() { if (codes_buf.empty()) { return true; @@ -427,8 +420,7 @@ private: llama_token tts_eos = LLAMA_TOKEN_NULL; std::vector tok_embd; // whole token embedding matrix, n_vocab * n_embd - // matches hparams.wav_tfm_sliding_window hardcoded in clip.cpp; code2wav - // batches exactly this many frames per call + // must match hparams.wav_tfm_swa hardcoded in clip.cpp size_t window_frames = 72; // per-generation state, cleared by reset() diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h index 420456e47a..7e5cf9b509 100644 --- a/tools/mtmd/mtmd-helper.h +++ b/tools/mtmd/mtmd-helper.h @@ -202,7 +202,6 @@ MTMD_API int32_t mtmd_helper_gen_audio_set_input( const struct mtmd_helper_gen_audio_inp * inp); // processes at most n_batch prompt tokens per call -// returns 0 (no more prompt left to process) // returns: >0 = number of prompt tokens remaining, 0 = done, <0 = error MTMD_API int32_t mtmd_helper_gen_audio_step_prompt( mtmd_helper_gen_audio * ctx, @@ -217,8 +216,7 @@ MTMD_API int32_t mtmd_helper_gen_audio_step_gen( const float ** h_state_out); // out_data valid until next get_output() or reset() call -// out_n_samples (optional, can be NULL) receives the number of generated PCM samples, -// which combined with out_sample_rate gives the output audio duration +// out_n_samples (optional, can be NULL) receives the number of generated PCM samples MTMD_API int32_t mtmd_helper_gen_audio_get_output( mtmd_helper_gen_audio * ctx, int32_t * out_sample_rate,