fix(nano): correct punctuation timestamps across VAD merges

funasr/models/fun_asr_nano/inference_vllm.py

@@ -694,7 +694,10 @@ def generate(
     @torch.no_grad()
     def _compute_timestamps(self, encoder_out, encoder_out_lens, text):
         """CTC forced alignment for character-level timestamps."""
-        from funasr.models.fun_asr_nano.tools.utils import forced_align
+        from funasr.models.fun_asr_nano.tools.utils import (
+            anchor_punctuation_timestamps,
+            forced_align,
+        )
 
         decoder_out, decoder_out_lens = self.ctc_decoder(encoder_out, encoder_out_lens)
         ctc_logits = self.ctc.log_softmax(decoder_out)
@@ -709,6 +712,9 @@ def _compute_timestamps(self, encoder_out, encoder_out_lens, text):
             ts["token"] = self.ctc_tokenizer.decode([ts["token"]])
             ts["start_time"] = ts["start_time"] * 6 * 10 / 1000
             ts["end_time"] = ts["end_time"] * 6 * 10 / 1000
+        # See model.py: anchor punctuation at the preceding spoken token's end
+        # (issue #3702); punctuation has no acoustic extent to align.
+        anchor_punctuation_timestamps(timestamps)
         return timestamps
 
     @classmethod

funasr/models/fun_asr_nano/inference_vllm_pipeline.py

@@ -322,7 +322,10 @@ def _process_one(self, audio_path, **kwargs):
 
     def _compute_all_timestamps(self, segment_audios, vad_segments, asr_results):
         """Compute CTC timestamps for all segments with VAD offsets."""
-        from funasr.models.fun_asr_nano.tools.utils import forced_align
+        from funasr.models.fun_asr_nano.tools.utils import (
+            anchor_punctuation_timestamps,
+            forced_align,
+        )
 
         all_timestamps = []
         for seg_audio, vad_seg, text in zip(segment_audios, vad_segments, asr_results):
@@ -356,6 +359,13 @@ def _compute_all_timestamps(self, segment_audios, vad_segments, asr_results):
                     ts["token"] = self.asr_engine.ctc_tokenizer.decode([ts["token"]])
                     ts["start_time"] = ts["start_time"] * 6 * 10 / 1000 + vad_offset_ms / 1000
                     ts["end_time"] = ts["end_time"] * 6 * 10 / 1000 + vad_offset_ms / 1000
+                # Same invariant as the offline/vLLM paths (issue #3702), applied
+                # per independently aligned VAD segment: punctuation carries no
+                # acoustic extent, so pin sandwiched punctuation at the preceding
+                # spoken token's end. Segment-local on purpose — speech in a
+                # later VAD segment must not rewrite this segment's trailing
+                # punctuation.
+                anchor_punctuation_timestamps(timestamps)
                 all_timestamps.extend(timestamps)
             except Exception as e:
                 logger.debug(f"Timestamp failed for segment: {e}")

funasr/models/fun_asr_nano/model.py

@@ -24,7 +24,7 @@
 from .ctc import CTC
 from .checkpoint_utils import disable_incomplete_ctc, normalize_checkpoint_state
 from .device_utils import resolve_autocast_device_type
-from .tools.utils import forced_align
+from .tools.utils import anchor_punctuation_timestamps, forced_align
 
 dtype_map = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}
 
@@ -1069,6 +1069,12 @@ def inference_llm(
                     timestamp["token"] = self.ctc_tokenizer.decode([timestamp["token"]])
                     timestamp["start_time"] = timestamp["start_time"] * 6 * 10 / 1000
                     timestamp["end_time"] = timestamp["end_time"] * 6 * 10 / 1000
+            # Punctuation tokens have no acoustic realization; without anchoring,
+            # forced alignment places them at the next sentence's onset inside
+            # merged multi-sentence windows (issue #3702). Applied after decoding
+            # (tokens are still integer ids above). ctc_timestamps is left
+            # untouched: greedy CTC text carries no punctuation to anchor.
+            anchor_punctuation_timestamps(result["timestamps"])
 
         if ibest_writer is not None:
             ibest_writer["text"][key[0]] = response.replace("\n", " ")

funasr/models/fun_asr_nano/tools/utils.py

@@ -1,4 +1,5 @@
 from itertools import groupby
+import re
 
 import librosa
 import soundfile as sf
@@ -77,4 +78,77 @@ def forced_align(log_probs: torch.Tensor, targets: torch.Tensor, blank: int = 0)
             )
     except Exception:
         pass
-    return items
+    return items
+
+
+_PUNCTUATION_TOKEN_RE = re.compile(r"^[^\w\s]+$")
+_SPECIAL_TOKEN_RE = re.compile(r"^<[^>]*>$")
+# tiktoken decodes a partial UTF-8 byte sequence (one id of a multi-id
+# character, e.g. SenseVoice ids [10958, 245] for 郗) as U+FFFD. All three
+# Nano timestamp paths decode each token id independently, so such fragments
+# reach classification as "�". They are spoken content, never punctuation.
+_UNDECODABLE_FRAGMENT_MARK = "�"
+
+
+def _classify_timestamp_token(token):
+    """Classify one decoded timestamp token into spoken/punctuation/special.
+
+    Special tokens (``<sil>``-shaped) are their own class: they are neither
+    spoken content nor punctuation, so they neither trigger anchoring nor
+    serve as anchors. Undecodable byte fragments (containing U+FFFD) are
+    spoken: they are slices of a multi-id character with real acoustic
+    extent, and collapsing their spans would delete spoken timing (issue
+    #3703). Non-string tokens (integer ids, if called pre-decode) fall back
+    to spoken to preserve pass-through behavior.
+    """
+    if not isinstance(token, str) or not token:
+        return "spoken"
+    if _SPECIAL_TOKEN_RE.match(token):
+        return "special"
+    if _UNDECODABLE_FRAGMENT_MARK in token:
+        return "spoken"
+    if _PUNCTUATION_TOKEN_RE.match(token):
+        return "punctuation"
+    return "spoken"
+
+
+def anchor_punctuation_timestamps(timestamps):
+    """Anchor punctuation-only tokens at the preceding spoken token's end.
+
+    CTC forced alignment must place every target token — including punctuation
+    tokens with no acoustic realization — on at least one frame. Inside a
+    merged multi-sentence window (see FunASR issue #3702) such a token lands
+    at the next sentence's onset frame, so the current sentence's punctuation
+    timestamp reads as the next sentence start. Punctuation carries no acoustic
+    extent, so a punctuation span that is followed by more speech is pinned
+    (zero-width) at the end of the speech it terminates.
+
+    Leading punctuation (no predecessor) and trailing punctuation (no
+    successor, e.g. single-segment output) are left untouched, as are special
+    tokens such as ``<sil>``. This preserves existing single-segment behavior.
+
+    Args:
+        timestamps: List of ``{"token": str, "start_time": float,
+            "end_time": float}`` dicts in time order. Mutated in place.
+
+    Returns:
+        The same list, for convenient chaining.
+    """
+    last_spoken_end = None
+    total = len(timestamps)
+    for index, ts in enumerate(timestamps):
+        kind = _classify_timestamp_token(ts.get("token", ""))
+        if kind != "punctuation":
+            if kind == "spoken":
+                end = ts.get("end_time")
+                if isinstance(end, (int, float)):
+                    last_spoken_end = end
+            continue
+        following_spoken = any(
+            _classify_timestamp_token(later.get("token", "")) == "spoken"
+            for later in timestamps[index + 1 : total]
+        )
+        if last_spoken_end is not None and following_spoken:
+            ts["start_time"] = last_spoken_end
+            ts["end_time"] = last_spoken_end
+    return timestamps

tests/test_fun_asr_nano_byte_fragment_timestamps.py

@@ -0,0 +1,205 @@
+"""Byte-fragment spoken tokens must never be treated as punctuation.
+
+Review regression for FunASR PR #3703 (issue #3702 follow-up): Nano's CTC
+tokenizer is byte-based, so one spoken character can span several token ids
+(official SenseVoiceTokenizer: 郗 -> [10958, 245]). All three Nano timestamp
+paths decode each token id independently, and each fragment decodes to U+FFFD
+while the id sequence decodes to real speech. The punctuation classifier must
+not collapse such fragments' valid acoustic spans.
+
+These tests instantiate the repository's real ``SenseVoiceTokenizer``
+production path (``SenseVoiceTokenizer`` factory -> ``get_tokenizer`` ->
+``get_encoding`` -> production ``Tokenizer``) against a deterministic
+temporary offline ``.tiktoken`` vocab, so they run without the Fun-ASR-Nano
+``multilingual.tiktoken`` model artifact (which ships with the model, not
+this repo) and without weights/GPU. The fixture vocab mirrors the official
+vocab shape: ordinary CJK chars and punctuation are single ids, while 郗
+(deliberately given no merged rank) splits into single-byte ids whose
+independent decodes are U+FFFD — the exact byte-fragment mechanism
+(official: 郗 -> [10958, 245]). Official-artifact cross-check
+(SenseVoiceTokenizer from FunAudioLLM/Fun-ASR-Nano-2512
+``multilingual.tiktoken`` @272c57b, sha256 74797963...): 郗 -> [10958, 245],
+per-id decodes U+FFFD, roundtrip intact; helper/vLLM/pipeline probes preserve
+[0.60, 0.72]/[0.84, 0.96]-style spans post-fix (see PR review evidence).
+"""
+
+import base64
+import types
+
+import numpy as np
+import pytest
+import torch
+
+from funasr.models.fun_asr_nano.tools.utils import (
+    _classify_timestamp_token,
+    anchor_punctuation_timestamps,
+    forced_align,
+)
+from funasr.tokenizer.whisper_tokenizer import SenseVoiceTokenizer
+
+TEXT = "我叫郗明。你好。"
+FRAMES_PER_TOKEN = 10
+FRAME_TO_SEC = 6 * 10 / 1000
+
+
+def _write_fragment_vocab(path):
+    """Deterministic offline vocab: byte-level base plus single-id merged
+    ranks for every fixture char except 郗, which stays byte-split. Format
+    matches production ``.tiktoken`` files (``base64(token) rank`` lines)."""
+    lines = []
+    for byte in range(256):
+        lines.append(f"{base64.b64encode(bytes([byte])).decode()} {byte}")
+    next_rank = 256
+    # Prefix pair first so BPE completes each 3-byte merge (tiktoken merges
+    # lowest-rank pairs first); 郗 gets no ranks and stays byte-split.
+    for char in "我叫明你好。，":
+        raw = char.encode("utf-8")
+        assert len(raw) == 3, char
+        lines.append(f"{base64.b64encode(raw[:2]).decode()} {next_rank}")
+        next_rank += 1
+        lines.append(f"{base64.b64encode(raw).decode()} {next_rank}")
+        next_rank += 1
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+@pytest.fixture()
+def sensevoice_tokenizer(tmp_path):
+    """The real production Tokenizer via the SenseVoiceTokenizer factory."""
+    vocab = tmp_path / "byte-fragment.tiktoken"
+    _write_fragment_vocab(vocab)
+    tokenizer = SenseVoiceTokenizer(vocab_path=str(vocab))
+    target_ids = tokenizer.encode(TEXT)
+    assert tokenizer.decode(target_ids) == TEXT
+    return tokenizer
+
+
+def _peaky_log_probs(target_ids, vocab_size, frames_per_token=FRAMES_PER_TOKEN):
+    n_frames = len(target_ids) * frames_per_token
+    log_probs = torch.full((n_frames, vocab_size), -30.0)
+    for i, tid in enumerate(target_ids):
+        log_probs[i * frames_per_token : (i + 1) * frames_per_token, tid] = 0.0
+    return log_probs
+
+
+def _blank_id(tokenizer):
+    return tokenizer.get_vocab_size() - 1
+
+
+def _production_timestamps(tokenizer, target_ids):
+    """Mirror the production decode-per-id + scale path shared by all three
+    Nano timestamp methods, over the repo's real forced_align."""
+    items = forced_align(
+        _peaky_log_probs(target_ids, tokenizer.get_vocab_size()),
+        torch.tensor(target_ids, dtype=torch.int64),
+        _blank_id(tokenizer),
+    )
+    assert [it["token"] for it in items] == list(target_ids)
+    return [
+        {
+            "token": tokenizer.decode([it["token"]]),
+            "start_time": it["start_time"] * FRAME_TO_SEC,
+            "end_time": it["end_time"] * FRAME_TO_SEC,
+        }
+        for it in items
+    ]
+
+
+def test_undecodable_fragment_is_spoken_not_punctuation():
+    assert _classify_timestamp_token("�") == "spoken"
+    assert _classify_timestamp_token("��") == "spoken"
+    # Genuine punctuation still classifies as punctuation.
+    assert _classify_timestamp_token("。") == "punctuation"
+    assert _classify_timestamp_token(",") == "punctuation"
+    # Special tokens keep their own class.
+    assert _classify_timestamp_token("<sil>") == "special"
+
+
+def test_byte_fragment_spans_survive_anchoring(sensevoice_tokenizer):
+    tokenizer = sensevoice_tokenizer
+    target_ids = tokenizer.encode(TEXT)
+    assert len(target_ids) > len(TEXT), "郗 must stay byte-split"
+    assert tokenizer.decode(target_ids) == TEXT
+    assert any(tokenizer.decode([i]) == "�" for i in target_ids)
+
+    timestamps = _production_timestamps(tokenizer, target_ids)
+    before = [(t["token"], t["start_time"], t["end_time"]) for t in timestamps]
+    anchor_punctuation_timestamps(timestamps)
+
+    for (token, start, end), ts in zip(before, timestamps):
+        if token == "�":
+            assert (ts["start_time"], ts["end_time"]) == (start, end)
+            assert ts["end_time"] > ts["start_time"], "fragment keeps acoustic extent"
+    # Genuine 。 between speech still anchors (control).
+    punct = [t for t in timestamps if t["token"] == "。"]
+    assert punct, "expected genuine punctuation in fixtures"
+    assert punct[0]["start_time"] == punct[0]["end_time"]
+
+
+def test_vllm_method_preserves_byte_fragment_spans(sensevoice_tokenizer):
+    from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM
+
+    tokenizer = sensevoice_tokenizer
+    target_ids = tokenizer.encode(TEXT)
+    n_frames = len(target_ids) * FRAMES_PER_TOKEN
+    log_probs = _peaky_log_probs(target_ids, tokenizer.get_vocab_size())
+
+    engine = FunASRNanoVLLM.__new__(FunASRNanoVLLM)
+    engine.ctc_decoder = lambda e, l: (log_probs.unsqueeze(0), torch.tensor([n_frames]))
+    engine.ctc = types.SimpleNamespace(log_softmax=lambda d: d)
+    engine.ctc_tokenizer = tokenizer
+    engine.blank_id = _blank_id(tokenizer)
+
+    result = engine._compute_timestamps(
+        torch.zeros(1, n_frames, 8), torch.tensor([n_frames]), TEXT
+    )
+    fragments = [t for t in result if t["token"] == "�"]
+    assert len(fragments) >= 2
+    for t in fragments:
+        assert t["end_time"] > t["start_time"]
+    assert any(t["token"] == "。" and t["start_time"] == t["end_time"] for t in result)
+
+
+def test_pipeline_method_preserves_byte_fragment_spans(
+    sensevoice_tokenizer, monkeypatch
+):
+    from funasr.models.fun_asr_nano.inference_vllm_pipeline import (
+        FunASRNanoVLLMPipeline,
+    )
+
+    tokenizer = sensevoice_tokenizer
+    target_ids = tokenizer.encode(TEXT)
+    n_frames = len(target_ids) * FRAMES_PER_TOKEN
+    log_probs = _peaky_log_probs(target_ids, tokenizer.get_vocab_size())
+
+    monkeypatch.setattr(
+        "funasr.utils.load_utils.extract_fbank",
+        lambda *a, **k: (torch.zeros(1, 4, 80), torch.tensor([[4]])),
+    )
+    pipeline = FunASRNanoVLLMPipeline.__new__(FunASRNanoVLLMPipeline)
+    pipeline.device = "cpu"
+    pipeline.asr_engine = types.SimpleNamespace(
+        frontend=object(),
+        audio_encoder=lambda s, sl: (
+            torch.zeros(1, n_frames, 8),
+            torch.tensor([n_frames]),
+        ),
+        ctc_decoder=lambda e, l: (log_probs.unsqueeze(0), torch.tensor([n_frames])),
+        ctc=types.SimpleNamespace(log_softmax=lambda d: d),
+        ctc_tokenizer=tokenizer,
+        blank_id=_blank_id(tokenizer),
+    )
+    result = pipeline._compute_all_timestamps(
+        [np.zeros(32000, dtype=np.float32)], [[1000, 9000]], [TEXT]
+    )
+    fragments = [t for t in result if t["token"] == "�"]
+    assert len(fragments) >= 2
+    for t in fragments:
+        assert t["end_time"] > t["start_time"]
+    # +1s VAD offset applied on top of preserved spans: compare against the
+    # un-offset helper path for the identical target sequence.
+    expected = _production_timestamps(tokenizer, target_ids)
+    expected_frags = [t for t in expected if t["token"] == "�"]
+    assert len(fragments) == len(expected_frags)
+    for got, want in zip(fragments, expected_frags):
+        assert got["start_time"] == pytest.approx(want["start_time"] + 1.0)
+        assert got["end_time"] == pytest.approx(want["end_time"] + 1.0)