Cloyir/index-tts2-ForDgxSpark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Warn if input text contains UNK tokens

Browse Source 
Added warnings for unknown tokens in input text.
This commit is contained in:
Christopher Özbek 2025-09-26 13:36:11 +02:00 committed by Christopher Oezbek
parent 84a5ef97b8
commit 34be9bfb14
1 changed files with 7 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
indextts/infer_v2.py
View File

@ -447,6 +447,13 @@ class IndexTTS2:
text_tokens_list = self.tokenizer.tokenize(text)
segments = self.tokenizer.split_segments(text_tokens_list, max_text_tokens_per_segment)
segments_count = len(segments)
text_token_ids = self.tokenizer.convert_tokens_to_ids(text_tokens_list)
if self.tokenizer.unk_token_id in text_token_ids:
print(f" >> Warning: input text contains {text_token_ids.count(self.tokenizer.unk_token_id)} unknown tokens (id={self.tokenizer.unk_token_id}):")
print( " Tokens which can't be encoded: ", [t for t, id in zip(text_tokens_list, text_token_ids) if id == self.tokenizer.unk_token_id])
print(f" Consider updating the BPE model or modifying the text to avoid unknown tokens.")
if verbose:
print("text_tokens_list:", text_tokens_list)
print("segments count:", segments_count)