Cloyir/index-tts2-ForDgxSpark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复拼音问题和分句问题，支持轻音声调（如yi1 shang5） (#83)

Browse Source 
* Update Pinyin tone handling in TextNormalizer

* Enhance sentence splitting and improve tokenizer integration in inference

* Update character replacement mappings

test: "在电影《肖申克的救赎》中，安迪·杜佛兰被错误地判处终身监禁..."

* Refactor TextNormalizer and enhance testing with additional cases
This commit is contained in:
Yrom Wang 2025-04-14 19:50:36 +08:00 committed by GitHub
parent f07a9032c1
commit 18c32c06b1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 116 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
indextts/infer.py
View File

@ -75,6 +75,8 @@ class IndexTTS:
self.bigvgan.eval()
print(">> bigvgan weights restored from:", self.bigvgan_path)
self.bpe_path = os.path.join(self.model_dir, self.cfg.dataset['bpe_model'])
self.tokenizer = spm.SentencePieceProcessor(model_file=self.bpe_path)
print(">> bpe model loaded from:", self.bpe_path)
self.normalizer = TextNormalizer()
self.normalizer.load()
print(">> TextNormalizer loaded")
@ -134,6 +136,18 @@ class IndexTTS:
code_lens = torch.LongTensor(code_lens).to(device, dtype=dtype)
return codes, code_lens
def split_sentences(self, text):
"""
Split the text into sentences based on punctuation marks.
"""
# 匹配标点符号（包括中英文标点）
pattern = r'(?<=[.!?;。！？；])\s*'
sentences = re.split(pattern, text)
# 过滤掉空字符串和仅包含标点符号的字符串
return [
sentence.strip() for sentence in sentences if sentence.strip() and sentence.strip() not in {"'", ".", ","}
]
def infer(self, audio_prompt, text, output_path):
print(f"origin text:{text}")
text = self.preprocess_text(text)
@ -150,14 +164,8 @@ class IndexTTS:
auto_conditioning = cond_mel
tokenizer = spm.SentencePieceProcessor()
tokenizer.load(self.bpe_path)
punctuation = ["!", "?", ".", ";", "！", "？", "。", "；"]
pattern = r"(?<=[{0}])\s*".format("".join(punctuation))
sentences = [i for i in re.split(pattern, text) if i.strip() != ""]
sentences = self.split_sentences(text)
print("sentences:", sentences)
top_p = .8
top_k = 30
temperature = 1.0
@ -167,8 +175,8 @@ class IndexTTS:
repetition_penalty = 10.0
max_mel_tokens = 600
sampling_rate = 24000
lang = "EN"
lang = "ZH"
# lang = "EN"
# lang = "ZH"
wavs = []
print(">> start inference...")
@ -181,19 +189,19 @@ class IndexTTS:
# cleand_text = "他 那 像 HONG3 小 孩 似 的 话 , 引 得 人 们 HONG1 堂 大 笑 , 大 家 听 了 一 HONG3 而 散 ."
print("cleand_text:", cleand_text)
text_tokens = torch.IntTensor(tokenizer.encode(cleand_text)).unsqueeze(0).to(self.device)
text_tokens = torch.tensor(self.tokenizer.EncodeAsIds(cleand_text),dtype=torch.int32, device=self.device).unsqueeze(0)
# text_tokens = F.pad(text_tokens, (0, 1)) # This may not be necessary.
# text_tokens = F.pad(text_tokens, (1, 0), value=0)
# text_tokens = F.pad(text_tokens, (0, 1), value=1)
# text_tokens = text_tokens.to(self.device)
print(text_tokens)
print(f"text_tokens shape: {text_tokens.shape}, text_tokens type: {text_tokens.dtype}")
text_token_syms = [tokenizer.IdToPiece(idx) for idx in text_tokens[0].tolist()]
# debug tokenizer
text_token_syms = self.tokenizer.IdToPiece(text_tokens[0].tolist())
print(text_token_syms)
text_len = [text_tokens.size(1)]
text_len = torch.IntTensor(text_len).to(self.device)
print(text_len)
# text_len = torch.IntTensor([text_tokens.size(1)], device=text_tokens.device)
# print(text_len)
with torch.no_grad():
with torch.amp.autocast(self.device, enabled=self.dtype is not None, dtype=self.dtype):
@ -234,8 +242,7 @@ class IndexTTS:
wav, _ = self.bigvgan(latent.transpose(1, 2), auto_conditioning.transpose(1, 2))
wav = wav.squeeze(1).cpu()
wav = 32767 * wav
torch.clip(wav, -32767.0, 32767.0)
wav = torch.clip(32767 * wav, -32767.0, 32767.0)
print(f"wav shape: {wav.shape}")
# wavs.append(wav[:, :-512])
wavs.append(wav)
@ -244,7 +251,7 @@ class IndexTTS:
elapsed_time = end_time - start_time
minutes, seconds = divmod(int(elapsed_time), 60)
milliseconds = int((elapsed_time - int(elapsed_time)) * 1000)
print(f">> inference done. time: {minutes}:{seconds}.{milliseconds}")
print(f">> inference done. time: {minutes:02d}:{seconds:02d}.{milliseconds:03d}")
print(">> saving wav file")
wav = torch.cat(wavs, dim=1)
torchaudio.save(output_path, wav.type(torch.int16), sampling_rate)
@ -259,4 +266,4 @@ if __name__ == "__main__":
text="There is a vehicle arriving in dock number 7?"
tts = IndexTTS(cfg_path="checkpoints/config.yaml", model_dir="checkpoints", is_fp16=True)
tts.infer(audio_prompt=prompt_wav, text=text, output_path="gen.wav")
tts.infer(audio_prompt=prompt_wav, text=text, output_path="gen.wav", verbose=True)

167
indextts/utils/front.py
View File

@ -4,7 +4,6 @@ import re
class TextNormalizer:
def __init__(self):
# self.normalizer = Normalizer(cache_dir="textprocessing/tn")
self.zh_normalizer = None
self.en_normalizer = None
self.char_rep_map = {
@ -15,8 +14,8 @@ class TextNormalizer:
"。": ".",
"！": "!",
"？": "?",
"\n": ".",
"·": ",",
"\n": " ",
"·": "-",
"、": ",",
"...": "…",
"……": "…",
@ -48,16 +47,20 @@ class TextNormalizer:
# 正则表达式匹配邮箱格式：数字英文@数字英文.英文
pattern = r'^[a-zA-Z0-9]+@[a-zA-Z0-9]+\.[a-zA-Z]+$'
return re.match(pattern, email) is not None
"""
匹配拼音声调格式：pinyin+数字，声调1-5，5表示轻声
例如：xuan4, jve2, ying1, zhong4, shang5
"""
PINYIN_TONE_PATTERN = r"([bmnpqdfghjklzcsxwy]?h?[aeiouüv]{1,2}[ng]*|ng)([1-5])"
def use_chinese(self, s):
has_chinese = bool(re.search(r'[\u4e00-\u9fff]', s))
has_digit = bool(re.search(r'\d', s))
has_alpha = bool(re.search(r'[a-zA-Z]', s))
is_email = self.match_email(s)
if has_chinese or not has_alpha or is_email:
return True
else:
return False
has_pinyin = bool(re.search(self.PINYIN_TONE_PATTERN, s, re.IGNORECASE))
return has_pinyin
def load(self):
# print(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
@ -73,94 +76,102 @@ class TextNormalizer:
self.zh_normalizer = NormalizerZh(remove_interjections=False, remove_erhua=False,overwrite_cache=False)
self.en_normalizer = NormalizerEn(overwrite_cache=False)
def infer(self, text):
pattern = re.compile("|".join(re.escape(p) for p in self.char_rep_map.keys()))
replaced_text = pattern.sub(lambda x: self.char_rep_map[x.group()], text)
def infer(self, text: str):
if not self.zh_normalizer or not self.en_normalizer:
print("Error, text normalizer is not initialized !!!")
return ""
replaced_text, pinyin_list = self.save_pinyin_tones(text.rstrip())
try:
normalizer = self.zh_normalizer if self.use_chinese(replaced_text) else self.en_normalizer
result = normalizer.normalize(replaced_text)
except Exception:
result = ""
print(traceback.format_exc())
result = self.restore_pinyin_tone_numbers(replaced_text, result)
result = self.restore_pinyin_tones(result, pinyin_list)
pattern = re.compile("|".join(re.escape(p) for p in self.char_rep_map.keys()))
result = pattern.sub(lambda x: self.char_rep_map[x.group()], result)
return result
def pinyin_match(self, pinyin):
pattern = r"(qun)(\d)"
repl = r"qvn\g<2>"
pinyin = re.sub(pattern, repl, pinyin)
pattern = r"(quan)(\d)"
repl = r"qvan\g<2>"
pinyin = re.sub(pattern, repl, pinyin)
pattern = r"(que)(\d)"
repl = r"qve\g<2>"
pinyin = re.sub(pattern, repl, pinyin)
pattern = r"(qu)(\d)"
repl = r"qv\g<2>"
pinyin = re.sub(pattern, repl, pinyin)
pattern = r"(ju)(\d)"
repl = r"jv\g<2>"
pinyin = re.sub(pattern, repl, pinyin)
pattern = r"(jue)(\d)"
repl = r"jve\g<2>"
pinyin = re.sub(pattern, repl, pinyin)
pattern = r"(xun)(\d)"
repl = r"xvn\g<2>"
pinyin = re.sub(pattern, repl, pinyin)
pattern = r"(xue)(\d)"
repl = r"xve\g<2>"
pinyin = re.sub(pattern, repl, pinyin)
pattern = r"(xu)(\d)"
repl = r"xv\g<2>"
pinyin = re.sub(pattern, repl, pinyin)
pattern = r"(juan)(\d)"
repl = r"jvan\g<2>"
pinyin = re.sub(pattern, repl, pinyin)
pattern = r"(jun)(\d)"
repl = r"jvn\g<2>"
pinyin = re.sub(pattern, repl, pinyin)
pattern = r"(xuan)(\d)"
repl = r"xvan\g<2>"
def correct_pinyin(self, pinyin):
"""
将 jqx 的韵母为 u/ü 的拼音转换为 v
如：ju -> jv , que -> qve, xün -> xvn
"""
if pinyin[0] not in "jqx":
return pinyin
# 匹配 jqx 的韵母为 u/ü 的拼音
pattern = r"([jqx])[uü](n|e|an)*(\d)"
repl = r"\g<1>v\g<2>\g<3>"
pinyin = re.sub(pattern, repl, pinyin)
return pinyin
def restore_pinyin_tone_numbers(self,original_text, processed_text):
# 第一步：恢复拼音后的音调数字（1-4）
# 建立中文数字到阿拉伯数字的映射
chinese_to_num = {'一': '1', '二': '2', '三': '3', '四': '4'}
def save_pinyin_tones(self, original_text):
"""
替换拼音声调为占位符 <pinyin_a>, <pinyin_b>, ...
例如：xuan4 -> <pinyin_a>
"""
# 声母韵母+声调数字
origin_pinyin_pattern = re.compile(self.PINYIN_TONE_PATTERN, re.IGNORECASE)
original_pinyin_list = re.findall(origin_pinyin_pattern, original_text)
if len(original_pinyin_list) == 0:
return (original_text, None)
original_pinyin_list = list(set(''.join(p) for p in original_pinyin_list))
transformed_text = original_text
# 替换为占位符 <pinyin_a>, <pinyin_b>, ...
for i, pinyin in enumerate(original_pinyin_list):
number = chr(ord("a") + i)
transformed_text = transformed_text.replace(pinyin, f"<pinyin_{number}>")
# print("original_text: ", original_text)
# print("transformed_text: ", transformed_text)
return transformed_text, original_pinyin_list
# 使用正则表达式找到拼音+中文数字的组合（如 "xuan四"）
def replace_tone(match):
pinyin = match.group(1) # 拼音部分
chinese_num = match.group(2) # 中文数字部分
# 将中文数字转换为阿拉伯数字
num = chinese_to_num.get(chinese_num, chinese_num)
return f"{pinyin}{num}"
# 匹配拼音后跟中文数字（一、二、三、四）的情况
pattern = r'([a-zA-Z]+)([一二三四])'
restored_text = re.sub(pattern, replace_tone, processed_text)
restored_text = restored_text.lower()
restored_text = self.pinyin_match(restored_text)
return restored_text
def restore_pinyin_tones(self, normalized_text, original_pinyin_list):
"""
恢复拼音中的音调数字（1-5）为原来的拼音
例如：<pinyin_a> -> original_pinyin_list[0]
"""
if not original_pinyin_list or len(original_pinyin_list) == 0:
return normalized_text
transformed_text = normalized_text
# 替换为占位符 <pinyin_a>, <pinyin_b>, ...
for i, pinyin in enumerate(original_pinyin_list):
number = chr(ord("a") + i)
pinyin = self.correct_pinyin(pinyin)
transformed_text = transformed_text.replace(f"<pinyin_{number}>", pinyin)
# print("normalized_text: ", normalized_text)
# print("transformed_text: ", transformed_text)
return transformed_text
if __name__ == '__main__':
# 测试程序
text_normalizer = TextNormalizer()
print(text_normalizer.infer("2.5平方电线"))
text_normalizer.load()
cases = [
"我爱你！",
"I love you!",
"我爱你的英语是”I love you“",
"2.5平方电线",
"共465篇，约315万字",
"2002年的第一场雪，下在了2003年",
"速度是10km/h",
"现在是北京时间2025年01月11日 20:00",
"他这条裤子是2012年买的，花了200块钱",
"电话：135-4567-8900",
"1键3连",
"他这条视频点赞3000+，评论1000+，收藏500+",
"这是1024元的手机，你要吗？",
"受不liao3你了",
"”衣裳“不读衣chang2，而是读衣shang5",
"最zhong4要的是：不要chong2蹈覆辙",
"IndexTTS 正式发布1.0版本了，效果666",
"See you at 8:00 AM",
"8:00 AM 开会",
"苹果于2030/1/2发布新 iPhone 2X 系列手机，最低售价仅 ¥12999",
]
for case in cases:
print(f"原始文本: {case}")
print(f"处理后文本: {text_normalizer.infer(case)}")
print("-" * 50)