diff --git a/indextts/infer.py b/indextts/infer.py index f1f419e..ff707f5 100644 --- a/indextts/infer.py +++ b/indextts/infer.py @@ -42,6 +42,7 @@ class IndexTTS: self.bigvgan = self.bigvgan.to(self.device) self.bigvgan.eval() print(">> bigvgan weights restored from:", self.bigvgan_path) + self.bpe_path = os.path.join(self.model_dir, self.cfg.dataset['bpe_model']) self.normalizer = TextNormalizer() self.normalizer.load() print(">> TextNormalizer loaded") @@ -57,6 +58,40 @@ class IndexTTS: # return text.translate(punctuation_map) return self.normalizer.infer(text) + def remove_long_silence(self, codes): + code_lens = [] + for i in range(0, codes.shape[0]): + code = codes[i] + if self.cfg.gpt.stop_mel_token not in code: + code_lens.append(len(code)) + len_ = len(code) + else: + # len_ = code.cpu().tolist().index(8193)+1 + len_ = (code == self.stop_mel_token).nonzero(as_tuple=False)[0] + 1 + len_ = len_ - 2 + + count = torch.sum(code == 52).item() + if count > 50: + code = code.cpu().tolist() + ncode = [] + n = 0 + for k in range(0, len_): + if code[k] != 52: + ncode.append(code[k]) + n = 0 + elif code[k] == 52 and n < 30: + ncode.append(code[k]) + n += 1 + # if (k == 0 and code[k] == 52) or (code[k] == 52 and code[k-1] == 52): + # n += 1 + len_ = len(ncode) + ncode = torch.LongTensor(ncode) + codes[i] = self.stop_mel_token + codes[i, 0:len_] = ncode + code_lens.append(len_) + code_lens = torch.LongTensor(code_lens).cuda() + return codes, code_lens + def infer(self, audio_prompt, text, output_path): print(f"origin text:{text}") text = self.preprocess_text(text) @@ -74,7 +109,7 @@ class IndexTTS: auto_conditioning = cond_mel tokenizer = spm.SentencePieceProcessor() - tokenizer.load(os.path.join(self.model_dir,self.cfg.dataset['bpe_model'])) + tokenizer.load(self.bpe_path) punctuation = ["!", "?", ".", ";", "!", "?", "。", ";"] pattern = r"(?<=[{0}])\s*".format("".join(punctuation)) @@ -128,15 +163,19 @@ class IndexTTS: num_beams=num_beams, repetition_penalty=repetition_penalty, max_generate_length=max_mel_tokens) + codes = codes[:, :-2] + # code_lens = torch.tensor([codes.shape[-1]]) print(codes) print(f"codes shape: {codes.shape}") - codes = codes[:, :-2] + # remove ultra-long silence if exits + codes, code_lens = self.remove_long_silence(codes) + print(f"codes shape: {codes.shape}") # latent, text_lens_out, code_lens_out = \ latent = \ self.gpt(auto_conditioning, text_tokens, torch.tensor([text_tokens.shape[-1]], device=text_tokens.device), codes, - torch.tensor([codes.shape[-1] * self.gpt.mel_length_compression], device=text_tokens.device), + code_lens*self.gpt.mel_length_compression, cond_mel_lengths=torch.tensor([auto_conditioning.shape[-1]], device=text_tokens.device), return_latent=True, clip_inputs=False) latent = latent.transpose(1, 2) @@ -162,6 +201,10 @@ class IndexTTS: torchaudio.save(output_path, wav.type(torch.int16), 24000) +prompt_wav="/juicefs/users/wd007/work2024/tts/indextts/testwav/spk_1743041132.wav" +text="晕 XUAN4 是 一 种 GAN3 觉" +text='大家好,我现在正在bilibili 体验 ai 科技,说实话,来之前我绝对想不到!AI技术已经发展到这样匪夷所思的地步了!' if __name__ == "__main__": tts = IndexTTS(cfg_path="checkpoints/config.yaml", model_dir="checkpoints") - tts.infer(audio_prompt='test_data/input.wav', text='大家好,我现在正在bilibili 体验 ai 科技,说实话,来之前我绝对想不到!AI技术已经发展到这样匪夷所思的地步了!', output_path="gen.wav") + #tts.infer(audio_prompt='test_data/input.wav', text='大家好,我现在正在bilibili 体验 ai 科技,说实话,来之前我绝对想不到!AI技术已经发展到这样匪夷所思的地步了!', output_path="gen.wav") + tts.infer(audio_prompt=prompt_wav, text=text, output_path="gen.wav")