From 54243347014ce6b16563237d51dd706b2e1fe25e Mon Sep 17 00:00:00 2001 From: Plachta Date: Tue, 14 Feb 2023 17:51:36 +0800 Subject: [PATCH] upload files --- VC_inference.py | 90 ++++++++ configs/finetune_speaker.json | 156 ++++++++++++- finetune_speaker.py | 4 +- models_infer.py | 401 ++++++++++++++++++++++++++++++++++ utils.py | 2 +- 5 files changed, 648 insertions(+), 5 deletions(-) create mode 100644 VC_inference.py create mode 100644 models_infer.py diff --git a/VC_inference.py b/VC_inference.py new file mode 100644 index 0000000..2d0b720 --- /dev/null +++ b/VC_inference.py @@ -0,0 +1,90 @@ +import os +import json +import math +import numpy as np +import torch +from torch import no_grad, LongTensor +import librosa +from torch.nn import functional as F +import argparse +from mel_processing import spectrogram_torch +import commons +import utils +from models_infer import SynthesizerTrn +from text import text_to_sequence +import gradio as gr +import torchaudio + +def get_text(text, hps): + text_norm = text_to_sequence(text, hps.symbols, hps.data.text_cleaners) + if hps.data.add_blank: + text_norm = commons.intersperse(text_norm, 0) + text_norm = torch.LongTensor(text_norm) + return text_norm + +def create_vc_fn(model, hps, speaker_ids): + def vc_fn(original_speaker, target_speaker, record_audio, upload_audio): + input_audio = record_audio if record_audio is not None else upload_audio + if input_audio is None: + return "You need to record or upload an audio", None + sampling_rate, audio = input_audio + original_speaker_id = speaker_ids[original_speaker] + target_speaker_id = speaker_ids[target_speaker] + + audio = (audio / np.iinfo(audio.dtype).max).astype(np.float32) + if len(audio.shape) > 1: + audio = librosa.to_mono(audio.transpose(1, 0)) + if sampling_rate != hps.data.sampling_rate: + audio = librosa.resample(audio, orig_sr=sampling_rate, target_sr=hps.data.sampling_rate) + with no_grad(): + y = torch.FloatTensor(audio) + y = y.unsqueeze(0) + spec = spectrogram_torch(y, hps.data.filter_length, + hps.data.sampling_rate, hps.data.hop_length, hps.data.win_length, + center=False) + spec_lengths = LongTensor([spec.size(-1)]) + sid_src = LongTensor([original_speaker_id]) + sid_tgt = LongTensor([target_speaker_id]) + audio = model.voice_conversion(spec, spec_lengths, sid_src=sid_src, sid_tgt=sid_tgt)[0][ + 0, 0].data.cpu().float().numpy() + del y, spec, spec_lengths, sid_src, sid_tgt + return "Success", (hps.data.sampling_rate, audio) + + return vc_fn +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--model_dir", default="./G_latest.pth", help="directory to your fine-tuned model") + + args = parser.parse_args() + hps = utils.get_hparams_from_file("./configs/finetune_speaker.json") + device = "cpu" + + net_g = SynthesizerTrn( + len(hps.symbols), + hps.data.filter_length // 2 + 1, + hps.train.segment_size // hps.data.hop_length, + n_speakers=hps.data.n_speakers, + **hps.model).to(device) + _ = net_g.eval() + + _ = utils.load_checkpoint(args.model_dir, net_g, None) + speaker_ids = hps.speakers + speakers = list(hps.speakers.keys()) + vc_fn = create_vc_fn(net_g, hps, speaker_ids) + app = gr.Blocks() + with app: + gr.Markdown(""" + 录制或上传声音，并选择要转换的音色。User代表的音色是你自己。 + """) + with gr.Column(): + record_audio = gr.Audio(label="record your voice", source="microphone") + upload_audio = gr.Audio(label="or upload audio here", source="upload") + source_speaker = gr.Dropdown(choices=speakers, value="User", label="source speaker") + target_speaker = gr.Dropdown(choices=speakers, value=speakers[0], label="target speaker") + with gr.Column(): + message_box = gr.Textbox(label="Message") + converted_audio = gr.Audio(label='converted audio') + btn = gr.Button("Convert!") + btn.click(vc_fn, inputs=[source_speaker, target_speaker, record_audio, upload_audio], + outputs=[message_box, converted_audio]) + app.launch() diff --git a/configs/finetune_speaker.json b/configs/finetune_speaker.json index 307bb64..085293a 100644 --- a/configs/finetune_speaker.json +++ b/configs/finetune_speaker.json @@ -1,13 +1,13 @@ { "train": { "log_interval": 100, - "eval_interval": 1000, + "eval_interval": 200, "seed": 1234, "epochs": 10000, "learning_rate": 2e-4, "betas": [0.8, 0.99], "eps": 1e-9, - "batch_size": 1, + "batch_size": 16, "fp16_run": true, "lr_decay": 0.999875, "segment_size": 8192, @@ -50,5 +50,155 @@ "use_spectral_norm": false, "gin_channels": 256 }, - "symbols": ["_", ",", ".", "!", "?", "-", "~", "\u2026", "N", "Q", "a", "b", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "s", "t", "u", "v", "w", "x", "y", "z", "\u0251", "\u00e6", "\u0283", "\u0291", "\u00e7", "\u026f", "\u026a", "\u0254", "\u025b", "\u0279", "\u00f0", "\u0259", "\u026b", "\u0265", "\u0278", "\u028a", "\u027e", "\u0292", "\u03b8", "\u03b2", "\u014b", "\u0266", "\u207c", "\u02b0", "`", "^", "#", "*", "=", "\u02c8", "\u02cc", "\u2192", "\u2193", "\u2191", " "] + "symbols": ["_", ",", ".", "!", "?", "-", "~", "\u2026", "N", "Q", "a", "b", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "s", "t", "u", "v", "w", "x", "y", "z", "\u0251", "\u00e6", "\u0283", "\u0291", "\u00e7", "\u026f", "\u026a", "\u0254", "\u025b", "\u0279", "\u00f0", "\u0259", "\u026b", "\u0265", "\u0278", "\u028a", "\u027e", "\u0292", "\u03b8", "\u03b2", "\u014b", "\u0266", "\u207c", "\u02b0", "`", "^", "#", "*", "=", "\u02c8", "\u02cc", "\u2192", "\u2193", "\u2191", " "], + "speakers": {"特别周 Special Week (Umamusume Pretty Derby)": 0, + "无声铃鹿 Silence Suzuka (Umamusume Pretty Derby)": 1, + "东海帝王 Tokai Teio (Umamusume Pretty Derby)": 2, + "丸善斯基 Maruzensky (Umamusume Pretty Derby)": 3, + "富士奇迹 Fuji Kiseki (Umamusume Pretty Derby)": 4, + "小栗帽 Oguri Cap (Umamusume Pretty Derby)": 5, + "黄金船 Gold Ship (Umamusume Pretty Derby)": 6, + "伏特加 Vodka (Umamusume Pretty Derby)": 7, + "大和赤骥 Daiwa Scarlet (Umamusume Pretty Derby)": 8, + "大树快车 Taiki Shuttle (Umamusume Pretty Derby)": 9, + "草上飞 Grass Wonder (Umamusume Pretty Derby)": 10, + "菱亚马逊 Hishi Amazon (Umamusume Pretty Derby)": 11, + "目白麦昆 Mejiro Mcqueen (Umamusume Pretty Derby)": 12, + "神鹰 El Condor Pasa (Umamusume Pretty Derby)": 13, + "好歌剧 T.M. Opera O (Umamusume Pretty Derby)": 14, + "成田白仁 Narita Brian (Umamusume Pretty Derby)": 15, + "鲁道夫象征 Symboli Rudolf (Umamusume Pretty Derby)": 16, + "气槽 Air Groove (Umamusume Pretty Derby)": 17, + "爱丽数码 Agnes Digital (Umamusume Pretty Derby)": 18, + "青云天空 Seiun Sky (Umamusume Pretty Derby)": 19, + "玉藻十字 Tamamo Cross (Umamusume Pretty Derby)": 20, + "美妙姿势 Fine Motion (Umamusume Pretty Derby)": 21, + "琵琶晨光 Biwa Hayahide (Umamusume Pretty Derby)": 22, + "重炮 Mayano Topgun (Umamusume Pretty Derby)": 23, + "曼城茶座 Manhattan Cafe (Umamusume Pretty Derby)": 24, + "美普波旁 Mihono Bourbon (Umamusume Pretty Derby)": 25, + "目白雷恩 Mejiro Ryan (Umamusume Pretty Derby)": 26, + "雪之美人 Yukino Bijin (Umamusume Pretty Derby)": 28, + "米浴 Rice Shower (Umamusume Pretty Derby)": 29, + "艾尼斯风神 Ines Fujin (Umamusume Pretty Derby)": 30, + "爱丽速子 Agnes Tachyon (Umamusume Pretty Derby)": 31, + "爱慕织姬 Admire Vega (Umamusume Pretty Derby)": 32, + "稻荷一 Inari One (Umamusume Pretty Derby)": 33, + "胜利奖券 Winning Ticket (Umamusume Pretty Derby)": 34, + "空中神宫 Air Shakur (Umamusume Pretty Derby)": 35, + "荣进闪耀 Eishin Flash (Umamusume Pretty Derby)": 36, + "真机伶 Curren Chan (Umamusume Pretty Derby)": 37, + "川上公主 Kawakami Princess (Umamusume Pretty Derby)": 38, + "黄金城市 Gold City (Umamusume Pretty Derby)": 39, + "樱花进王 Sakura Bakushin O (Umamusume Pretty Derby)": 40, + "采珠 Seeking the Pearl (Umamusume Pretty Derby)": 41, + "新光风 Shinko Windy (Umamusume Pretty Derby)": 42, + "东商变革 Sweep Tosho (Umamusume Pretty Derby)": 43, + "超级小溪 Super Creek (Umamusume Pretty Derby)": 44, + "醒目飞鹰 Smart Falcon (Umamusume Pretty Derby)": 45, + "荒漠英雄 Zenno Rob Roy (Umamusume Pretty Derby)": 46, + "东瀛佐敦 Tosen Jordan (Umamusume Pretty Derby)": 47, + "中山庆典 Nakayama Festa (Umamusume Pretty Derby)": 48, + "成田大进 Narita Taishin (Umamusume Pretty Derby)": 49, + "西野花 Nishino Flower (Umamusume Pretty Derby)": 50, + "春乌拉拉 Haru Urara (Umamusume Pretty Derby)": 51, + "青竹回忆 Bamboo Memory (Umamusume Pretty Derby)": 52, + "待兼福来 Matikane Fukukitaru (Umamusume Pretty Derby)": 55, + "名将怒涛 Meisho Doto (Umamusume Pretty Derby)": 57, + "目白多伯 Mejiro Dober (Umamusume Pretty Derby)": 58, + "优秀素质 Nice Nature (Umamusume Pretty Derby)": 59, + "帝王光环 King Halo (Umamusume Pretty Derby)": 60, + "待兼诗歌剧 Matikane Tannhauser (Umamusume Pretty Derby)": 61, + "生野狄杜斯 Ikuno Dictus (Umamusume Pretty Derby)": 62, + "目白善信 Mejiro Palmer (Umamusume Pretty Derby)": 63, + "大拓太阳神 Daitaku Helios (Umamusume Pretty Derby)": 64, + "双涡轮 Twin Turbo (Umamusume Pretty Derby)": 65, + "里见光钻 Satono Diamond (Umamusume Pretty Derby)": 66, + "北部玄驹 Kitasan Black (Umamusume Pretty Derby)": 67, + "樱花千代王 Sakura Chiyono O (Umamusume Pretty Derby)": 68, + "天狼星象征 Sirius Symboli (Umamusume Pretty Derby)": 69, + "目白阿尔丹 Mejiro Ardan (Umamusume Pretty Derby)": 70, + "八重无敌 Yaeno Muteki (Umamusume Pretty Derby)": 71, + "鹤丸刚志 Tsurumaru Tsuyoshi (Umamusume Pretty Derby)": 72, + "目白光明 Mejiro Bright (Umamusume Pretty Derby)": 73, + "樱花桂冠 Sakura Laurel (Umamusume Pretty Derby)": 74, + "成田路 Narita Top Road (Umamusume Pretty Derby)": 75, + "也文摄辉 Yamanin Zephyr (Umamusume Pretty Derby)": 76, + "真弓快车 Aston Machan (Umamusume Pretty Derby)": 80, + "骏川手纲 Hayakawa Tazuna (Umamusume Pretty Derby)": 81, + "小林历奇 Kopano Rickey (Umamusume Pretty Derby)": 83, + "奇锐骏 Wonder Acute (Umamusume Pretty Derby)": 85, + "秋川理事长 President Akikawa (Umamusume Pretty Derby)": 86, + "綾地 寧々 Ayachi Nene (Sanoba Witch)": 87, + "因幡 めぐる Inaba Meguru (Sanoba Witch)": 88, + "椎葉 紬 Shiiba Tsumugi (Sanoba Witch)": 89, + "仮屋 和奏 Kariya Wakama (Sanoba Witch)": 90, + "戸隠 憧子 Togakushi Touko (Sanoba Witch)": 91, + "九条裟罗 Kujou Sara (Genshin Impact)": 92, + "芭芭拉 Barbara (Genshin Impact)": 93, + "派蒙 Paimon (Genshin Impact)": 94, + "荒泷一斗 Arataki Itto (Genshin Impact)": 96, + "早柚 Sayu (Genshin Impact)": 97, + "香菱 Xiangling (Genshin Impact)": 98, + "神里绫华 Kamisato Ayaka (Genshin Impact)": 99, + "重云 Chongyun (Genshin Impact)": 100, + "流浪者 Wanderer (Genshin Impact)": 102, + "优菈 Eula (Genshin Impact)": 103, + "凝光 Ningguang (Genshin Impact)": 105, + "钟离 Zhongli (Genshin Impact)": 106, + "雷电将军 Raiden Shogun (Genshin Impact)": 107, + "枫原万叶 Kaedehara Kazuha (Genshin Impact)": 108, + "赛诺 Cyno (Genshin Impact)": 109, + "诺艾尔 Noelle (Genshin Impact)": 112, + "八重神子 Yae Miko (Genshin Impact)": 113, + "凯亚 Kaeya (Genshin Impact)": 114, + "魈 Xiao (Genshin Impact)": 115, + "托马 Thoma (Genshin Impact)": 116, + "可莉 Klee (Genshin Impact)": 117, + "迪卢克 Diluc (Genshin Impact)": 120, + "夜兰 Yelan (Genshin Impact)": 121, + "鹿野院平藏 Shikanoin Heizou (Genshin Impact)": 123, + "辛焱 Xinyan (Genshin Impact)": 124, + "丽莎 Lisa (Genshin Impact)": 125, + "云堇 Yun Jin (Genshin Impact)": 126, + "坎蒂丝 Candace (Genshin Impact)": 127, + "罗莎莉亚 Rosaria (Genshin Impact)": 128, + "北斗 Beidou (Genshin Impact)": 129, + "珊瑚宫心海 Sangonomiya Kokomi (Genshin Impact)": 132, + "烟绯 Yanfei (Genshin Impact)": 133, + "久岐忍 Kuki Shinobu (Genshin Impact)": 136, + "宵宫 Yoimiya (Genshin Impact)": 139, + "安柏 Amber (Genshin Impact)": 143, + "迪奥娜 Diona (Genshin Impact)": 144, + "班尼特 Bennett (Genshin Impact)": 146, + "雷泽 Razor (Genshin Impact)": 147, + "阿贝多 Albedo (Genshin Impact)": 151, + "温迪 Venti (Genshin Impact)": 152, + "空 Player Male (Genshin Impact)": 153, + "神里绫人 Kamisato Ayato (Genshin Impact)": 154, + "琴 Jean (Genshin Impact)": 155, + "艾尔海森 Alhaitham (Genshin Impact)": 156, + "莫娜 Mona (Genshin Impact)": 157, + "妮露 Nilou (Genshin Impact)": 159, + "胡桃 Hu Tao (Genshin Impact)": 160, + "甘雨 Ganyu (Genshin Impact)": 161, + "纳西妲 Nahida (Genshin Impact)": 162, + "刻晴 Keqing (Genshin Impact)": 165, + "荧 Player Female (Genshin Impact)": 169, + "埃洛伊 Aloy (Genshin Impact)": 179, + "柯莱 Collei (Genshin Impact)": 182, + "多莉 Dori (Genshin Impact)": 184, + "提纳里 Tighnari (Genshin Impact)": 186, + "砂糖 Sucrose (Genshin Impact)": 188, + "行秋 Xingqiu (Genshin Impact)": 190, + "奥兹 Oz (Genshin Impact)": 193, + "五郎 Gorou (Genshin Impact)": 198, + "达达利亚 Tartalia (Genshin Impact)": 202, + "七七 Qiqi (Genshin Impact)": 207, + "申鹤 Shenhe (Genshin Impact)": 217, + "莱依拉 Layla (Genshin Impact)": 228, + "菲谢尔 Fishl (Genshin Impact)": 230, + "User": 999 + } + } \ No newline at end of file diff --git a/finetune_speaker.py b/finetune_speaker.py index d727353..2641d60 100644 --- a/finetune_speaker.py +++ b/finetune_speaker.py @@ -240,8 +240,10 @@ def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler, loade if global_step % hps.train.eval_interval == 0: evaluate(hps, net_g, eval_loader, writer_eval) utils.save_checkpoint(net_g, None, hps.train.learning_rate, epoch, os.path.join(hps.model_dir, "G_{}.pth".format(global_step))) + utils.save_checkpoint(net_g, None, hps.train.learning_rate, epoch, + os.path.join(hps.model_dir, "G_latest.pth".format(global_step))) # utils.save_checkpoint(net_d, optim_d, hps.train.learning_rate, epoch, os.path.join(hps.model_dir, "D_{}.pth".format(global_step))) - old_g=os.path.join(hps.model_dir, "G_{}.pth".format(global_step-4000)) + old_g=os.path.join(hps.model_dir, "G_{}.pth".format(global_step-400)) # old_d=os.path.join(hps.model_dir, "D_{}.pth".format(global_step-400)) if os.path.exists(old_g): os.remove(old_g) diff --git a/models_infer.py b/models_infer.py new file mode 100644 index 0000000..8f5e0e4 --- /dev/null +++ b/models_infer.py @@ -0,0 +1,401 @@ +import math +import torch +from torch import nn +from torch.nn import functional as F + +import commons +import modules +import attentions + +from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d +from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm +from commons import init_weights, get_padding + + +class StochasticDurationPredictor(nn.Module): + def __init__(self, in_channels, filter_channels, kernel_size, p_dropout, n_flows=4, gin_channels=0): + super().__init__() + filter_channels = in_channels # it needs to be removed from future version. + self.in_channels = in_channels + self.filter_channels = filter_channels + self.kernel_size = kernel_size + self.p_dropout = p_dropout + self.n_flows = n_flows + self.gin_channels = gin_channels + + self.log_flow = modules.Log() + self.flows = nn.ModuleList() + self.flows.append(modules.ElementwiseAffine(2)) + for i in range(n_flows): + self.flows.append(modules.ConvFlow(2, filter_channels, kernel_size, n_layers=3)) + self.flows.append(modules.Flip()) + + self.post_pre = nn.Conv1d(1, filter_channels, 1) + self.post_proj = nn.Conv1d(filter_channels, filter_channels, 1) + self.post_convs = modules.DDSConv(filter_channels, kernel_size, n_layers=3, p_dropout=p_dropout) + self.post_flows = nn.ModuleList() + self.post_flows.append(modules.ElementwiseAffine(2)) + for i in range(4): + self.post_flows.append(modules.ConvFlow(2, filter_channels, kernel_size, n_layers=3)) + self.post_flows.append(modules.Flip()) + + self.pre = nn.Conv1d(in_channels, filter_channels, 1) + self.proj = nn.Conv1d(filter_channels, filter_channels, 1) + self.convs = modules.DDSConv(filter_channels, kernel_size, n_layers=3, p_dropout=p_dropout) + if gin_channels != 0: + self.cond = nn.Conv1d(gin_channels, filter_channels, 1) + + def forward(self, x, x_mask, w=None, g=None, reverse=False, noise_scale=1.0): + x = torch.detach(x) + x = self.pre(x) + if g is not None: + g = torch.detach(g) + x = x + self.cond(g) + x = self.convs(x, x_mask) + x = self.proj(x) * x_mask + + if not reverse: + flows = self.flows + assert w is not None + + logdet_tot_q = 0 + h_w = self.post_pre(w) + h_w = self.post_convs(h_w, x_mask) + h_w = self.post_proj(h_w) * x_mask + e_q = torch.randn(w.size(0), 2, w.size(2)).to(device=x.device, dtype=x.dtype) * x_mask + z_q = e_q + for flow in self.post_flows: + z_q, logdet_q = flow(z_q, x_mask, g=(x + h_w)) + logdet_tot_q += logdet_q + z_u, z1 = torch.split(z_q, [1, 1], 1) + u = torch.sigmoid(z_u) * x_mask + z0 = (w - u) * x_mask + logdet_tot_q += torch.sum((F.logsigmoid(z_u) + F.logsigmoid(-z_u)) * x_mask, [1,2]) + logq = torch.sum(-0.5 * (math.log(2*math.pi) + (e_q**2)) * x_mask, [1,2]) - logdet_tot_q + + logdet_tot = 0 + z0, logdet = self.log_flow(z0, x_mask) + logdet_tot += logdet + z = torch.cat([z0, z1], 1) + for flow in flows: + z, logdet = flow(z, x_mask, g=x, reverse=reverse) + logdet_tot = logdet_tot + logdet + nll = torch.sum(0.5 * (math.log(2*math.pi) + (z**2)) * x_mask, [1,2]) - logdet_tot + return nll + logq # [b] + else: + flows = list(reversed(self.flows)) + flows = flows[:-2] + [flows[-1]] # remove a useless vflow + z = torch.randn(x.size(0), 2, x.size(2)).to(device=x.device, dtype=x.dtype) * noise_scale + for flow in flows: + z = flow(z, x_mask, g=x, reverse=reverse) + z0, z1 = torch.split(z, [1, 1], 1) + logw = z0 + return logw + + +class DurationPredictor(nn.Module): + def __init__(self, in_channels, filter_channels, kernel_size, p_dropout, gin_channels=0): + super().__init__() + + self.in_channels = in_channels + self.filter_channels = filter_channels + self.kernel_size = kernel_size + self.p_dropout = p_dropout + self.gin_channels = gin_channels + + self.drop = nn.Dropout(p_dropout) + self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size, padding=kernel_size//2) + self.norm_1 = modules.LayerNorm(filter_channels) + self.conv_2 = nn.Conv1d(filter_channels, filter_channels, kernel_size, padding=kernel_size//2) + self.norm_2 = modules.LayerNorm(filter_channels) + self.proj = nn.Conv1d(filter_channels, 1, 1) + + if gin_channels != 0: + self.cond = nn.Conv1d(gin_channels, in_channels, 1) + + def forward(self, x, x_mask, g=None): + x = torch.detach(x) + if g is not None: + g = torch.detach(g) + x = x + self.cond(g) + x = self.conv_1(x * x_mask) + x = torch.relu(x) + x = self.norm_1(x) + x = self.drop(x) + x = self.conv_2(x * x_mask) + x = torch.relu(x) + x = self.norm_2(x) + x = self.drop(x) + x = self.proj(x * x_mask) + return x * x_mask + + +class TextEncoder(nn.Module): + def __init__(self, + n_vocab, + out_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout): + super().__init__() + self.n_vocab = n_vocab + self.out_channels = out_channels + self.hidden_channels = hidden_channels + self.filter_channels = filter_channels + self.n_heads = n_heads + self.n_layers = n_layers + self.kernel_size = kernel_size + self.p_dropout = p_dropout + + self.emb = nn.Embedding(n_vocab, hidden_channels) + nn.init.normal_(self.emb.weight, 0.0, hidden_channels**-0.5) + + self.encoder = attentions.Encoder( + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout) + self.proj= nn.Conv1d(hidden_channels, out_channels * 2, 1) + + def forward(self, x, x_lengths): + x = self.emb(x) * math.sqrt(self.hidden_channels) # [b, t, h] + x = torch.transpose(x, 1, -1) # [b, h, t] + x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(x.dtype) + + x = self.encoder(x * x_mask, x_mask) + stats = self.proj(x) * x_mask + + m, logs = torch.split(stats, self.out_channels, dim=1) + return x, m, logs, x_mask + + +class ResidualCouplingBlock(nn.Module): + def __init__(self, + channels, + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + n_flows=4, + gin_channels=0): + super().__init__() + self.channels = channels + self.hidden_channels = hidden_channels + self.kernel_size = kernel_size + self.dilation_rate = dilation_rate + self.n_layers = n_layers + self.n_flows = n_flows + self.gin_channels = gin_channels + + self.flows = nn.ModuleList() + for i in range(n_flows): + self.flows.append(modules.ResidualCouplingLayer(channels, hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=gin_channels, mean_only=True)) + self.flows.append(modules.Flip()) + + def forward(self, x, x_mask, g=None, reverse=False): + if not reverse: + for flow in self.flows: + x, _ = flow(x, x_mask, g=g, reverse=reverse) + else: + for flow in reversed(self.flows): + x = flow(x, x_mask, g=g, reverse=reverse) + return x + + +class PosteriorEncoder(nn.Module): + def __init__(self, + in_channels, + out_channels, + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + gin_channels=0): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.hidden_channels = hidden_channels + self.kernel_size = kernel_size + self.dilation_rate = dilation_rate + self.n_layers = n_layers + self.gin_channels = gin_channels + + self.pre = nn.Conv1d(in_channels, hidden_channels, 1) + self.enc = modules.WN(hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=gin_channels) + self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1) + + def forward(self, x, x_lengths, g=None): + x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(x.dtype) + x = self.pre(x) * x_mask + x = self.enc(x, x_mask, g=g) + stats = self.proj(x) * x_mask + m, logs = torch.split(stats, self.out_channels, dim=1) + z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask + return z, m, logs, x_mask + + +class Generator(torch.nn.Module): + def __init__(self, initial_channel, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates, upsample_initial_channel, upsample_kernel_sizes, gin_channels=0): + super(Generator, self).__init__() + self.num_kernels = len(resblock_kernel_sizes) + self.num_upsamples = len(upsample_rates) + self.conv_pre = Conv1d(initial_channel, upsample_initial_channel, 7, 1, padding=3) + resblock = modules.ResBlock1 if resblock == '1' else modules.ResBlock2 + + self.ups = nn.ModuleList() + for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)): + self.ups.append(weight_norm( + ConvTranspose1d(upsample_initial_channel//(2**i), upsample_initial_channel//(2**(i+1)), + k, u, padding=(k-u)//2))) + + self.resblocks = nn.ModuleList() + for i in range(len(self.ups)): + ch = upsample_initial_channel//(2**(i+1)) + for j, (k, d) in enumerate(zip(resblock_kernel_sizes, resblock_dilation_sizes)): + self.resblocks.append(resblock(ch, k, d)) + + self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False) + self.ups.apply(init_weights) + + if gin_channels != 0: + self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1) + + def forward(self, x, g=None): + x = self.conv_pre(x) + if g is not None: + x = x + self.cond(g) + + for i in range(self.num_upsamples): + x = F.leaky_relu(x, modules.LRELU_SLOPE) + x = self.ups[i](x) + xs = None + for j in range(self.num_kernels): + if xs is None: + xs = self.resblocks[i*self.num_kernels+j](x) + else: + xs += self.resblocks[i*self.num_kernels+j](x) + x = xs / self.num_kernels + x = F.leaky_relu(x) + x = self.conv_post(x) + x = torch.tanh(x) + + return x + + def remove_weight_norm(self): + print('Removing weight norm...') + for l in self.ups: + remove_weight_norm(l) + for l in self.resblocks: + l.remove_weight_norm() + + + +class SynthesizerTrn(nn.Module): + """ + Synthesizer for Training + """ + + def __init__(self, + n_vocab, + spec_channels, + segment_size, + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + n_speakers=0, + gin_channels=0, + use_sdp=True, + **kwargs): + + super().__init__() + self.n_vocab = n_vocab + self.spec_channels = spec_channels + self.inter_channels = inter_channels + self.hidden_channels = hidden_channels + self.filter_channels = filter_channels + self.n_heads = n_heads + self.n_layers = n_layers + self.kernel_size = kernel_size + self.p_dropout = p_dropout + self.resblock = resblock + self.resblock_kernel_sizes = resblock_kernel_sizes + self.resblock_dilation_sizes = resblock_dilation_sizes + self.upsample_rates = upsample_rates + self.upsample_initial_channel = upsample_initial_channel + self.upsample_kernel_sizes = upsample_kernel_sizes + self.segment_size = segment_size + self.n_speakers = n_speakers + self.gin_channels = gin_channels + + self.use_sdp = use_sdp + + self.enc_p = TextEncoder(n_vocab, + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout) + self.dec = Generator(inter_channels, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates, upsample_initial_channel, upsample_kernel_sizes, gin_channels=gin_channels) + self.enc_q = PosteriorEncoder(spec_channels, inter_channels, hidden_channels, 5, 1, 16, gin_channels=gin_channels) + self.flow = ResidualCouplingBlock(inter_channels, hidden_channels, 5, 1, 4, gin_channels=gin_channels) + + if use_sdp: + self.dp = StochasticDurationPredictor(hidden_channels, 192, 3, 0.5, 4, gin_channels=gin_channels) + else: + self.dp = DurationPredictor(hidden_channels, 256, 3, 0.5, gin_channels=gin_channels) + + if n_speakers > 1: + self.emb_g = nn.Embedding(n_speakers, gin_channels) + + def infer(self, x, x_lengths, sid=None, noise_scale=1, length_scale=1, noise_scale_w=1., max_len=None): + x, m_p, logs_p, x_mask = self.enc_p(x, x_lengths) + if self.n_speakers > 0: + g = self.emb_g(sid).unsqueeze(-1) # [b, h, 1] + else: + g = None + + if self.use_sdp: + logw = self.dp(x, x_mask, g=g, reverse=True, noise_scale=noise_scale_w) + else: + logw = self.dp(x, x_mask, g=g) + w = torch.exp(logw) * x_mask * length_scale + w_ceil = torch.ceil(w) + y_lengths = torch.clamp_min(torch.sum(w_ceil, [1, 2]), 1).long() + y_mask = torch.unsqueeze(commons.sequence_mask(y_lengths, None), 1).to(x_mask.dtype) + attn_mask = torch.unsqueeze(x_mask, 2) * torch.unsqueeze(y_mask, -1) + attn = commons.generate_path(w_ceil, attn_mask) + + m_p = torch.matmul(attn.squeeze(1), m_p.transpose(1, 2)).transpose(1, 2) # [b, t', t], [b, t, d] -> [b, d, t'] + logs_p = torch.matmul(attn.squeeze(1), logs_p.transpose(1, 2)).transpose(1, 2) # [b, t', t], [b, t, d] -> [b, d, t'] + + z_p = m_p + torch.randn_like(m_p) * torch.exp(logs_p) * noise_scale + z = self.flow(z_p, y_mask, g=g, reverse=True) + o = self.dec((z * y_mask)[:,:,:max_len], g=g) + return o, attn, y_mask, (z, z_p, m_p, logs_p) + + def voice_conversion(self, y, y_lengths, sid_src, sid_tgt): + assert self.n_speakers > 0, "n_speakers have to be larger than 0." + g_src = self.emb_g(sid_src).unsqueeze(-1) + g_tgt = self.emb_g(sid_tgt).unsqueeze(-1) + z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g_src) + z_p = self.flow(z, y_mask, g=g_src) + z_hat = self.flow(z_p, y_mask, g=g_tgt, reverse=True) + o_hat = self.dec(z_hat * y_mask, g=g_tgt) + return o_hat, y_mask, (z, z_p, z_hat) \ No newline at end of file diff --git a/utils.py b/utils.py index 181e746..4d8e261 100644 --- a/utils.py +++ b/utils.py @@ -188,7 +188,7 @@ def get_hparams_from_dir(model_dir): def get_hparams_from_file(config_path): - with open(config_path, "r") as f: + with open(config_path, "r", encoding="utf-8") as f: data = f.read() config = json.loads(data)