diff --git a/configs/modified_finetune_speaker.json b/configs/modified_finetune_speaker.json new file mode 100644 index 0000000..197afc6 --- /dev/null +++ b/configs/modified_finetune_speaker.json @@ -0,0 +1,204 @@ +{ + "train": { + "log_interval": 100, + "eval_interval": 1000, + "seed": 1234, + "epochs": 10000, + "learning_rate": 2e-4, + "betas": [0.8, 0.99], + "eps": 1e-9, + "batch_size": 16, + "fp16_run": true, + "lr_decay": 0.999875, + "segment_size": 8192, + "init_lr_ratio": 1, + "warmup_epochs": 0, + "c_mel": 45, + "c_kl": 1.0 + }, + "data": { + "training_files":"final_annotation_train.txt", + "validation_files":"final_annotation_val.txt", + "text_cleaners":["cjke_cleaners2"], + "max_wav_value": 32768.0, + "sampling_rate": 22050, + "filter_length": 1024, + "hop_length": 256, + "win_length": 1024, + "n_mel_channels": 80, + "mel_fmin": 0.0, + "mel_fmax": null, + "add_blank": true, + "n_speakers": 1001, + "cleaned_text": true + }, + "model": { + "inter_channels": 192, + "hidden_channels": 192, + "filter_channels": 768, + "n_heads": 2, + "n_layers": 6, + "kernel_size": 3, + "p_dropout": 0.1, + "resblock": "1", + "resblock_kernel_sizes": [3,7,11], + "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], + "upsample_rates": [8,8,2,2], + "upsample_initial_channel": 512, + "upsample_kernel_sizes": [16,16,4,4], + "n_layers_q": 3, + "use_spectral_norm": false, + "gin_channels": 256 + }, + "symbols": ["_", ",", ".", "!", "?", "-", "~", "\u2026", "N", "Q", "a", "b", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "s", "t", "u", "v", "w", "x", "y", "z", "\u0251", "\u00e6", "\u0283", "\u0291", "\u00e7", "\u026f", "\u026a", "\u0254", "\u025b", "\u0279", "\u00f0", "\u0259", "\u026b", "\u0265", "\u0278", "\u028a", "\u027e", "\u0292", "\u03b8", "\u03b2", "\u014b", "\u0266", "\u207c", "\u02b0", "`", "^", "#", "*", "=", "\u02c8", "\u02cc", "\u2192", "\u2193", "\u2191", " "], + "speakers": {"特别周 Special Week (Umamusume Pretty Derby)": 0, + "无声铃鹿 Silence Suzuka (Umamusume Pretty Derby)": 1, + "东海帝王 Tokai Teio (Umamusume Pretty Derby)": 2, + "丸善斯基 Maruzensky (Umamusume Pretty Derby)": 3, + "富士奇迹 Fuji Kiseki (Umamusume Pretty Derby)": 4, + "小栗帽 Oguri Cap (Umamusume Pretty Derby)": 5, + "黄金船 Gold Ship (Umamusume Pretty Derby)": 6, + "伏特加 Vodka (Umamusume Pretty Derby)": 7, + "大和赤骥 Daiwa Scarlet (Umamusume Pretty Derby)": 8, + "大树快车 Taiki Shuttle (Umamusume Pretty Derby)": 9, + "草上飞 Grass Wonder (Umamusume Pretty Derby)": 10, + "菱亚马逊 Hishi Amazon (Umamusume Pretty Derby)": 11, + "目白麦昆 Mejiro Mcqueen (Umamusume Pretty Derby)": 12, + "神鹰 El Condor Pasa (Umamusume Pretty Derby)": 13, + "好歌剧 T.M. Opera O (Umamusume Pretty Derby)": 14, + "成田白仁 Narita Brian (Umamusume Pretty Derby)": 15, + "鲁道夫象征 Symboli Rudolf (Umamusume Pretty Derby)": 16, + "气槽 Air Groove (Umamusume Pretty Derby)": 17, + "爱丽数码 Agnes Digital (Umamusume Pretty Derby)": 18, + "青云天空 Seiun Sky (Umamusume Pretty Derby)": 19, + "玉藻十字 Tamamo Cross (Umamusume Pretty Derby)": 20, + "美妙姿势 Fine Motion (Umamusume Pretty Derby)": 21, + "琵琶晨光 Biwa Hayahide (Umamusume Pretty Derby)": 22, + "重炮 Mayano Topgun (Umamusume Pretty Derby)": 23, + "曼城茶座 Manhattan Cafe (Umamusume Pretty Derby)": 24, + "美普波旁 Mihono Bourbon (Umamusume Pretty Derby)": 25, + "目白雷恩 Mejiro Ryan (Umamusume Pretty Derby)": 26, + "雪之美人 Yukino Bijin (Umamusume Pretty Derby)": 28, + "米浴 Rice Shower (Umamusume Pretty Derby)": 29, + "艾尼斯风神 Ines Fujin (Umamusume Pretty Derby)": 30, + "爱丽速子 Agnes Tachyon (Umamusume Pretty Derby)": 31, + "爱慕织姬 Admire Vega (Umamusume Pretty Derby)": 32, + "稻荷一 Inari One (Umamusume Pretty Derby)": 33, + "胜利奖券 Winning Ticket (Umamusume Pretty Derby)": 34, + "空中神宫 Air Shakur (Umamusume Pretty Derby)": 35, + "荣进闪耀 Eishin Flash (Umamusume Pretty Derby)": 36, + "真机伶 Curren Chan (Umamusume Pretty Derby)": 37, + "川上公主 Kawakami Princess (Umamusume Pretty Derby)": 38, + "黄金城市 Gold City (Umamusume Pretty Derby)": 39, + "樱花进王 Sakura Bakushin O (Umamusume Pretty Derby)": 40, + "采珠 Seeking the Pearl (Umamusume Pretty Derby)": 41, + "新光风 Shinko Windy (Umamusume Pretty Derby)": 42, + "东商变革 Sweep Tosho (Umamusume Pretty Derby)": 43, + "超级小溪 Super Creek (Umamusume Pretty Derby)": 44, + "醒目飞鹰 Smart Falcon (Umamusume Pretty Derby)": 45, + "荒漠英雄 Zenno Rob Roy (Umamusume Pretty Derby)": 46, + "东瀛佐敦 Tosen Jordan (Umamusume Pretty Derby)": 47, + "中山庆典 Nakayama Festa (Umamusume Pretty Derby)": 48, + "成田大进 Narita Taishin (Umamusume Pretty Derby)": 49, + "西野花 Nishino Flower (Umamusume Pretty Derby)": 50, + "春乌拉拉 Haru Urara (Umamusume Pretty Derby)": 51, + "青竹回忆 Bamboo Memory (Umamusume Pretty Derby)": 52, + "待兼福来 Matikane Fukukitaru (Umamusume Pretty Derby)": 55, + "名将怒涛 Meisho Doto (Umamusume Pretty Derby)": 57, + "目白多伯 Mejiro Dober (Umamusume Pretty Derby)": 58, + "优秀素质 Nice Nature (Umamusume Pretty Derby)": 59, + "帝王光环 King Halo (Umamusume Pretty Derby)": 60, + "待兼诗歌剧 Matikane Tannhauser (Umamusume Pretty Derby)": 61, + "生野狄杜斯 Ikuno Dictus (Umamusume Pretty Derby)": 62, + "目白善信 Mejiro Palmer (Umamusume Pretty Derby)": 63, + "大拓太阳神 Daitaku Helios (Umamusume Pretty Derby)": 64, + "双涡轮 Twin Turbo (Umamusume Pretty Derby)": 65, + "里见光钻 Satono Diamond (Umamusume Pretty Derby)": 66, + "北部玄驹 Kitasan Black (Umamusume Pretty Derby)": 67, + "樱花千代王 Sakura Chiyono O (Umamusume Pretty Derby)": 68, + "天狼星象征 Sirius Symboli (Umamusume Pretty Derby)": 69, + "目白阿尔丹 Mejiro Ardan (Umamusume Pretty Derby)": 70, + "八重无敌 Yaeno Muteki (Umamusume Pretty Derby)": 71, + "鹤丸刚志 Tsurumaru Tsuyoshi (Umamusume Pretty Derby)": 72, + "目白光明 Mejiro Bright (Umamusume Pretty Derby)": 73, + "樱花桂冠 Sakura Laurel (Umamusume Pretty Derby)": 74, + "成田路 Narita Top Road (Umamusume Pretty Derby)": 75, + "也文摄辉 Yamanin Zephyr (Umamusume Pretty Derby)": 76, + "真弓快车 Aston Machan (Umamusume Pretty Derby)": 80, + "骏川手纲 Hayakawa Tazuna (Umamusume Pretty Derby)": 81, + "小林历奇 Kopano Rickey (Umamusume Pretty Derby)": 83, + "奇锐骏 Wonder Acute (Umamusume Pretty Derby)": 85, + "秋川理事长 President Akikawa (Umamusume Pretty Derby)": 86, + "綾地 寧々 Ayachi Nene (Sanoba Witch)": 87, + "因幡 めぐる Inaba Meguru (Sanoba Witch)": 88, + "椎葉 紬 Shiiba Tsumugi (Sanoba Witch)": 89, + "仮屋 和奏 Kariya Wakama (Sanoba Witch)": 90, + "戸隠 憧子 Togakushi Touko (Sanoba Witch)": 91, + "九条裟罗 Kujou Sara (Genshin Impact)": 92, + "芭芭拉 Barbara (Genshin Impact)": 93, + "派蒙 Paimon (Genshin Impact)": 94, + "荒泷一斗 Arataki Itto (Genshin Impact)": 96, + "早柚 Sayu (Genshin Impact)": 97, + "香菱 Xiangling (Genshin Impact)": 98, + "神里绫华 Kamisato Ayaka (Genshin Impact)": 99, + "重云 Chongyun (Genshin Impact)": 100, + "流浪者 Wanderer (Genshin Impact)": 102, + "优菈 Eula (Genshin Impact)": 103, + "凝光 Ningguang (Genshin Impact)": 105, + "钟离 Zhongli (Genshin Impact)": 106, + "雷电将军 Raiden Shogun (Genshin Impact)": 107, + "枫原万叶 Kaedehara Kazuha (Genshin Impact)": 108, + "赛诺 Cyno (Genshin Impact)": 109, + "诺艾尔 Noelle (Genshin Impact)": 112, + "八重神子 Yae Miko (Genshin Impact)": 113, + "凯亚 Kaeya (Genshin Impact)": 114, + "魈 Xiao (Genshin Impact)": 115, + "托马 Thoma (Genshin Impact)": 116, + "可莉 Klee (Genshin Impact)": 117, + "迪卢克 Diluc (Genshin Impact)": 120, + "夜兰 Yelan (Genshin Impact)": 121, + "鹿野院平藏 Shikanoin Heizou (Genshin Impact)": 123, + "辛焱 Xinyan (Genshin Impact)": 124, + "丽莎 Lisa (Genshin Impact)": 125, + "云堇 Yun Jin (Genshin Impact)": 126, + "坎蒂丝 Candace (Genshin Impact)": 127, + "罗莎莉亚 Rosaria (Genshin Impact)": 128, + "北斗 Beidou (Genshin Impact)": 129, + "珊瑚宫心海 Sangonomiya Kokomi (Genshin Impact)": 132, + "烟绯 Yanfei (Genshin Impact)": 133, + "久岐忍 Kuki Shinobu (Genshin Impact)": 136, + "宵宫 Yoimiya (Genshin Impact)": 139, + "安柏 Amber (Genshin Impact)": 143, + "迪奥娜 Diona (Genshin Impact)": 144, + "班尼特 Bennett (Genshin Impact)": 146, + "雷泽 Razor (Genshin Impact)": 147, + "阿贝多 Albedo (Genshin Impact)": 151, + "温迪 Venti (Genshin Impact)": 152, + "空 Player Male (Genshin Impact)": 153, + "神里绫人 Kamisato Ayato (Genshin Impact)": 154, + "琴 Jean (Genshin Impact)": 155, + "艾尔海森 Alhaitham (Genshin Impact)": 156, + "莫娜 Mona (Genshin Impact)": 157, + "妮露 Nilou (Genshin Impact)": 159, + "胡桃 Hu Tao (Genshin Impact)": 160, + "甘雨 Ganyu (Genshin Impact)": 161, + "纳西妲 Nahida (Genshin Impact)": 162, + "刻晴 Keqing (Genshin Impact)": 165, + "荧 Player Female (Genshin Impact)": 169, + "埃洛伊 Aloy (Genshin Impact)": 179, + "柯莱 Collei (Genshin Impact)": 182, + "多莉 Dori (Genshin Impact)": 184, + "提纳里 Tighnari (Genshin Impact)": 186, + "砂糖 Sucrose (Genshin Impact)": 188, + "行秋 Xingqiu (Genshin Impact)": 190, + "奥兹 Oz (Genshin Impact)": 193, + "五郎 Gorou (Genshin Impact)": 198, + "达达利亚 Tartalia (Genshin Impact)": 202, + "七七 Qiqi (Genshin Impact)": 207, + "申鹤 Shenhe (Genshin Impact)": 217, + "莱依拉 Layla (Genshin Impact)": 228, + "菲谢尔 Fishl (Genshin Impact)": 230, + "User": 999 + } + +} \ No newline at end of file diff --git a/demucs_denoise.py b/demucs_denoise.py index 33ef76e..718ca71 100644 --- a/demucs_denoise.py +++ b/demucs_denoise.py @@ -1,23 +1,31 @@ import os import torch import torchaudio +import argparse +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--denoise_user', type=bool, default=False, + help='whether to denoise user recorded voice') + parser.add_argument('--denoise_character', type=bool, default=False, + help='whether to denoise uploaded character voice') + args = parser.parse_args() + if args.denoise_user: + audio_dir = "./user_voice/" + wavfiles = [] + for filename in list(os.walk(audio_dir))[0][2]: + if filename.endswith(".wav"): + wavfiles.append(filename) -audio_dir = "./user_voice/" -wavfiles = [] -for filename in list(os.walk(audio_dir))[0][2]: - if filename.endswith(".wav"): - wavfiles.append(filename) + # denoise with demucs + for i, wavfile in enumerate(wavfiles): + os.system(f"demucs --two-stems=vocals {audio_dir}{wavfile}") -# denoise with demucs -for i, wavfile in enumerate(wavfiles): - os.system(f"demucs --two-stems=vocals {audio_dir}{wavfile}") - -# read & store the denoised vocals back -for wavfile in wavfiles: - i = wavfile.strip(".wav") - wav, sr = torchaudio.load(f"./separated/htdemucs/{i}/vocals.wav", frame_offset=0, num_frames=-1, normalize=True, channels_first=True) - # merge two channels into one - wav = wav.mean(dim=0).unsqueeze(0) - if sr != 22050: - wav = torchaudio.transforms.Resample(orig_freq=sr, new_freq=22050)(wav) - torchaudio.save(f"./user_voice/{i}.wav", wav, 22050, channels_first=True) \ No newline at end of file + # read & store the denoised vocals back + for wavfile in wavfiles: + i = wavfile.strip(".wav") + wav, sr = torchaudio.load(f"./separated/htdemucs/{i}/vocals.wav", frame_offset=0, num_frames=-1, normalize=True, channels_first=True) + # merge two channels into one + wav = wav.mean(dim=0).unsqueeze(0) + if sr != 22050: + wav = torchaudio.transforms.Resample(orig_freq=sr, new_freq=22050)(wav) + torchaudio.save(f"./user_voice/{i}.wav", wav, 22050, channels_first=True) \ No newline at end of file diff --git a/finetune_speaker.py b/finetune_speaker.py index d9294e5..cf14923 100644 --- a/finetune_speaker.py +++ b/finetune_speaker.py @@ -150,7 +150,7 @@ def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler, loade net_g.train() net_d.train() - for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths, speakers) in enumerate(tqdm(train_loader)): + for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths, speakers) in enumerate(train_loader): x, x_lengths = x.cuda(rank, non_blocking=True), x_lengths.cuda(rank, non_blocking=True) spec, spec_lengths = spec.cuda(rank, non_blocking=True), spec_lengths.cuda(rank, non_blocking=True) y, y_lengths = y.cuda(rank, non_blocking=True), y_lengths.cuda(rank, non_blocking=True) @@ -250,7 +250,8 @@ def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler, loade # if os.path.exists(old_d): # os.remove(old_d) global_step += 1 - if global_step == hps.n_steps + 1: + if epoch > hps.max_epochs: + print("Maximum epoch reached, closing training...") exit() if rank == 0: diff --git a/preprocess.py b/preprocess.py index 5a20ccb..dae747a 100644 --- a/preprocess.py +++ b/preprocess.py @@ -1,5 +1,4 @@ import os -MIN_VOICE_NUM = 10 if __name__ == "__main__": # load sampled_audio4ft with open("sampled_audio4ft.txt", 'r', encoding='utf-8') as f: @@ -8,16 +7,27 @@ if __name__ == "__main__": # load user text with open("./user_voice/user_voice.txt.cleaned", 'r', encoding='utf-8') as f: user_annos = f.readlines() + # check how many voices are recorded wavfiles = [file for file in list(os.walk("./user_voice"))[0][2] if file.endswith(".wav")] num_user_voices = len(wavfiles) - if num_user_voices < MIN_VOICE_NUM: - raise Exception(f"You need to record at least {MIN_VOICE_NUM} voices for fine-tuning!") # user voices need to occupy 1/4 of the total dataset - duplicate = num_old_voices // num_user_voices // 3 + if num_user_voices: + user_duplicate = num_old_voices // num_user_voices // 3 + else: + user_duplicate = 0 + # find corresponding existing annotation lines actual_user_annos = ["./user_voice/" + line for line in user_annos if line.split("|")[0] in wavfiles] - final_annos = old_annos + actual_user_annos * duplicate + final_annos = old_annos + actual_user_annos * user_duplicate + + # load custom characters + with open("custom_character_anno.txt", 'r', encoding='utf-8') as f: + custom_character_anno = f.readlines() + # custom character voices need to be at least equal to number of sample_audio4ft + num_character_voices = len(custom_character_anno) + cc_duplicate = num_old_voices // num_character_voices + final_annos = final_annos + custom_character_anno * cc_duplicate # save annotation file with open("final_annotation_train.txt", 'w', encoding='utf-8') as f: for line in final_annos: @@ -26,3 +36,5 @@ if __name__ == "__main__": with open("final_annotation_val.txt", 'w', encoding='utf-8') as f: for line in actual_user_annos: f.write(line) + for line in custom_character_anno: + f.write(line) diff --git a/utils.py b/utils.py index 4550728..960a3b4 100644 --- a/utils.py +++ b/utils.py @@ -148,12 +148,12 @@ def load_filepaths_and_text(filename, split="|"): def get_hparams(init=True): parser = argparse.ArgumentParser() - parser.add_argument('-c', '--config', type=str, default="./configs/finetune_speaker.json", + parser.add_argument('-c', '--config', type=str, default="./configs/modified_finetune_speaker.json", help='JSON file for configuration') parser.add_argument('-m', '--model', type=str, default="pretrained_models", help='Model name') - parser.add_argument('-n', '--n_steps', type=int, default="2000", - help='finetune steps') + parser.add_argument('-n', '--max_epochs', type=int, default="50", + help='finetune epochs') args = parser.parse_args() model_dir = os.path.join("./", args.model) @@ -175,7 +175,7 @@ def get_hparams(init=True): hparams = HParams(**config) hparams.model_dir = model_dir - hparams.n_steps = args.n_steps + hparams.max_epochs = args.max_epochs return hparams diff --git a/voice_upload.py b/voice_upload.py new file mode 100644 index 0000000..839002f --- /dev/null +++ b/voice_upload.py @@ -0,0 +1,11 @@ +from google.colab import files +import shutil +import os +basepath = os.getcwd() +uploaded = files.upload() # 上传文件 +upload_path = "./custom_character_voice/" +if not os.path.exists(upload_path): + os.mkdir(upload_path) +for filename in uploaded.keys(): + #将上传的文件移动到指定的位置上 + shutil.move(os.path.join(basepath, filename), os.path.join(upload_path, filename)) diff --git a/whisper_transcribe.py b/whisper_transcribe.py new file mode 100644 index 0000000..cbcd5a1 --- /dev/null +++ b/whisper_transcribe.py @@ -0,0 +1,91 @@ +import whisper +import os +import torchaudio + +lang2token = { + 'zh': "[ZH]", + 'ja': "[JA]", + "en": "[EN]", +} + + + + +def transcribe_one(audio_path): + # load audio and pad/trim it to fit 30 seconds + audio = whisper.load_audio(audio_path) + audio = whisper.pad_or_trim(audio) + + # make log-Mel spectrogram and move to the same device as the model + mel = whisper.log_mel_spectrogram(audio).to(model.device) + + # detect the spoken language + _, probs = model.detect_language(mel) + print(f"Detected language: {max(probs, key=probs.get)}") + lang = max(probs, key=probs.get) + # decode the audio + options = whisper.DecodingOptions() + result = whisper.decode(model, mel, options) + + # print the recognized text + print(result.text) + return lang, result.text +if __name__ == "__main__": + model = whisper.load_model("medium") + parent_dir = "./custom_character_voice/" + speaker_names = list(os.walk(parent_dir))[0][1] + speaker2id = {} + speaker_annos = [] + # resample audios + for speaker in speaker_names: + speaker2id[speaker] = 1000 + len(speaker2id) + for i, wavfile in enumerate(list(os.walk(parent_dir + speaker))[0][2]): + # try to load file as audio + if wavfile.startswith("processed_"): + continue + try: + wav, sr = torchaudio.load(parent_dir + speaker + "/" + wavfile, frame_offset=0, num_frames=-1, normalize=True, + channels_first=True) + wav = wav.mean(dim=0).unsqueeze(0) + if sr != 22050: + wav = torchaudio.transforms.Resample(orig_freq=sr, new_freq=22050)(wav) + if wav.shape[1] / sr > 20: + print(f"{wavfile} too long, ignoring\n") + save_path = parent_dir + speaker + "/" + f"processed_{i}.wav" + torchaudio.save(save_path, wav, 22050, channels_first=True) + # transcribe text + lang, text = transcribe_one(save_path) + if lang not in ['zh', 'en', 'ja']: + print(f"{lang} not supported, ignoring\n") + text = lang2token[lang] + text + lang2token[lang] + "\n" + speaker_annos.append(save_path + "|" + str(speaker2id[speaker]) + "|" + text) + except: + continue + + # clean annotation + import argparse + import text + from utils import load_filepaths_and_text + for i, line in enumerate(speaker_annos): + path, sid, txt = line.split("|") + cleaned_text = text._clean_text(txt, ["cjke_cleaners2"]) + cleaned_text += "\n" if not cleaned_text.endswith("\n") else "" + speaker_annos[i] = path + "|" + sid + "|" + cleaned_text + # write into annotation + with open("custom_character_anno.txt", 'w', encoding='utf-8') as f: + for line in speaker_annos: + f.write(line) + + import json + # generate new config + with open("./configs/finetune_speaker.json", 'r', encoding='utf-8') as f: + hps = json.load(f) + # modify n_speakers + hps['data']["n_speakers"] = 999 + len(speaker2id) + # add speaker names + for speaker in speaker_names: + hps['speakers'][speaker] = speaker2id[speaker] + # save modified config + with open("./configs/modified_finetune_speaker.json", 'w', encoding='utf-8') as f: + json.dump(hps, f, indent=2) + print("finished")