From 5b228b39be901b7806b7969aa6f36f590e677236 Mon Sep 17 00:00:00 2001 From: Plachta Date: Sun, 26 Feb 2023 20:06:31 +0800 Subject: [PATCH] updated pipeline --- DATA.MD | 43 +++ README_ZH.md | 59 ++-- clean_character_anno.py | 19 ++ configs/modified_finetune_speaker.json | 429 +++++++++++++++---------- data_utils.py | 145 +-------- denoise_audio.py | 18 ++ download_model.py | 5 +- download_video.py | 23 ++ finetune_speaker_v2.py | 320 ++++++++++++++++++ long_audio_transcribe.py | 61 ++++ preprocess_v2.py | 127 ++++++++ rearrange_speaker.py | 33 ++ short_audio_transcribe.py | 100 ++++++ text/__init__.py | 5 +- utils.py | 8 +- video2audio.py | 10 + video_transcribe.py | 43 --- voice_upload.py | 36 ++- 18 files changed, 1083 insertions(+), 401 deletions(-) create mode 100644 DATA.MD create mode 100644 clean_character_anno.py create mode 100644 denoise_audio.py create mode 100644 download_video.py create mode 100644 finetune_speaker_v2.py create mode 100644 long_audio_transcribe.py create mode 100644 preprocess_v2.py create mode 100644 rearrange_speaker.py create mode 100644 short_audio_transcribe.py create mode 100644 video2audio.py delete mode 100644 video_transcribe.py diff --git a/DATA.MD b/DATA.MD new file mode 100644 index 0000000..58ea151 --- /dev/null +++ b/DATA.MD @@ -0,0 +1,43 @@ +本仓库的pipeline支持多种声音样本上传方式，您只需根据您所持有的样本选择任意一种或其中几种即可。 + +1.`.zip`文件打包的，按角色名排列的短音频，该压缩文件结构应如下所示： +``` +Your-zip-file.zip +├───Character_name_1 +├ ├───xxx.wav +├ ├───... +├ ├───yyy.mp3 +├ └───zzz.wav +├───Character_name_2 +├ ├───xxx.wav +├ ├───... +├ ├───yyy.mp3 +├ └───zzz.wav +├───... +├ +└───Character_name_n + ├───xxx.wav + ├───... + ├───yyy.mp3 + └───zzz.wav +``` +注意音频的格式和名称都不重要，只要它们是音频文件。 +质量要求：2秒以上，10秒以内，尽量不要有背景噪音。 +数量要求：一个角色至少10条，最好每个角色20条以上。 +2. 以角色名命名的长音频文件，音频内只能有单说话人，背景音会被自动去除。命名格式为：`{CharacterName}_{random_number}.wav` +(例如：`Diana_234135.wav`, `MinatoAqua_234252.wav`)，必须是`.wav`文件。 + + +3. 以角色名命名的长视频文件，视频内只能有单说话人，背景音会被自动去除。命名格式为：`{CharacterName}_{random_number}.mp4` +(例如：`Taffy_332452.mp4`, `Dingzhen_957315.mp4`)，必须是`.mp4`文件。 +注意：命名中，`CharacterName`必须是英文字符，`random_number`是为了区分同一个角色的多个文件，必须要添加，该数字可以为0~999999之间的任意整数。 + +4. 包含多行`{CharacterName}|{video_url}`的`.txt`文件，格式应如下所示： +``` +Char1|https://xyz.com/video1/ +Char2|https://xyz.com/video2/ +Char2|https://xyz.com/video3/ +Char3|https://xyz.com/video4/ +``` +视频内只能有单说话人，背景音会被自动去除。目前仅支持来自bilibili的视频，其它网站视频的url还没测试过。 +若对格式有疑问，可以在[这里](https://drive.google.com/file/d/132l97zjanpoPY4daLgqXoM7HKXPRbS84/view?usp=sharing)找到所有格式对应的数据样本。 \ No newline at end of file diff --git a/README_ZH.md b/README_ZH.md index 51d4fad..78b7ce1 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -1,17 +1,22 @@ English Documentation Please Click [here](https://github.com/Plachtaa/VITS-fast-fine-tuning/blob/main/README_EN.md) # VITS 快速微调 -这个代码库会指导你如何将自定义角色，甚至你自己的声线加入一个现有的VITS模型中，在1小时内的微调使模型具备如下功能： -1. 在 你 & 你加入的角色 & 预设角色 之间进行任意声线转换 -2. 以 你的声线 & 你加入的角色声线 & 预设角色声线 进行中日英三语 文本到语音合成。 +这个代码库会指导你如何将自定义角色（甚至你自己），加入预训练的VITS模型中，在1小时内的微调使模型具备如下功能： +1. 在 模型所包含的任意两个角色 之间进行声线转换 +2. 以 你加入的角色声线 进行中日英三语 文本到语音合成。 本项目使用的底模涵盖常见二次元男/女配音声线（来自原神数据集）以及现实世界常见男/女声线（来自VCTK数据集），支持中日英三语，保证能够在微调时快速适应新的声线。 -欢迎体验微调所使用的底模！ -[![Hugging Face Spaces](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue)](https://huggingface.co/spaces/Plachta/VITS-Umamusume-voice-synthesizer) +欢迎体验微调所使用的底模！ + +中日英：[![Hugging Face Spaces](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue)](https://huggingface.co/spaces/Plachta/VITS-Umamusume-voice-synthesizer) + +中日：[![Hugging Face Spaces](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue)](https://huggingface.co/spaces/sayashi/vits-uma-genshin-honkai) ### 目前支持的任务: -- [x] 转换用户声线到 [这些角色](https://github.com/SongtingLiu/VITS_voice_conversion/blob/main/configs/finetune_speaker.json) -- [x] 自定义角色的中日英三语TTS！ +- [x] 从 10条以上的短音频 克隆角色声音 +- [x] 从 3分钟以上的长音频 克隆角色声音 +- [x] 从 3分钟以上的视频（只包含单说话人） 克隆角色声音 +- [x] 通过输入 bilibili视频链接（只包含单说话人） 克隆角色声音 ### 目前支持声线转换和中日英三语TTS的角色 - [x] 赛马娘 （仅已实装角色）（预训练时使用的角色） @@ -26,36 +31,10 @@ English Documentation Please Click [here](https://github.com/Plachtaa/VITS-fast- 建议使用 [Google Colab](https://colab.research.google.com/drive/1omMhfYKrAAQ7a6zOCsyqpla-wU-QyfZn?usp=sharing) 进行微调任务，因为VITS在多语言情况下的某些环境依赖相当难以配置。 ### 在Google Colab里，我需要花多长时间？ -1. 安装依赖 (2 min) -2. 录入你自己的声音，阅读内容会在UI中提供，每句不超过20个字。 (5~10 min) -3. 上传你希望加入的其它角色声音，用一个`.zip`文件打包 -文件结构应该如下所示: -``` -Your-zip-file.zip -├───Character_name_1 -├ ├───xxx.wav -├ ├───... -├ ├───yyy.mp3 -├ └───zzz.wav -├───Character_name_2 -├ ├───xxx.wav -├ ├───... -├ ├───yyy.mp3 -├ └───zzz.wav -├───... -├ -└───Character_name_n - ├───xxx.wav - ├───... - ├───yyy.mp3 - └───zzz.wav -``` -注意音频的格式和名称都不重要，只要它们是音频文件。 -质量要求：2秒以上，20秒以内，尽量不要有背景噪音。 -数量要求：一个角色至少10条，最好每个角色20条以上。 -你可以选择进行步骤2或3，或二者一起，取决于你的需求。 - -4. 进行微调 (30 min) +1. 安装依赖 (3 min) +2. 选择预训练模型，详细区别参见Colab笔记本页面。 +3. 上传你希望加入的其它角色声音，详细上传方式见[DATA.MD]() +4. 进行微调，根据选择的微调方式和样本数量不同，花费时长可能在20分钟到2小时不等。 微调结束后可以直接下载微调好的模型，日后在本地运行（不需要GPU） @@ -64,7 +43,7 @@ Your-zip-file.zip 1. 下载最新的Release包（在Github页面的右侧） 2. 把下载的模型和config文件放在 `inference`文件夹下, 确保模型的文件名为 `G_latest.pth` ，config文件名为 `finetune_speaker.json` 3. 一切准备就绪后，文件结构应该如下所示: -```shell +``` inference ├───inference.exe ├───... @@ -73,3 +52,7 @@ inference ``` 4. 运行 `inference.exe`, 浏览器会自动弹出窗口, 注意其所在路径不能有中文字符或者空格. +## 在MoeGoe使用 +0. MoeGoe以及类似其它VITS推理UI使用的config格式略有不同，需要下载的文件为模型`G_latest.pth`和配置文件`moegoe_config.json` +1. 按照[MoeGoe](https://github.com/CjangCjengh/MoeGoe)页面的提示配置路径即可使用。 + diff --git a/clean_character_anno.py b/clean_character_anno.py new file mode 100644 index 0000000..240d62e --- /dev/null +++ b/clean_character_anno.py @@ -0,0 +1,19 @@ +import text + +with open("custom_character_anno.txt", 'r', encoding='utf-8') as f: + speaker_annos = f.readlines() +# clean annotation +cleaned_speaker_annos = [] +for i, line in enumerate(speaker_annos): + path, sid, txt = line.split("|") + if len(txt) > 100: + continue + cleaned_text = text._clean_text(txt, ["cjke_cleaners2"]) + cleaned_text += "\n" if not cleaned_text.endswith("\n") else "" + cleaned_speaker_annos.append(path + "|" + sid + "|" + cleaned_text) +# write into annotation +with open("custom_character_anno.txt", 'w', encoding='utf-8') as f: + for line in speaker_annos: + f.write(line) + + diff --git a/configs/modified_finetune_speaker.json b/configs/modified_finetune_speaker.json index 29d237f..cb193cb 100644 --- a/configs/modified_finetune_speaker.json +++ b/configs/modified_finetune_speaker.json @@ -4,9 +4,12 @@ "eval_interval": 1000, "seed": 1234, "epochs": 10000, - "learning_rate": 2e-4, - "betas": [0.8, 0.99], - "eps": 1e-9, + "learning_rate": 0.0002, + "betas": [ + 0.8, + 0.99 + ], + "eps": 1e-09, "batch_size": 12, "fp16_run": true, "lr_decay": 0.999875, @@ -17,9 +20,11 @@ "c_kl": 1.0 }, "data": { - "training_files":"final_annotation_train.txt", - "validation_files":"final_annotation_val.txt", - "text_cleaners":["cjke_cleaners2"], + "training_files": "final_annotation_train.txt", + "validation_files": "final_annotation_val.txt", + "text_cleaners": [ + "cjke_cleaners2" + ], "max_wav_value": 32768.0, "sampling_rate": 22050, "filter_length": 1024, @@ -29,7 +34,7 @@ "mel_fmin": 0.0, "mel_fmax": null, "add_blank": true, - "n_speakers": 1001, + "n_speakers": 1003, "cleaned_text": true }, "model": { @@ -41,164 +46,266 @@ "kernel_size": 3, "p_dropout": 0.1, "resblock": "1", - "resblock_kernel_sizes": [3,7,11], - "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], - "upsample_rates": [8,8,2,2], + "resblock_kernel_sizes": [ + 3, + 7, + 11 + ], + "resblock_dilation_sizes": [ + [ + 1, + 3, + 5 + ], + [ + 1, + 3, + 5 + ], + [ + 1, + 3, + 5 + ] + ], + "upsample_rates": [ + 8, + 8, + 2, + 2 + ], "upsample_initial_channel": 512, - "upsample_kernel_sizes": [16,16,4,4], + "upsample_kernel_sizes": [ + 16, + 16, + 4, + 4 + ], "n_layers_q": 3, "use_spectral_norm": false, "gin_channels": 256 }, - "symbols": ["_", ",", ".", "!", "?", "-", "~", "\u2026", "N", "Q", "a", "b", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "s", "t", "u", "v", "w", "x", "y", "z", "\u0251", "\u00e6", "\u0283", "\u0291", "\u00e7", "\u026f", "\u026a", "\u0254", "\u025b", "\u0279", "\u00f0", "\u0259", "\u026b", "\u0265", "\u0278", "\u028a", "\u027e", "\u0292", "\u03b8", "\u03b2", "\u014b", "\u0266", "\u207c", "\u02b0", "`", "^", "#", "*", "=", "\u02c8", "\u02cc", "\u2192", "\u2193", "\u2191", " "], - "speakers": {"特别周 Special Week (Umamusume Pretty Derby)": 0, - "无声铃鹿 Silence Suzuka (Umamusume Pretty Derby)": 1, - "东海帝王 Tokai Teio (Umamusume Pretty Derby)": 2, - "丸善斯基 Maruzensky (Umamusume Pretty Derby)": 3, - "富士奇迹 Fuji Kiseki (Umamusume Pretty Derby)": 4, - "小栗帽 Oguri Cap (Umamusume Pretty Derby)": 5, - "黄金船 Gold Ship (Umamusume Pretty Derby)": 6, - "伏特加 Vodka (Umamusume Pretty Derby)": 7, - "大和赤骥 Daiwa Scarlet (Umamusume Pretty Derby)": 8, - "大树快车 Taiki Shuttle (Umamusume Pretty Derby)": 9, - "草上飞 Grass Wonder (Umamusume Pretty Derby)": 10, - "菱亚马逊 Hishi Amazon (Umamusume Pretty Derby)": 11, - "目白麦昆 Mejiro Mcqueen (Umamusume Pretty Derby)": 12, - "神鹰 El Condor Pasa (Umamusume Pretty Derby)": 13, - "好歌剧 T.M. Opera O (Umamusume Pretty Derby)": 14, - "成田白仁 Narita Brian (Umamusume Pretty Derby)": 15, - "鲁道夫象征 Symboli Rudolf (Umamusume Pretty Derby)": 16, - "气槽 Air Groove (Umamusume Pretty Derby)": 17, - "爱丽数码 Agnes Digital (Umamusume Pretty Derby)": 18, - "青云天空 Seiun Sky (Umamusume Pretty Derby)": 19, - "玉藻十字 Tamamo Cross (Umamusume Pretty Derby)": 20, - "美妙姿势 Fine Motion (Umamusume Pretty Derby)": 21, - "琵琶晨光 Biwa Hayahide (Umamusume Pretty Derby)": 22, - "重炮 Mayano Topgun (Umamusume Pretty Derby)": 23, - "曼城茶座 Manhattan Cafe (Umamusume Pretty Derby)": 24, - "美普波旁 Mihono Bourbon (Umamusume Pretty Derby)": 25, - "目白雷恩 Mejiro Ryan (Umamusume Pretty Derby)": 26, - "雪之美人 Yukino Bijin (Umamusume Pretty Derby)": 28, - "米浴 Rice Shower (Umamusume Pretty Derby)": 29, - "艾尼斯风神 Ines Fujin (Umamusume Pretty Derby)": 30, - "爱丽速子 Agnes Tachyon (Umamusume Pretty Derby)": 31, - "爱慕织姬 Admire Vega (Umamusume Pretty Derby)": 32, - "稻荷一 Inari One (Umamusume Pretty Derby)": 33, - "胜利奖券 Winning Ticket (Umamusume Pretty Derby)": 34, - "空中神宫 Air Shakur (Umamusume Pretty Derby)": 35, - "荣进闪耀 Eishin Flash (Umamusume Pretty Derby)": 36, - "真机伶 Curren Chan (Umamusume Pretty Derby)": 37, - "川上公主 Kawakami Princess (Umamusume Pretty Derby)": 38, - "黄金城市 Gold City (Umamusume Pretty Derby)": 39, - "樱花进王 Sakura Bakushin O (Umamusume Pretty Derby)": 40, - "采珠 Seeking the Pearl (Umamusume Pretty Derby)": 41, - "新光风 Shinko Windy (Umamusume Pretty Derby)": 42, - "东商变革 Sweep Tosho (Umamusume Pretty Derby)": 43, - "超级小溪 Super Creek (Umamusume Pretty Derby)": 44, - "醒目飞鹰 Smart Falcon (Umamusume Pretty Derby)": 45, - "荒漠英雄 Zenno Rob Roy (Umamusume Pretty Derby)": 46, - "东瀛佐敦 Tosen Jordan (Umamusume Pretty Derby)": 47, - "中山庆典 Nakayama Festa (Umamusume Pretty Derby)": 48, - "成田大进 Narita Taishin (Umamusume Pretty Derby)": 49, - "西野花 Nishino Flower (Umamusume Pretty Derby)": 50, - "春乌拉拉 Haru Urara (Umamusume Pretty Derby)": 51, - "青竹回忆 Bamboo Memory (Umamusume Pretty Derby)": 52, - "待兼福来 Matikane Fukukitaru (Umamusume Pretty Derby)": 55, - "名将怒涛 Meisho Doto (Umamusume Pretty Derby)": 57, - "目白多伯 Mejiro Dober (Umamusume Pretty Derby)": 58, - "优秀素质 Nice Nature (Umamusume Pretty Derby)": 59, - "帝王光环 King Halo (Umamusume Pretty Derby)": 60, - "待兼诗歌剧 Matikane Tannhauser (Umamusume Pretty Derby)": 61, - "生野狄杜斯 Ikuno Dictus (Umamusume Pretty Derby)": 62, - "目白善信 Mejiro Palmer (Umamusume Pretty Derby)": 63, - "大拓太阳神 Daitaku Helios (Umamusume Pretty Derby)": 64, - "双涡轮 Twin Turbo (Umamusume Pretty Derby)": 65, - "里见光钻 Satono Diamond (Umamusume Pretty Derby)": 66, - "北部玄驹 Kitasan Black (Umamusume Pretty Derby)": 67, - "樱花千代王 Sakura Chiyono O (Umamusume Pretty Derby)": 68, - "天狼星象征 Sirius Symboli (Umamusume Pretty Derby)": 69, - "目白阿尔丹 Mejiro Ardan (Umamusume Pretty Derby)": 70, - "八重无敌 Yaeno Muteki (Umamusume Pretty Derby)": 71, - "鹤丸刚志 Tsurumaru Tsuyoshi (Umamusume Pretty Derby)": 72, - "目白光明 Mejiro Bright (Umamusume Pretty Derby)": 73, - "樱花桂冠 Sakura Laurel (Umamusume Pretty Derby)": 74, - "成田路 Narita Top Road (Umamusume Pretty Derby)": 75, - "也文摄辉 Yamanin Zephyr (Umamusume Pretty Derby)": 76, - "真弓快车 Aston Machan (Umamusume Pretty Derby)": 80, - "骏川手纲 Hayakawa Tazuna (Umamusume Pretty Derby)": 81, - "小林历奇 Kopano Rickey (Umamusume Pretty Derby)": 83, - "奇锐骏 Wonder Acute (Umamusume Pretty Derby)": 85, - "秋川理事长 President Akikawa (Umamusume Pretty Derby)": 86, - "綾地 寧々 Ayachi Nene (Sanoba Witch)": 87, - "因幡 めぐる Inaba Meguru (Sanoba Witch)": 88, - "椎葉 紬 Shiiba Tsumugi (Sanoba Witch)": 89, - "仮屋 和奏 Kariya Wakama (Sanoba Witch)": 90, - "戸隠 憧子 Togakushi Touko (Sanoba Witch)": 91, - "九条裟罗 Kujou Sara (Genshin Impact)": 92, - "芭芭拉 Barbara (Genshin Impact)": 93, - "派蒙 Paimon (Genshin Impact)": 94, - "荒泷一斗 Arataki Itto (Genshin Impact)": 96, - "早柚 Sayu (Genshin Impact)": 97, - "香菱 Xiangling (Genshin Impact)": 98, - "神里绫华 Kamisato Ayaka (Genshin Impact)": 99, - "重云 Chongyun (Genshin Impact)": 100, - "流浪者 Wanderer (Genshin Impact)": 102, - "优菈 Eula (Genshin Impact)": 103, - "凝光 Ningguang (Genshin Impact)": 105, - "钟离 Zhongli (Genshin Impact)": 106, - "雷电将军 Raiden Shogun (Genshin Impact)": 107, - "枫原万叶 Kaedehara Kazuha (Genshin Impact)": 108, - "赛诺 Cyno (Genshin Impact)": 109, - "诺艾尔 Noelle (Genshin Impact)": 112, - "八重神子 Yae Miko (Genshin Impact)": 113, - "凯亚 Kaeya (Genshin Impact)": 114, - "魈 Xiao (Genshin Impact)": 115, - "托马 Thoma (Genshin Impact)": 116, - "可莉 Klee (Genshin Impact)": 117, - "迪卢克 Diluc (Genshin Impact)": 120, - "夜兰 Yelan (Genshin Impact)": 121, - "鹿野院平藏 Shikanoin Heizou (Genshin Impact)": 123, - "辛焱 Xinyan (Genshin Impact)": 124, - "丽莎 Lisa (Genshin Impact)": 125, - "云堇 Yun Jin (Genshin Impact)": 126, - "坎蒂丝 Candace (Genshin Impact)": 127, - "罗莎莉亚 Rosaria (Genshin Impact)": 128, - "北斗 Beidou (Genshin Impact)": 129, - "珊瑚宫心海 Sangonomiya Kokomi (Genshin Impact)": 132, - "烟绯 Yanfei (Genshin Impact)": 133, - "久岐忍 Kuki Shinobu (Genshin Impact)": 136, - "宵宫 Yoimiya (Genshin Impact)": 139, - "安柏 Amber (Genshin Impact)": 143, - "迪奥娜 Diona (Genshin Impact)": 144, - "班尼特 Bennett (Genshin Impact)": 146, - "雷泽 Razor (Genshin Impact)": 147, - "阿贝多 Albedo (Genshin Impact)": 151, - "温迪 Venti (Genshin Impact)": 152, - "空 Player Male (Genshin Impact)": 153, - "神里绫人 Kamisato Ayato (Genshin Impact)": 154, - "琴 Jean (Genshin Impact)": 155, - "艾尔海森 Alhaitham (Genshin Impact)": 156, - "莫娜 Mona (Genshin Impact)": 157, - "妮露 Nilou (Genshin Impact)": 159, - "胡桃 Hu Tao (Genshin Impact)": 160, - "甘雨 Ganyu (Genshin Impact)": 161, - "纳西妲 Nahida (Genshin Impact)": 162, - "刻晴 Keqing (Genshin Impact)": 165, - "荧 Player Female (Genshin Impact)": 169, - "埃洛伊 Aloy (Genshin Impact)": 179, - "柯莱 Collei (Genshin Impact)": 182, - "多莉 Dori (Genshin Impact)": 184, - "提纳里 Tighnari (Genshin Impact)": 186, - "砂糖 Sucrose (Genshin Impact)": 188, - "行秋 Xingqiu (Genshin Impact)": 190, - "奥兹 Oz (Genshin Impact)": 193, - "五郎 Gorou (Genshin Impact)": 198, - "达达利亚 Tartalia (Genshin Impact)": 202, - "七七 Qiqi (Genshin Impact)": 207, - "申鹤 Shenhe (Genshin Impact)": 217, - "莱依拉 Layla (Genshin Impact)": 228, - "菲谢尔 Fishl (Genshin Impact)": 230, - "User": 999 + "symbols": [ + "_", + ",", + ".", + "!", + "?", + "-", + "~", + "\u2026", + "N", + "Q", + "a", + "b", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "\u0251", + "\u00e6", + "\u0283", + "\u0291", + "\u00e7", + "\u026f", + "\u026a", + "\u0254", + "\u025b", + "\u0279", + "\u00f0", + "\u0259", + "\u026b", + "\u0265", + "\u0278", + "\u028a", + "\u027e", + "\u0292", + "\u03b8", + "\u03b2", + "\u014b", + "\u0266", + "\u207c", + "\u02b0", + "`", + "^", + "#", + "*", + "=", + "\u02c8", + "\u02cc", + "\u2192", + "\u2193", + "\u2191", + " " + ], + "speakers": { + "\u7279\u522b\u5468 Special Week (Umamusume Pretty Derby)": 0, + "\u65e0\u58f0\u94c3\u9e7f Silence Suzuka (Umamusume Pretty Derby)": 1, + "\u4e1c\u6d77\u5e1d\u738b Tokai Teio (Umamusume Pretty Derby)": 2, + "\u4e38\u5584\u65af\u57fa Maruzensky (Umamusume Pretty Derby)": 3, + "\u5bcc\u58eb\u5947\u8ff9 Fuji Kiseki (Umamusume Pretty Derby)": 4, + "\u5c0f\u6817\u5e3d Oguri Cap (Umamusume Pretty Derby)": 5, + "\u9ec4\u91d1\u8239 Gold Ship (Umamusume Pretty Derby)": 6, + "\u4f0f\u7279\u52a0 Vodka (Umamusume Pretty Derby)": 7, + "\u5927\u548c\u8d64\u9aa5 Daiwa Scarlet (Umamusume Pretty Derby)": 8, + "\u5927\u6811\u5feb\u8f66 Taiki Shuttle (Umamusume Pretty Derby)": 9, + "\u8349\u4e0a\u98de Grass Wonder (Umamusume Pretty Derby)": 10, + "\u83f1\u4e9a\u9a6c\u900a Hishi Amazon (Umamusume Pretty Derby)": 11, + "\u76ee\u767d\u9ea6\u6606 Mejiro Mcqueen (Umamusume Pretty Derby)": 12, + "\u795e\u9e70 El Condor Pasa (Umamusume Pretty Derby)": 13, + "\u597d\u6b4c\u5267 T.M. Opera O (Umamusume Pretty Derby)": 14, + "\u6210\u7530\u767d\u4ec1 Narita Brian (Umamusume Pretty Derby)": 15, + "\u9c81\u9053\u592b\u8c61\u5f81 Symboli Rudolf (Umamusume Pretty Derby)": 16, + "\u6c14\u69fd Air Groove (Umamusume Pretty Derby)": 17, + "\u7231\u4e3d\u6570\u7801 Agnes Digital (Umamusume Pretty Derby)": 18, + "\u9752\u4e91\u5929\u7a7a Seiun Sky (Umamusume Pretty Derby)": 19, + "\u7389\u85fb\u5341\u5b57 Tamamo Cross (Umamusume Pretty Derby)": 20, + "\u7f8e\u5999\u59ff\u52bf Fine Motion (Umamusume Pretty Derby)": 21, + "\u7435\u7436\u6668\u5149 Biwa Hayahide (Umamusume Pretty Derby)": 22, + "\u91cd\u70ae Mayano Topgun (Umamusume Pretty Derby)": 23, + "\u66fc\u57ce\u8336\u5ea7 Manhattan Cafe (Umamusume Pretty Derby)": 24, + "\u7f8e\u666e\u6ce2\u65c1 Mihono Bourbon (Umamusume Pretty Derby)": 25, + "\u76ee\u767d\u96f7\u6069 Mejiro Ryan (Umamusume Pretty Derby)": 26, + "\u96ea\u4e4b\u7f8e\u4eba Yukino Bijin (Umamusume Pretty Derby)": 28, + "\u7c73\u6d74 Rice Shower (Umamusume Pretty Derby)": 29, + "\u827e\u5c3c\u65af\u98ce\u795e Ines Fujin (Umamusume Pretty Derby)": 30, + "\u7231\u4e3d\u901f\u5b50 Agnes Tachyon (Umamusume Pretty Derby)": 31, + "\u7231\u6155\u7ec7\u59ec Admire Vega (Umamusume Pretty Derby)": 32, + "\u7a3b\u8377\u4e00 Inari One (Umamusume Pretty Derby)": 33, + "\u80dc\u5229\u5956\u5238 Winning Ticket (Umamusume Pretty Derby)": 34, + "\u7a7a\u4e2d\u795e\u5bab Air Shakur (Umamusume Pretty Derby)": 35, + "\u8363\u8fdb\u95ea\u8000 Eishin Flash (Umamusume Pretty Derby)": 36, + "\u771f\u673a\u4f36 Curren Chan (Umamusume Pretty Derby)": 37, + "\u5ddd\u4e0a\u516c\u4e3b Kawakami Princess (Umamusume Pretty Derby)": 38, + "\u9ec4\u91d1\u57ce\u5e02 Gold City (Umamusume Pretty Derby)": 39, + "\u6a31\u82b1\u8fdb\u738b Sakura Bakushin O (Umamusume Pretty Derby)": 40, + "\u91c7\u73e0 Seeking the Pearl (Umamusume Pretty Derby)": 41, + "\u65b0\u5149\u98ce Shinko Windy (Umamusume Pretty Derby)": 42, + "\u4e1c\u5546\u53d8\u9769 Sweep Tosho (Umamusume Pretty Derby)": 43, + "\u8d85\u7ea7\u5c0f\u6eaa Super Creek (Umamusume Pretty Derby)": 44, + "\u9192\u76ee\u98de\u9e70 Smart Falcon (Umamusume Pretty Derby)": 45, + "\u8352\u6f20\u82f1\u96c4 Zenno Rob Roy (Umamusume Pretty Derby)": 46, + "\u4e1c\u701b\u4f50\u6566 Tosen Jordan (Umamusume Pretty Derby)": 47, + "\u4e2d\u5c71\u5e86\u5178 Nakayama Festa (Umamusume Pretty Derby)": 48, + "\u6210\u7530\u5927\u8fdb Narita Taishin (Umamusume Pretty Derby)": 49, + "\u897f\u91ce\u82b1 Nishino Flower (Umamusume Pretty Derby)": 50, + "\u6625\u4e4c\u62c9\u62c9 Haru Urara (Umamusume Pretty Derby)": 51, + "\u9752\u7af9\u56de\u5fc6 Bamboo Memory (Umamusume Pretty Derby)": 52, + "\u5f85\u517c\u798f\u6765 Matikane Fukukitaru (Umamusume Pretty Derby)": 55, + "\u540d\u5c06\u6012\u6d9b Meisho Doto (Umamusume Pretty Derby)": 57, + "\u76ee\u767d\u591a\u4f2f Mejiro Dober (Umamusume Pretty Derby)": 58, + "\u4f18\u79c0\u7d20\u8d28 Nice Nature (Umamusume Pretty Derby)": 59, + "\u5e1d\u738b\u5149\u73af King Halo (Umamusume Pretty Derby)": 60, + "\u5f85\u517c\u8bd7\u6b4c\u5267 Matikane Tannhauser (Umamusume Pretty Derby)": 61, + "\u751f\u91ce\u72c4\u675c\u65af Ikuno Dictus (Umamusume Pretty Derby)": 62, + "\u76ee\u767d\u5584\u4fe1 Mejiro Palmer (Umamusume Pretty Derby)": 63, + "\u5927\u62d3\u592a\u9633\u795e Daitaku Helios (Umamusume Pretty Derby)": 64, + "\u53cc\u6da1\u8f6e Twin Turbo (Umamusume Pretty Derby)": 65, + "\u91cc\u89c1\u5149\u94bb Satono Diamond (Umamusume Pretty Derby)": 66, + "\u5317\u90e8\u7384\u9a79 Kitasan Black (Umamusume Pretty Derby)": 67, + "\u6a31\u82b1\u5343\u4ee3\u738b Sakura Chiyono O (Umamusume Pretty Derby)": 68, + "\u5929\u72fc\u661f\u8c61\u5f81 Sirius Symboli (Umamusume Pretty Derby)": 69, + "\u76ee\u767d\u963f\u5c14\u4e39 Mejiro Ardan (Umamusume Pretty Derby)": 70, + "\u516b\u91cd\u65e0\u654c Yaeno Muteki (Umamusume Pretty Derby)": 71, + "\u9e64\u4e38\u521a\u5fd7 Tsurumaru Tsuyoshi (Umamusume Pretty Derby)": 72, + "\u76ee\u767d\u5149\u660e Mejiro Bright (Umamusume Pretty Derby)": 73, + "\u6a31\u82b1\u6842\u51a0 Sakura Laurel (Umamusume Pretty Derby)": 74, + "\u6210\u7530\u8def Narita Top Road (Umamusume Pretty Derby)": 75, + "\u4e5f\u6587\u6444\u8f89 Yamanin Zephyr (Umamusume Pretty Derby)": 76, + "\u771f\u5f13\u5feb\u8f66 Aston Machan (Umamusume Pretty Derby)": 80, + "\u9a8f\u5ddd\u624b\u7eb2 Hayakawa Tazuna (Umamusume Pretty Derby)": 81, + "\u5c0f\u6797\u5386\u5947 Kopano Rickey (Umamusume Pretty Derby)": 83, + "\u5947\u9510\u9a8f Wonder Acute (Umamusume Pretty Derby)": 85, + "\u79cb\u5ddd\u7406\u4e8b\u957f President Akikawa (Umamusume Pretty Derby)": 86, + "\u7dbe\u5730 \u5be7\u3005 Ayachi Nene (Sanoba Witch)": 87, + "\u56e0\u5e61 \u3081\u3050\u308b Inaba Meguru (Sanoba Witch)": 88, + "\u690e\u8449 \u7d2c Shiiba Tsumugi (Sanoba Witch)": 89, + "\u4eee\u5c4b \u548c\u594f Kariya Wakama (Sanoba Witch)": 90, + "\u6238\u96a0 \u61a7\u5b50 Togakushi Touko (Sanoba Witch)": 91, + "\u4e5d\u6761\u88df\u7f57 Kujou Sara (Genshin Impact)": 92, + "\u82ad\u82ad\u62c9 Barbara (Genshin Impact)": 93, + "\u6d3e\u8499 Paimon (Genshin Impact)": 94, + "\u8352\u6cf7\u4e00\u6597 Arataki Itto (Genshin Impact)": 96, + "\u65e9\u67da Sayu (Genshin Impact)": 97, + "\u9999\u83f1 Xiangling (Genshin Impact)": 98, + "\u795e\u91cc\u7eeb\u534e Kamisato Ayaka (Genshin Impact)": 99, + "\u91cd\u4e91 Chongyun (Genshin Impact)": 100, + "\u6d41\u6d6a\u8005 Wanderer (Genshin Impact)": 102, + "\u4f18\u83c8 Eula (Genshin Impact)": 103, + "\u51dd\u5149 Ningguang (Genshin Impact)": 105, + "\u949f\u79bb Zhongli (Genshin Impact)": 106, + "\u96f7\u7535\u5c06\u519b Raiden Shogun (Genshin Impact)": 107, + "\u67ab\u539f\u4e07\u53f6 Kaedehara Kazuha (Genshin Impact)": 108, + "\u8d5b\u8bfa Cyno (Genshin Impact)": 109, + "\u8bfa\u827e\u5c14 Noelle (Genshin Impact)": 112, + "\u516b\u91cd\u795e\u5b50 Yae Miko (Genshin Impact)": 113, + "\u51ef\u4e9a Kaeya (Genshin Impact)": 114, + "\u9b48 Xiao (Genshin Impact)": 115, + "\u6258\u9a6c Thoma (Genshin Impact)": 116, + "\u53ef\u8389 Klee (Genshin Impact)": 117, + "\u8fea\u5362\u514b Diluc (Genshin Impact)": 120, + "\u591c\u5170 Yelan (Genshin Impact)": 121, + "\u9e7f\u91ce\u9662\u5e73\u85cf Shikanoin Heizou (Genshin Impact)": 123, + "\u8f9b\u7131 Xinyan (Genshin Impact)": 124, + "\u4e3d\u838e Lisa (Genshin Impact)": 125, + "\u4e91\u5807 Yun Jin (Genshin Impact)": 126, + "\u574e\u8482\u4e1d Candace (Genshin Impact)": 127, + "\u7f57\u838e\u8389\u4e9a Rosaria (Genshin Impact)": 128, + "\u5317\u6597 Beidou (Genshin Impact)": 129, + "\u73ca\u745a\u5bab\u5fc3\u6d77 Sangonomiya Kokomi (Genshin Impact)": 132, + "\u70df\u7eef Yanfei (Genshin Impact)": 133, + "\u4e45\u5c90\u5fcd Kuki Shinobu (Genshin Impact)": 136, + "\u5bb5\u5bab Yoimiya (Genshin Impact)": 139, + "\u5b89\u67cf Amber (Genshin Impact)": 143, + "\u8fea\u5965\u5a1c Diona (Genshin Impact)": 144, + "\u73ed\u5c3c\u7279 Bennett (Genshin Impact)": 146, + "\u96f7\u6cfd Razor (Genshin Impact)": 147, + "\u963f\u8d1d\u591a Albedo (Genshin Impact)": 151, + "\u6e29\u8fea Venti (Genshin Impact)": 152, + "\u7a7a Player Male (Genshin Impact)": 153, + "\u795e\u91cc\u7eeb\u4eba Kamisato Ayato (Genshin Impact)": 154, + "\u7434 Jean (Genshin Impact)": 155, + "\u827e\u5c14\u6d77\u68ee Alhaitham (Genshin Impact)": 156, + "\u83ab\u5a1c Mona (Genshin Impact)": 157, + "\u59ae\u9732 Nilou (Genshin Impact)": 159, + "\u80e1\u6843 Hu Tao (Genshin Impact)": 160, + "\u7518\u96e8 Ganyu (Genshin Impact)": 161, + "\u7eb3\u897f\u59b2 Nahida (Genshin Impact)": 162, + "\u523b\u6674 Keqing (Genshin Impact)": 165, + "\u8367 Player Female (Genshin Impact)": 169, + "\u57c3\u6d1b\u4f0a Aloy (Genshin Impact)": 179, + "\u67ef\u83b1 Collei (Genshin Impact)": 182, + "\u591a\u8389 Dori (Genshin Impact)": 184, + "\u63d0\u7eb3\u91cc Tighnari (Genshin Impact)": 186, + "\u7802\u7cd6 Sucrose (Genshin Impact)": 188, + "\u884c\u79cb Xingqiu (Genshin Impact)": 190, + "\u5965\u5179 Oz (Genshin Impact)": 193, + "\u4e94\u90ce Gorou (Genshin Impact)": 198, + "\u8fbe\u8fbe\u5229\u4e9a Tartalia (Genshin Impact)": 202, + "\u4e03\u4e03 Qiqi (Genshin Impact)": 207, + "\u7533\u9e64 Shenhe (Genshin Impact)": 217, + "\u83b1\u4f9d\u62c9 Layla (Genshin Impact)": 228, + "\u83f2\u8c22\u5c14 Fishl (Genshin Impact)": 230, + "User": 999, + "rosalia": 1000, + "taffy": 1001, + "YaeSakura": 1002 } - } \ No newline at end of file diff --git a/data_utils.py b/data_utils.py index a129aa4..cd997fa 100644 --- a/data_utils.py +++ b/data_utils.py @@ -10,146 +10,6 @@ import commons from mel_processing import spectrogram_torch from utils import load_wav_to_torch, load_filepaths_and_text from text import text_to_sequence, cleaned_text_to_sequence - - -class TextAudioLoader(torch.utils.data.Dataset): - """ - 1) loads audio, text pairs - 2) normalizes text and converts them to sequences of integers - 3) computes spectrograms from audio files. - """ - - def __init__(self, audiopaths_and_text, hparams): - self.audiopaths_and_text = load_filepaths_and_text(audiopaths_and_text) - self.text_cleaners = hparams.text_cleaners - self.max_wav_value = hparams.max_wav_value - self.sampling_rate = hparams.sampling_rate - self.filter_length = hparams.filter_length - self.hop_length = hparams.hop_length - self.win_length = hparams.win_length - self.sampling_rate = hparams.sampling_rate - - self.cleaned_text = getattr(hparams, "cleaned_text", False) - - self.add_blank = hparams.add_blank - self.min_text_len = getattr(hparams, "min_text_len", 1) - self.max_text_len = getattr(hparams, "max_text_len", 190) - - random.seed(1234) - random.shuffle(self.audiopaths_and_text) - self._filter() - - def _filter(self): - """ - Filter text & store spec lengths - """ - # Store spectrogram lengths for Bucketing - # wav_length ~= file_size / (wav_channels * Bytes per dim) = file_size / (1 * 2) - # spec_length = wav_length // hop_length - - audiopaths_and_text_new = [] - lengths = [] - for audiopath, text in self.audiopaths_and_text: - if self.min_text_len <= len(text) and len(text) <= self.max_text_len: - audiopaths_and_text_new.append([audiopath, text]) - lengths.append(os.path.getsize(audiopath) // (2 * self.hop_length)) - self.audiopaths_and_text = audiopaths_and_text_new - self.lengths = lengths - - def get_audio_text_pair(self, audiopath_and_text): - # separate filename and text - audiopath, text = audiopath_and_text[0], audiopath_and_text[1] - text = self.get_text(text) - spec, wav = self.get_audio(audiopath) - return (text, spec, wav) - - def get_audio(self, filename): - audio, sampling_rate = load_wav_to_torch(filename) - if sampling_rate != self.sampling_rate: - raise ValueError("{} {} SR doesn't match target {} SR".format( - sampling_rate, self.sampling_rate)) - audio_norm = audio / self.max_wav_value - audio_norm = audio_norm.unsqueeze(0) - spec_filename = filename.replace(".wav", ".spec.pt") - if os.path.exists(spec_filename): - spec = torch.load(spec_filename) - else: - spec = spectrogram_torch(audio_norm, self.filter_length, - self.sampling_rate, self.hop_length, self.win_length, - center=False) - spec = torch.squeeze(spec, 0) - torch.save(spec, spec_filename) - return spec, audio_norm - - def get_text(self, text): - if self.cleaned_text: - text_norm = cleaned_text_to_sequence(text) - else: - text_norm = text_to_sequence(text, self.text_cleaners) - if self.add_blank: - text_norm = commons.intersperse(text_norm, 0) - text_norm = torch.LongTensor(text_norm) - return text_norm - - def __getitem__(self, index): - return self.get_audio_text_pair(self.audiopaths_and_text[index]) - - def __len__(self): - return len(self.audiopaths_and_text) - - -class TextAudioCollate(): - """ Zero-pads model inputs and targets - """ - - def __init__(self, return_ids=False): - self.return_ids = return_ids - - def __call__(self, batch): - """Collate's training batch from normalized text and aduio - PARAMS - ------ - batch: [text_normalized, spec_normalized, wav_normalized] - """ - # Right zero-pad all one-hot text sequences to max input length - _, ids_sorted_decreasing = torch.sort( - torch.LongTensor([x[1].size(1) for x in batch]), - dim=0, descending=True) - - max_text_len = max([len(x[0]) for x in batch]) - max_spec_len = max([x[1].size(1) for x in batch]) - max_wav_len = max([x[2].size(1) for x in batch]) - - text_lengths = torch.LongTensor(len(batch)) - spec_lengths = torch.LongTensor(len(batch)) - wav_lengths = torch.LongTensor(len(batch)) - - text_padded = torch.LongTensor(len(batch), max_text_len) - spec_padded = torch.FloatTensor(len(batch), batch[0][1].size(0), max_spec_len) - wav_padded = torch.FloatTensor(len(batch), 1, max_wav_len) - text_padded.zero_() - spec_padded.zero_() - wav_padded.zero_() - for i in range(len(ids_sorted_decreasing)): - row = batch[ids_sorted_decreasing[i]] - - text = row[0] - text_padded[i, :text.size(0)] = text - text_lengths[i] = text.size(0) - - spec = row[1] - spec_padded[i, :, :spec.size(1)] = spec - spec_lengths[i] = spec.size(1) - - wav = row[2] - wav_padded[i, :, :wav.size(1)] = wav - wav_lengths[i] = wav.size(1) - - if self.return_ids: - return text_padded, text_lengths, spec_padded, spec_lengths, wav_padded, wav_lengths, ids_sorted_decreasing - return text_padded, text_lengths, spec_padded, spec_lengths, wav_padded, wav_lengths - - """Multi speaker version""" @@ -160,7 +20,7 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset): 3) computes spectrograms from audio files. """ - def __init__(self, audiopaths_sid_text, hparams): + def __init__(self, audiopaths_sid_text, hparams, symbols): self.audiopaths_sid_text = load_filepaths_and_text(audiopaths_sid_text) self.text_cleaners = hparams.text_cleaners self.max_wav_value = hparams.max_wav_value @@ -175,6 +35,7 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset): self.add_blank = hparams.add_blank self.min_text_len = getattr(hparams, "min_text_len", 1) self.max_text_len = getattr(hparams, "max_text_len", 190) + self.symbols = symbols random.seed(1234) random.shuffle(self.audiopaths_sid_text) @@ -232,7 +93,7 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset): def get_text(self, text): if self.cleaned_text: - text_norm = cleaned_text_to_sequence(text) + text_norm = cleaned_text_to_sequence(text, self.symbols) else: text_norm = text_to_sequence(text, self.text_cleaners) if self.add_blank: diff --git a/denoise_audio.py b/denoise_audio.py new file mode 100644 index 0000000..757e926 --- /dev/null +++ b/denoise_audio.py @@ -0,0 +1,18 @@ +import os +import torchaudio +raw_audio_dir = "./raw_audio/" +denoise_audio_dir = "./denoised_audio/" +filelist = list(os.walk(raw_audio_dir))[0][2] + +for file in filelist: + if file.endswith(".wav"): + os.system(f"demucs --two-stems=vocals {raw_audio_dir}{file}") +for file in filelist: + file = file.replace(".wav", "") + wav, sr = torchaudio.load(f"./separated/htdemucs/{file}/vocals.wav", frame_offset=0, num_frames=-1, normalize=True, + channels_first=True) + # merge two channels into one + wav = wav.mean(dim=0).unsqueeze(0) + if sr != 22050: + wav = torchaudio.transforms.Resample(orig_freq=sr, new_freq=22050)(wav) + torchaudio.save(denoise_audio_dir + file + ".wav", wav, 22050, channels_first=True) \ No newline at end of file diff --git a/download_model.py b/download_model.py index fda2716..9f1ab59 100644 --- a/download_model.py +++ b/download_model.py @@ -1,3 +1,4 @@ from google.colab import files -files.download("./OUTPUT_MODEL/G_latest.pth") -files.download("./finetune_speaker.json") \ No newline at end of file +files.download("./G_latest.pth") +files.download("./finetune_speaker.json") +files.download("./moegoe_config.json") \ No newline at end of file diff --git a/download_video.py b/download_video.py new file mode 100644 index 0000000..7d99b95 --- /dev/null +++ b/download_video.py @@ -0,0 +1,23 @@ +from google.colab import files +import shutil +import os +basepath = os.getcwd() +uploaded = files.upload() # 上传文件 +for filename in uploaded.keys(): + assert(filename.endswith(".txt")), "speaker-videolink info could only be .txt file!" + shutil.move(os.path.join(basepath, filename), os.path.join("./speaker_links.txt")) + +with open("./speaker_links.txt", 'r', encoding='utf-8') as f: + lines = f.readlines() +speakers = [] +for line in lines: + line = line.replace("\n", "").replace(" ", "") + if line == "": + continue + speaker, link = line.split("|") + if speaker not in speakers: + speakers.append(speaker) + # download link + import random + filename = speaker + "_" + str(random.randint(0, 1000000)) + os.system(f"youtube-dl -f 0 {link} -o ./video_data/{filename}.mp4") \ No newline at end of file diff --git a/finetune_speaker_v2.py b/finetune_speaker_v2.py new file mode 100644 index 0000000..49ebdd7 --- /dev/null +++ b/finetune_speaker_v2.py @@ -0,0 +1,320 @@ +import os +import json +import argparse +import itertools +import math +import torch +from torch import nn, optim +from torch.nn import functional as F +from torch.utils.data import DataLoader +from torch.utils.tensorboard import SummaryWriter +import torch.multiprocessing as mp +import torch.distributed as dist +from torch.nn.parallel import DistributedDataParallel as DDP +from torch.cuda.amp import autocast, GradScaler +from tqdm import tqdm + +import librosa +import logging + +logging.getLogger('numba').setLevel(logging.WARNING) + +import commons +import utils +from data_utils import ( + TextAudioSpeakerLoader, + TextAudioSpeakerCollate, + DistributedBucketSampler +) +from models import ( + SynthesizerTrn, + MultiPeriodDiscriminator, +) +from losses import ( + generator_loss, + discriminator_loss, + feature_loss, + kl_loss +) +from mel_processing import mel_spectrogram_torch, spec_to_mel_torch + + +torch.backends.cudnn.benchmark = True +global_step = 0 + + +def main(): + """Assume Single Node Multi GPUs Training Only""" + assert torch.cuda.is_available(), "CPU training is not allowed." + + n_gpus = torch.cuda.device_count() + os.environ['MASTER_ADDR'] = 'localhost' + os.environ['MASTER_PORT'] = '8000' + + hps = utils.get_hparams() + mp.spawn(run, nprocs=n_gpus, args=(n_gpus, hps,)) + + +def run(rank, n_gpus, hps): + global global_step + symbols = hps['symbols'] + if rank == 0: + logger = utils.get_logger(hps.model_dir) + logger.info(hps) + utils.check_git_hash(hps.model_dir) + writer = SummaryWriter(log_dir=hps.model_dir) + writer_eval = SummaryWriter(log_dir=os.path.join(hps.model_dir, "eval")) + + dist.init_process_group(backend='nccl', init_method='env://', world_size=n_gpus, rank=rank) + torch.manual_seed(hps.train.seed) + torch.cuda.set_device(rank) + + train_dataset = TextAudioSpeakerLoader(hps.data.training_files, hps.data, symbols) + train_sampler = DistributedBucketSampler( + train_dataset, + hps.train.batch_size, + [32,300,400,500,600,700,800,900,1000], + num_replicas=n_gpus, + rank=rank, + shuffle=True) + collate_fn = TextAudioSpeakerCollate() + train_loader = DataLoader(train_dataset, num_workers=2, shuffle=False, pin_memory=True, + collate_fn=collate_fn, batch_sampler=train_sampler) + # train_loader = DataLoader(train_dataset, batch_size=hps.train.batch_size, num_workers=2, shuffle=False, pin_memory=True, + # collate_fn=collate_fn) + if rank == 0: + eval_dataset = TextAudioSpeakerLoader(hps.data.validation_files, hps.data, symbols) + eval_loader = DataLoader(eval_dataset, num_workers=0, shuffle=False, + batch_size=hps.train.batch_size, pin_memory=True, + drop_last=False, collate_fn=collate_fn) + + net_g = SynthesizerTrn( + len(symbols), + hps.data.filter_length // 2 + 1, + hps.train.segment_size // hps.data.hop_length, + n_speakers=hps.data.n_speakers, + **hps.model).cuda(rank) + net_d = MultiPeriodDiscriminator(hps.model.use_spectral_norm).cuda(rank) + + # load existing model + _, _, _, _ = utils.load_checkpoint("./pretrained_models/G_0.pth", net_g, None, drop_speaker_emb=hps.drop_speaker_embed) + _, _, _, _ = utils.load_checkpoint("./pretrained_models/D_0.pth", net_d, None) + epoch_str = 1 + global_step = 0 + # freeze all other layers except speaker embedding + for p in net_g.parameters(): + p.requires_grad = True + for p in net_d.parameters(): + p.requires_grad = True + # for p in net_d.parameters(): + # p.requires_grad = False + # net_g.emb_g.weight.requires_grad = True + optim_g = torch.optim.AdamW( + net_g.parameters(), + hps.train.learning_rate, + betas=hps.train.betas, + eps=hps.train.eps) + optim_d = torch.optim.AdamW( + net_d.parameters(), + hps.train.learning_rate, + betas=hps.train.betas, + eps=hps.train.eps) + # optim_d = None + net_g = DDP(net_g, device_ids=[rank]) + net_d = DDP(net_d, device_ids=[rank]) + + scheduler_g = torch.optim.lr_scheduler.ExponentialLR(optim_g, gamma=hps.train.lr_decay) + scheduler_d = torch.optim.lr_scheduler.ExponentialLR(optim_d, gamma=hps.train.lr_decay) + + scaler = GradScaler(enabled=hps.train.fp16_run) + + for epoch in range(epoch_str, hps.train.epochs + 1): + if rank==0: + train_and_evaluate(rank, epoch, hps, [net_g, net_d], [optim_g, optim_d], [scheduler_g, scheduler_d], scaler, [train_loader, eval_loader], logger, [writer, writer_eval]) + else: + train_and_evaluate(rank, epoch, hps, [net_g, net_d], [optim_g, optim_d], [scheduler_g, scheduler_d], scaler, [train_loader, None], None, None) + scheduler_g.step() + scheduler_d.step() + + +def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler, loaders, logger, writers): + net_g, net_d = nets + optim_g, optim_d = optims + scheduler_g, scheduler_d = schedulers + train_loader, eval_loader = loaders + if writers is not None: + writer, writer_eval = writers + + # train_loader.batch_sampler.set_epoch(epoch) + global global_step + + net_g.train() + net_d.train() + for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths, speakers) in enumerate(tqdm(train_loader)): + x, x_lengths = x.cuda(rank, non_blocking=True), x_lengths.cuda(rank, non_blocking=True) + spec, spec_lengths = spec.cuda(rank, non_blocking=True), spec_lengths.cuda(rank, non_blocking=True) + y, y_lengths = y.cuda(rank, non_blocking=True), y_lengths.cuda(rank, non_blocking=True) + speakers = speakers.cuda(rank, non_blocking=True) + + with autocast(enabled=hps.train.fp16_run): + y_hat, l_length, attn, ids_slice, x_mask, z_mask,\ + (z, z_p, m_p, logs_p, m_q, logs_q) = net_g(x, x_lengths, spec, spec_lengths, speakers) + + mel = spec_to_mel_torch( + spec, + hps.data.filter_length, + hps.data.n_mel_channels, + hps.data.sampling_rate, + hps.data.mel_fmin, + hps.data.mel_fmax) + y_mel = commons.slice_segments(mel, ids_slice, hps.train.segment_size // hps.data.hop_length) + y_hat_mel = mel_spectrogram_torch( + y_hat.squeeze(1), + hps.data.filter_length, + hps.data.n_mel_channels, + hps.data.sampling_rate, + hps.data.hop_length, + hps.data.win_length, + hps.data.mel_fmin, + hps.data.mel_fmax + ) + + y = commons.slice_segments(y, ids_slice * hps.data.hop_length, hps.train.segment_size) # slice + + # Discriminator + y_d_hat_r, y_d_hat_g, _, _ = net_d(y, y_hat.detach()) + with autocast(enabled=False): + loss_disc, losses_disc_r, losses_disc_g = discriminator_loss(y_d_hat_r, y_d_hat_g) + loss_disc_all = loss_disc + optim_d.zero_grad() + scaler.scale(loss_disc_all).backward() + scaler.unscale_(optim_d) + grad_norm_d = commons.clip_grad_value_(net_d.parameters(), None) + scaler.step(optim_d) + + with autocast(enabled=hps.train.fp16_run): + # Generator + y_d_hat_r, y_d_hat_g, fmap_r, fmap_g = net_d(y, y_hat) + with autocast(enabled=False): + loss_dur = torch.sum(l_length.float()) + loss_mel = F.l1_loss(y_mel, y_hat_mel) * hps.train.c_mel + loss_kl = kl_loss(z_p, logs_q, m_p, logs_p, z_mask) * hps.train.c_kl + + loss_fm = feature_loss(fmap_r, fmap_g) + loss_gen, losses_gen = generator_loss(y_d_hat_g) + loss_gen_all = loss_gen + loss_fm + loss_mel + loss_dur + loss_kl + optim_g.zero_grad() + scaler.scale(loss_gen_all).backward() + scaler.unscale_(optim_g) + grad_norm_g = commons.clip_grad_value_(net_g.parameters(), None) + scaler.step(optim_g) + scaler.update() + + if rank==0: + if global_step % hps.train.log_interval == 0: + lr = optim_g.param_groups[0]['lr'] + losses = [loss_disc, loss_gen, loss_fm, loss_mel, loss_dur, loss_kl] + logger.info('Train Epoch: {} [{:.0f}%]'.format( + epoch, + 100. * batch_idx / len(train_loader))) + logger.info([x.item() for x in losses] + [global_step, lr]) + + scalar_dict = {"loss/g/total": loss_gen_all, "loss/d/total": loss_disc_all, "learning_rate": lr, "grad_norm_g": grad_norm_g} + scalar_dict.update({"loss/g/fm": loss_fm, "loss/g/mel": loss_mel, "loss/g/dur": loss_dur, "loss/g/kl": loss_kl}) + + scalar_dict.update({"loss/g/{}".format(i): v for i, v in enumerate(losses_gen)}) + scalar_dict.update({"loss/d_r/{}".format(i): v for i, v in enumerate(losses_disc_r)}) + scalar_dict.update({"loss/d_g/{}".format(i): v for i, v in enumerate(losses_disc_g)}) + image_dict = { + "slice/mel_org": utils.plot_spectrogram_to_numpy(y_mel[0].data.cpu().numpy()), + "slice/mel_gen": utils.plot_spectrogram_to_numpy(y_hat_mel[0].data.cpu().numpy()), + "all/mel": utils.plot_spectrogram_to_numpy(mel[0].data.cpu().numpy()), + "all/attn": utils.plot_alignment_to_numpy(attn[0,0].data.cpu().numpy()) + } + utils.summarize( + writer=writer, + global_step=global_step, + images=image_dict, + scalars=scalar_dict) + + if global_step % hps.train.eval_interval == 0: + evaluate(hps, net_g, eval_loader, writer_eval) + utils.save_checkpoint(net_g, None, hps.train.learning_rate, epoch, os.path.join(hps.model_dir, "G_{}.pth".format(global_step))) + utils.save_checkpoint(net_g, None, hps.train.learning_rate, epoch, + os.path.join(hps.model_dir, "G_latest.pth".format(global_step))) + # utils.save_checkpoint(net_d, optim_d, hps.train.learning_rate, epoch, os.path.join(hps.model_dir, "D_{}.pth".format(global_step))) + old_g=os.path.join(hps.model_dir, "G_{}.pth".format(global_step-4000)) + # old_d=os.path.join(hps.model_dir, "D_{}.pth".format(global_step-400)) + if os.path.exists(old_g): + os.remove(old_g) + # if os.path.exists(old_d): + # os.remove(old_d) + global_step += 1 + if epoch > hps.max_epochs: + print("Maximum epoch reached, closing training...") + exit() + + if rank == 0: + logger.info('====> Epoch: {}'.format(epoch)) + + +def evaluate(hps, generator, eval_loader, writer_eval): + generator.eval() + with torch.no_grad(): + for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths, speakers) in enumerate(eval_loader): + x, x_lengths = x.cuda(0), x_lengths.cuda(0) + spec, spec_lengths = spec.cuda(0), spec_lengths.cuda(0) + y, y_lengths = y.cuda(0), y_lengths.cuda(0) + speakers = speakers.cuda(0) + + # remove else + x = x[:1] + x_lengths = x_lengths[:1] + spec = spec[:1] + spec_lengths = spec_lengths[:1] + y = y[:1] + y_lengths = y_lengths[:1] + speakers = speakers[:1] + break + y_hat, attn, mask, *_ = generator.module.infer(x, x_lengths, speakers, max_len=1000) + y_hat_lengths = mask.sum([1,2]).long() * hps.data.hop_length + + mel = spec_to_mel_torch( + spec, + hps.data.filter_length, + hps.data.n_mel_channels, + hps.data.sampling_rate, + hps.data.mel_fmin, + hps.data.mel_fmax) + y_hat_mel = mel_spectrogram_torch( + y_hat.squeeze(1).float(), + hps.data.filter_length, + hps.data.n_mel_channels, + hps.data.sampling_rate, + hps.data.hop_length, + hps.data.win_length, + hps.data.mel_fmin, + hps.data.mel_fmax + ) + image_dict = { + "gen/mel": utils.plot_spectrogram_to_numpy(y_hat_mel[0].cpu().numpy()) + } + audio_dict = { + "gen/audio": y_hat[0,:,:y_hat_lengths[0]] + } + if global_step == 0: + image_dict.update({"gt/mel": utils.plot_spectrogram_to_numpy(mel[0].cpu().numpy())}) + audio_dict.update({"gt/audio": y[0,:,:y_lengths[0]]}) + + utils.summarize( + writer=writer_eval, + global_step=global_step, + images=image_dict, + audios=audio_dict, + audio_sampling_rate=hps.data.sampling_rate + ) + generator.train() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/long_audio_transcribe.py b/long_audio_transcribe.py new file mode 100644 index 0000000..c4aee3b --- /dev/null +++ b/long_audio_transcribe.py @@ -0,0 +1,61 @@ +from moviepy.editor import AudioFileClip +import whisper +import os +import torchaudio +import librosa +import torch +import argparse +parent_dir = "./denoised_audio/" +filelist = list(os.walk(parent_dir))[0][2] +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--languages", default="CJE") + args = parser.parse_args() + if args.languages == "CJE": + lang2token = { + 'zh': "[ZH]", + 'ja': "[JA]", + "en": "[EN]", + } + elif args.languages == "CJ": + lang2token = { + 'zh': "[ZH]", + 'ja': "[JA]", + } + model = whisper.load_model("large") + speaker_annos = [] + for file in filelist: + print(f"transcribing {parent_dir + file}...\n") + options = dict(beam_size=5, best_of=5) + transcribe_options = dict(task="transcribe", **options) + result = model.transcribe(parent_dir + file, **transcribe_options) + segments = result["segments"] + # result = model.transcribe(parent_dir + file) + lang = result['language'] + if result['language'] not in list(lang2token.keys()): + print(f"{lang} not supported, ignoring...\n") + # segment audio based on segment results + character_name = file.rstrip(".wav").split("_")[0] + code = file.rstrip(".wav").split("_")[1] + if not os.path.exists("./segmented_character_voice/" + character_name): + os.mkdir("./segmented_character_voice/" + character_name) + wav, sr = torchaudio.load(parent_dir + file, frame_offset=0, num_frames=-1, normalize=True, + channels_first=True) + + for i, seg in enumerate(result['segments']): + start_time = seg['start'] + end_time = seg['end'] + text = seg['text'] + text = lang2token[lang] + text.replace("\n", "") + lang2token[lang] + text = text + "\n" + wav_seg = wav[:, int(start_time*sr):int(end_time*sr)] + wav_seg_name = f"{character_name}_{code}_{i}.wav" + savepth = "./segmented_character_voice/" + character_name + "/" + wav_seg_name + speaker_annos.append(savepth + "|" + character_name + "|" + text) + # trimmed_wav_seg = librosa.effects.trim(wav_seg.squeeze().numpy()) + # trimmed_wav_seg = torch.tensor(trimmed_wav_seg[0]).unsqueeze(0) + torchaudio.save(savepth, wav_seg, 22050, channels_first=True) + + with open("long_character_anno.txt", 'w', encoding='utf-8') as f: + for line in speaker_annos: + f.write(line) diff --git a/preprocess_v2.py b/preprocess_v2.py new file mode 100644 index 0000000..14bc16b --- /dev/null +++ b/preprocess_v2.py @@ -0,0 +1,127 @@ +import os +import argparse +import json +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--add_auxiliary_data", type=bool, help="Whether to add extra data as fine-tuning helper") + args = parser.parse_args() + + new_annos = [] + # Source 1: transcribed short audios + if os.path.exists("short_character_anno.txt"): + with open("short_character_anno.txt", 'r', encoding='utf-8') as f: + short_character_anno = f.readlines() + new_annos += short_character_anno + # Source 2: transcribed long audio segments + if os.path.exists("long_character_anno.txt"): + with open("long_character_anno.txt", 'r', encoding='utf-8') as f: + long_character_anno = f.readlines() + new_annos += long_character_anno + + # Get all speaker names + speakers = [] + for line in new_annos: + path, speaker, text = line.split("|") + if speaker not in speakers: + speakers.append(speaker) + assert (len(speakers) != 0), "no speaker found" + # Source 3 (Optional): sampled audios as extra training helpers + if args.add_auxiliary_data: + with open("sampled_audio4ft.txt", 'r', encoding='utf-8') as f: + old_annos = f.readlines() + num_old_voices = len(old_annos) + num_new_voices = len(new_annos) + # STEP 1: balance number of new & old voices + cc_duplicate = num_old_voices // num_new_voices + if cc_duplicate == 0: + cc_duplicate = 1 + + + # STEP 2: modify config file + with open("./configs/finetune_speaker.json", 'r', encoding='utf-8') as f: + hps = json.load(f) + + # assign ids to new speakers + speaker2id = {} + for i, speaker in enumerate(speakers): + speaker2id[speaker] = hps['data']["n_speakers"] + i + # modify n_speakers + hps['data']["n_speakers"] = hps['data']["n_speakers"] + len(speakers) + # add speaker names + for speaker in speakers: + hps['speakers'][speaker] = speaker2id[speaker] + hps['train']['log_interval'] = 100 + hps['train']['eval_interval'] = 1000 + hps['train']['batch_size'] = 16 + hps['data']['training_files'] = "final_annotation_train.txt" + hps['data']['validation_files'] = "final_annotation_val.txt" + # save modified config + with open("./configs/modified_finetune_speaker.json", 'w', encoding='utf-8') as f: + json.dump(hps, f, indent=2) + + # STEP 3: clean annotations, replace speaker names with assigned speaker IDs + import text + cleaned_new_annos = [] + for i, line in enumerate(new_annos): + path, speaker, txt = line.split("|") + if len(txt) > 150: + continue + cleaned_text = text._clean_text(txt, hps['data']['text_cleaners']) + cleaned_text += "\n" if not cleaned_text.endswith("\n") else "" + cleaned_new_annos.append(path + "|" + str(speaker2id[speaker]) + "|" + cleaned_text) + # merge with old annotation + final_annos = old_annos + cc_duplicate * cleaned_new_annos + # save annotation file + with open("final_annotation_train.txt", 'w', encoding='utf-8') as f: + for line in final_annos: + f.write(line) + # save annotation file for validation + with open("final_annotation_val.txt", 'w', encoding='utf-8') as f: + for line in cleaned_new_annos: + f.write(line) + print("finished") + else: + # Do not add extra helper data + # STEP 1: modify config file + with open("./configs/finetune_speaker.json", 'r', encoding='utf-8') as f: + hps = json.load(f) + + # assign ids to new speakers + speaker2id = {} + for i, speaker in enumerate(speakers): + speaker2id[speaker] = i + # modify n_speakers + hps['data']["n_speakers"] = len(speakers) + # overwrite speaker names + hps['speakers'] = speaker2id + hps['train']['log_interval'] = 10 + hps['train']['eval_interval'] = 100 + hps['train']['batch_size'] = 16 + hps['data']['training_files'] = "final_annotation_train.txt" + hps['data']['validation_files'] = "final_annotation_val.txt" + # save modified config + with open("./configs/modified_finetune_speaker.json", 'w', encoding='utf-8') as f: + json.dump(hps, f, indent=2) + + # STEP 2: clean annotations, replace speaker names with assigned speaker IDs + import text + + cleaned_new_annos = [] + for i, line in enumerate(new_annos): + path, speaker, txt = line.split("|") + if len(txt) > 150: + continue + cleaned_text = text._clean_text(txt, hps['data']['text_cleaners']) + cleaned_text += "\n" if not cleaned_text.endswith("\n") else "" + cleaned_new_annos.append(path + "|" + str(speaker2id[speaker]) + "|" + cleaned_text) + + final_annos = cleaned_new_annos + # save annotation file + with open("final_annotation_train.txt", 'w', encoding='utf-8') as f: + for line in final_annos: + f.write(line) + # save annotation file for validation + with open("final_annotation_val.txt", 'w', encoding='utf-8') as f: + for line in cleaned_new_annos: + f.write(line) + print("finished") \ No newline at end of file diff --git a/rearrange_speaker.py b/rearrange_speaker.py new file mode 100644 index 0000000..afe77f7 --- /dev/null +++ b/rearrange_speaker.py @@ -0,0 +1,33 @@ +import torch +import argparse +import json + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--model_dir", type=str, default="./OUTPUT_MODEL/G_latest.pth") + parser.add_argument("--config_dir", type=str, default="./configs/modified_finetune_speaker.json") + args = parser.parse_args() + + model_sd = torch.load(args.model_dir, map_location='cpu') + with open(args.config_dir, 'r', encoding='utf-8') as f: + hps = json.load(f) + + valid_speakers = list(hps['speakers'].keys()) + if hps['data']['n_speakers'] > len(valid_speakers): + new_emb_g = torch.zeros([len(valid_speakers), 256]) + old_emb_g = model_sd['model']['emb_g.weight'] + for i, speaker in enumerate(valid_speakers): + new_emb_g[i, :] = old_emb_g[hps['speakers'][speaker], :] + hps['speakers'][speaker] = i + hps['data']['n_speakers'] = len(valid_speakers) + model_sd['model']['emb_g.weight'] = new_emb_g + with open("./finetune_speaker.json", 'w', encoding='utf-8') as f: + json.dump(hps, f, indent=2) + torch.save("./G_latest.pth", args.model_dir) + # save another config file copy in MoeGoe format + hps['speakers'] = valid_speakers + with open("./moegoe_config.json", 'w', encoding='utf-8') as f: + json.dump(hps, f, indent=2) + + + diff --git a/short_audio_transcribe.py b/short_audio_transcribe.py new file mode 100644 index 0000000..0fc60c7 --- /dev/null +++ b/short_audio_transcribe.py @@ -0,0 +1,100 @@ +import whisper +import os +import torchaudio +import argparse + +lang2token = { + 'zh': "[ZH]", + 'ja': "[JA]", + "en": "[EN]", + } +def transcribe_one(audio_path): + # load audio and pad/trim it to fit 30 seconds + audio = whisper.load_audio(audio_path) + audio = whisper.pad_or_trim(audio) + + # make log-Mel spectrogram and move to the same device as the model + mel = whisper.log_mel_spectrogram(audio).to(model.device) + + # detect the spoken language + _, probs = model.detect_language(mel) + print(f"Detected language: {max(probs, key=probs.get)}") + lang = max(probs, key=probs.get) + # decode the audio + options = whisper.DecodingOptions() + result = whisper.decode(model, mel, options) + + # print the recognized text + print(result.text) + return lang, result.text +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--languages", default="CJE") + args = parser.parse_args() + if args.languages == "CJE": + lang2token = { + 'zh': "[ZH]", + 'ja': "[JA]", + "en": "[EN]", + } + elif args.languages == "CJ": + lang2token = { + 'zh': "[ZH]", + 'ja': "[JA]", + } + model = whisper.load_model("large") + parent_dir = "./custom_character_voice/" + speaker_names = list(os.walk(parent_dir))[0][1] + speaker_annos = [] + # resample audios + for speaker in speaker_names: + for i, wavfile in enumerate(list(os.walk(parent_dir + speaker))[0][2]): + # try to load file as audio + if wavfile.startswith("processed_"): + continue + try: + wav, sr = torchaudio.load(parent_dir + speaker + "/" + wavfile, frame_offset=0, num_frames=-1, normalize=True, + channels_first=True) + wav = wav.mean(dim=0).unsqueeze(0) + if sr != 22050: + wav = torchaudio.transforms.Resample(orig_freq=sr, new_freq=22050)(wav) + if wav.shape[1] / sr > 20: + print(f"{wavfile} too long, ignoring\n") + save_path = parent_dir + speaker + "/" + f"processed_{i}.wav" + torchaudio.save(save_path, wav, 22050, channels_first=True) + # transcribe text + lang, text = transcribe_one(save_path) + if lang not in list(lang2token.keys()): + print(f"{lang} not supported, ignoring\n") + text = lang2token[lang] + text + lang2token[lang] + "\n" + speaker_annos.append(save_path + "|" + speaker + "|" + text) + except: + continue + + # # clean annotation + # import argparse + # import text + # from utils import load_filepaths_and_text + # for i, line in enumerate(speaker_annos): + # path, sid, txt = line.split("|") + # cleaned_text = text._clean_text(txt, ["cjke_cleaners2"]) + # cleaned_text += "\n" if not cleaned_text.endswith("\n") else "" + # speaker_annos[i] = path + "|" + sid + "|" + cleaned_text + # write into annotation + with open("short_character_anno.txt", 'w', encoding='utf-8') as f: + for line in speaker_annos: + f.write(line) + + # import json + # # generate new config + # with open("./configs/finetune_speaker.json", 'r', encoding='utf-8') as f: + # hps = json.load(f) + # # modify n_speakers + # hps['data']["n_speakers"] = 1000 + len(speaker2id) + # # add speaker names + # for speaker in speaker_names: + # hps['speakers'][speaker] = speaker2id[speaker] + # # save modified config + # with open("./configs/modified_finetune_speaker.json", 'w', encoding='utf-8') as f: + # json.dump(hps, f, indent=2) + # print("finished") diff --git a/text/__init__.py b/text/__init__.py index b3455d6..e6798ed 100644 --- a/text/__init__.py +++ b/text/__init__.py @@ -30,14 +30,15 @@ def text_to_sequence(text, symbols, cleaner_names): return sequence -def cleaned_text_to_sequence(cleaned_text): +def cleaned_text_to_sequence(cleaned_text, symbols): '''Converts a string of text to a sequence of IDs corresponding to the symbols in the text. Args: text: string to convert to a sequence Returns: List of integers corresponding to the symbols in the text ''' - sequence = [_symbol_to_id[symbol] for symbol in cleaned_text if symbol in _symbol_to_id.keys()] + symbol_to_id = {s: i for i, s in enumerate(symbols)} + sequence = [symbol_to_id[symbol] for symbol in cleaned_text if symbol in symbol_to_id.keys()] return sequence diff --git a/utils.py b/utils.py index 478af8f..c9ba316 100644 --- a/utils.py +++ b/utils.py @@ -15,7 +15,7 @@ logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) logger = logging -def load_checkpoint(checkpoint_path, model, optimizer=None): +def load_checkpoint(checkpoint_path, model, optimizer=None, drop_speaker_emb=False): assert os.path.isfile(checkpoint_path) checkpoint_dict = torch.load(checkpoint_path, map_location='cpu') iteration = checkpoint_dict['iteration'] @@ -31,8 +31,10 @@ def load_checkpoint(checkpoint_path, model, optimizer=None): for k, v in state_dict.items(): try: if k == 'emb_g.weight': + if drop_speaker_emb: + new_state_dict[k] = v + continue v[:saved_state_dict[k].shape[0], :] = saved_state_dict[k] - # v[999, :] = saved_state_dict[k][154, :] new_state_dict[k] = v else: new_state_dict[k] = saved_state_dict[k] @@ -154,6 +156,7 @@ def get_hparams(init=True): help='Model name') parser.add_argument('-n', '--max_epochs', type=int, default=50, help='finetune epochs') + parser.add_argument('--drop_speaker_embed', type=bool, default=False, help='whether to drop existing characters') args = parser.parse_args() model_dir = os.path.join("./", args.model) @@ -176,6 +179,7 @@ def get_hparams(init=True): hparams = HParams(**config) hparams.model_dir = model_dir hparams.max_epochs = args.max_epochs + hparams.drop_speaker_embed = args.drop_speaker_embed return hparams diff --git a/video2audio.py b/video2audio.py new file mode 100644 index 0000000..411c861 --- /dev/null +++ b/video2audio.py @@ -0,0 +1,10 @@ +from moviepy.editor import AudioFileClip +import os +video_dir = "./video_data/" +audio_dir = "./raw_audio/" +filelist = list(os.walk(video_dir))[0][2] +for file in filelist: + if file.endswith(".mp4"): + my_audio_clip = AudioFileClip(video_dir + file) + my_audio_clip.write_audiofile(audio_dir + file.rstrip(".mp4") + ".wav") + diff --git a/video_transcribe.py b/video_transcribe.py deleted file mode 100644 index c820d8a..0000000 --- a/video_transcribe.py +++ /dev/null @@ -1,43 +0,0 @@ -from moviepy.editor import AudioFileClip -import whisper -import os -import torchaudio -parent_dir = "../" -filelist = ["taffy1.mp4", "taffy2.mp4"] -for file in filelist: - my_audio_clip = AudioFileClip(parent_dir + file) - my_audio_clip.write_audiofile(parent_dir + file.rstrip(".mp4") + ".wav") -for file in filelist: - file = file.replace(".mp4", ".wav") - os.system(f"demucs --two-stems=vocals {parent_dir}{file}") -for file in filelist: - file = file.strip(".mp4") - wav, sr = torchaudio.load(f"./separated/htdemucs/{file}/vocals.wav", frame_offset=0, num_frames=-1, normalize=True, - channels_first=True) - # merge two channels into one - wav = wav.mean(dim=0).unsqueeze(0) - if sr != 22050: - wav = torchaudio.transforms.Resample(orig_freq=sr, new_freq=22050)(wav) - torchaudio.save(file + ".wav", wav, 22050, channels_first=True) -model = whisper.load_model("medium") -def transcribe_one(audio_path): - # load audio and pad/trim it to fit 30 seconds - audio = whisper.load_audio(audio_path) - audio = whisper.pad_or_trim(audio) - - # make log-Mel spectrogram and move to the same device as the model - mel = whisper.log_mel_spectrogram(audio).to(model.device) - - # detect the spoken language - _, probs = model.detect_language(mel) - lang = max(probs, key=probs.get) - # decode the audio - options = whisper.DecodingOptions() - result = whisper.decode(model, mel, options) - - # print the recognized text - return result - -result = model.transcribe("taffy2.wav") -# segment audio based on segment results - diff --git a/voice_upload.py b/voice_upload.py index 8de1fb2..5c825a9 100644 --- a/voice_upload.py +++ b/voice_upload.py @@ -1,14 +1,28 @@ from google.colab import files import shutil import os -basepath = os.getcwd() -uploaded = files.upload() # 上传文件 -upload_path = "./custom_character_voice/" -if not os.path.exists(upload_path): - os.mkdir(upload_path) -for filename in uploaded.keys(): - #将上传的文件移动到指定的位置上 - if filename.endswith(".zip"): - shutil.move(os.path.join(basepath, filename), os.path.join(upload_path, "custom_character_voice.zip")) - elif filename.endswith(".rar"): - shutil.move(os.path.join(basepath, filename), os.path.join(upload_path, "custom_character_voice.rar")) \ No newline at end of file +import argparse +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--type", type=str, required=True, help="type of file to upload") + args = parser.parse_args() + file_type = args.type + + basepath = os.getcwd() + uploaded = files.upload() # 上传文件 + assert(file_type in ['zip', 'audio', 'video']) + if file_type == "zip": + upload_path = "./custom_character_voice/" + for filename in uploaded.keys(): + #将上传的文件移动到指定的位置上 + shutil.move(os.path.join(basepath, filename), os.path.join(upload_path, "custom_character_voice.zip")) + elif file_type == "audio": + upload_path = "./raw_audio/" + for filename in uploaded.keys(): + #将上传的文件移动到指定的位置上 + shutil.move(os.path.join(basepath, filename), os.path.join(upload_path, filename)) + elif file_type == "video": + upload_path = "./video_data/" + for filename in uploaded.keys(): + # 将上传的文件移动到指定的位置上 + shutil.move(os.path.join(basepath, filename), os.path.join(upload_path, filename)) \ No newline at end of file