rearranged repo

scripts/VC_inference.py

@@ -0,0 +1,139 @@
+import os
+import numpy as np
+import torch
+from torch import no_grad, LongTensor
+import argparse
+import commons
+from mel_processing import spectrogram_torch
+import utils
+from models import SynthesizerTrn
+import gradio as gr
+import librosa
+import webbrowser
+
+from text import text_to_sequence, _clean_text
+device = "cuda:0" if torch.cuda.is_available() else "cpu"
+language_marks = {
+    "Japanese": "",
+    "日本語": "[JA]",
+    "简体中文": "[ZH]",
+    "English": "[EN]",
+    "Mix": "",
+}
+lang = ['日本語', '简体中文', 'English', 'Mix']
+def get_text(text, hps, is_symbol):
+    text_norm = text_to_sequence(text, hps.symbols, [] if is_symbol else hps.data.text_cleaners)
+    if hps.data.add_blank:
+        text_norm = commons.intersperse(text_norm, 0)
+    text_norm = LongTensor(text_norm)
+    return text_norm
+
+def create_tts_fn(model, hps, speaker_ids):
+    def tts_fn(text, speaker, language, speed):
+        if language is not None:
+            text = language_marks[language] + text + language_marks[language]
+        speaker_id = speaker_ids[speaker]
+        stn_tst = get_text(text, hps, False)
+        with no_grad():
+            x_tst = stn_tst.unsqueeze(0).to(device)
+            x_tst_lengths = LongTensor([stn_tst.size(0)]).to(device)
+            sid = LongTensor([speaker_id]).to(device)
+            audio = model.infer(x_tst, x_tst_lengths, sid=sid, noise_scale=.667, noise_scale_w=0.8,
+                                length_scale=1.0 / speed)[0][0, 0].data.cpu().float().numpy()
+        del stn_tst, x_tst, x_tst_lengths, sid
+        return "Success", (hps.data.sampling_rate, audio)
+
+    return tts_fn
+
+def create_vc_fn(model, hps, speaker_ids):
+    def vc_fn(original_speaker, target_speaker, record_audio, upload_audio):
+        input_audio = record_audio if record_audio is not None else upload_audio
+        if input_audio is None:
+            return "You need to record or upload an audio", None
+        sampling_rate, audio = input_audio
+        original_speaker_id = speaker_ids[original_speaker]
+        target_speaker_id = speaker_ids[target_speaker]
+
+        audio = (audio / np.iinfo(audio.dtype).max).astype(np.float32)
+        if len(audio.shape) > 1:
+            audio = librosa.to_mono(audio.transpose(1, 0))
+        if sampling_rate != hps.data.sampling_rate:
+            audio = librosa.resample(audio, orig_sr=sampling_rate, target_sr=hps.data.sampling_rate)
+        with no_grad():
+            y = torch.FloatTensor(audio)
+            y = y / max(-y.min(), y.max()) / 0.99
+            y = y.to(device)
+            y = y.unsqueeze(0)
+            spec = spectrogram_torch(y, hps.data.filter_length,
+                                     hps.data.sampling_rate, hps.data.hop_length, hps.data.win_length,
+                                     center=False).to(device)
+            spec_lengths = LongTensor([spec.size(-1)]).to(device)
+            sid_src = LongTensor([original_speaker_id]).to(device)
+            sid_tgt = LongTensor([target_speaker_id]).to(device)
+            audio = model.voice_conversion(spec, spec_lengths, sid_src=sid_src, sid_tgt=sid_tgt)[0][
+                0, 0].data.cpu().float().numpy()
+        del y, spec, spec_lengths, sid_src, sid_tgt
+        return "Success", (hps.data.sampling_rate, audio)
+
+    return vc_fn
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model_dir", default="./G_latest.pth", help="directory to your fine-tuned model")
+    parser.add_argument("--config_dir", default="./finetune_speaker.json", help="directory to your model config file")
+    parser.add_argument("--share", default=False, help="make link public (used in colab)")
+
+    args = parser.parse_args()
+    hps = utils.get_hparams_from_file(args.config_dir)
+
+
+    net_g = SynthesizerTrn(
+        len(hps.symbols),
+        hps.data.filter_length // 2 + 1,
+        hps.train.segment_size // hps.data.hop_length,
+        n_speakers=hps.data.n_speakers,
+        **hps.model).to(device)
+    _ = net_g.eval()
+
+    _ = utils.load_checkpoint(args.model_dir, net_g, None)
+    speaker_ids = hps.speakers
+    speakers = list(hps.speakers.keys())
+    tts_fn = create_tts_fn(net_g, hps, speaker_ids)
+    vc_fn = create_vc_fn(net_g, hps, speaker_ids)
+    app = gr.Blocks()
+    with app:
+        with gr.Tab("Text-to-Speech"):
+            with gr.Row():
+                with gr.Column():
+                    textbox = gr.TextArea(label="Text",
+                                          placeholder="Type your sentence here",
+                                          value="こんにちわ。", elem_id=f"tts-input")
+                    # select character
+                    char_dropdown = gr.Dropdown(choices=speakers, value=speakers[0], label='character')
+                    language_dropdown = gr.Dropdown(choices=lang, value=lang[0], label='language')
+                    duration_slider = gr.Slider(minimum=0.1, maximum=5, value=1, step=0.1,
+                                                label='速度 Speed')
+                with gr.Column():
+                    text_output = gr.Textbox(label="Message")
+                    audio_output = gr.Audio(label="Output Audio", elem_id="tts-audio")
+                    btn = gr.Button("Generate!")
+                    btn.click(tts_fn,
+                              inputs=[textbox, char_dropdown, language_dropdown, duration_slider,],
+                              outputs=[text_output, audio_output])
+        with gr.Tab("Voice Conversion"):
+            gr.Markdown("""
+                            录制或上传声音，并选择要转换的音色。
+            """)
+            with gr.Column():
+                record_audio = gr.Audio(label="record your voice", source="microphone")
+                upload_audio = gr.Audio(label="or upload audio here", source="upload")
+                source_speaker = gr.Dropdown(choices=speakers, value=speakers[0], label="source speaker")
+                target_speaker = gr.Dropdown(choices=speakers, value=speakers[0], label="target speaker")
+            with gr.Column():
+                message_box = gr.Textbox(label="Message")
+                converted_audio = gr.Audio(label='converted audio')
+            btn = gr.Button("Convert!")
+            btn.click(vc_fn, inputs=[source_speaker, target_speaker, record_audio, upload_audio],
+                      outputs=[message_box, converted_audio])
+    webbrowser.open("http://127.0.0.1:7860")
+    app.launch(share=args.share)
+

scripts/denoise_audio.py

@@ -1,11 +1,11 @@
 import os
 import json
 import torchaudio
-raw_audio_dir = "./raw_audio/"
-denoise_audio_dir = "./denoised_audio/"
+raw_audio_dir = "../raw_audio/"
+denoise_audio_dir = "../denoised_audio/"
 filelist = list(os.walk(raw_audio_dir))[0][2]
 # 2023/4/21: Get the target sampling rate
-with open("./configs/finetune_speaker.json", 'r', encoding='utf-8') as f:
+with open("../configs/finetune_speaker.json", 'r', encoding='utf-8') as f:
     hps = json.load(f)
 target_sr = hps['data']['sampling_rate']
 for file in filelist:

scripts/long_audio_transcribe.py

@@ -6,7 +6,7 @@ import torchaudio
 import librosa
 import torch
 import argparse
-parent_dir = "./denoised_audio/"
+parent_dir = "../denoised_audio/"
 filelist = list(os.walk(parent_dir))[0][2]
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
@@ -29,7 +29,7 @@ if __name__ == "__main__":
             'zh': "[ZH]",
         }
     assert(torch.cuda.is_available()), "Please enable GPU in order to run Whisper!"
-    with open("./configs/finetune_speaker.json", 'r', encoding='utf-8') as f:
+    with open("../configs/finetune_speaker.json", 'r', encoding='utf-8') as f:
         hps = json.load(f)
     target_sr = hps['data']['sampling_rate']
     model = whisper.load_model(args.whisper_size)
@@ -70,6 +70,6 @@ if __name__ == "__main__":
     if len(speaker_annos) == 0:
         print("Warning: no long audios & videos found, this IS expected if you have only uploaded short audios")
         print("this IS NOT expected if you have uploaded any long audios, videos or video links. Please check your file structure or make sure your audio/video language is supported.")
-    with open("long_character_anno.txt", 'w', encoding='utf-8') as f:
+    with open("../long_character_anno.txt", 'w', encoding='utf-8') as f:
         for line in speaker_annos:
             f.write(line)

scripts/preprocess_v2.py

@@ -0,0 +1,151 @@
+import os
+import argparse
+import json
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--add_auxiliary_data", type=bool, help="Whether to add extra data as fine-tuning helper")
+    parser.add_argument("--languages", default="CJE")
+    args = parser.parse_args()
+    if args.languages == "CJE":
+        langs = ["[ZH]", "[JA]", "[EN]"]
+    elif args.languages == "CJ":
+        langs = ["[ZH]", "[JA]"]
+    elif args.languages == "C":
+        langs = ["[ZH]"]
+    new_annos = []
+    # Source 1: transcribed short audios
+    if os.path.exists("short_character_anno.txt"):
+        with open("short_character_anno.txt", 'r', encoding='utf-8') as f:
+            short_character_anno = f.readlines()
+            new_annos += short_character_anno
+    # Source 2: transcribed long audio segments
+    if os.path.exists("../long_character_anno.txt"):
+        with open("../long_character_anno.txt", 'r', encoding='utf-8') as f:
+            long_character_anno = f.readlines()
+            new_annos += long_character_anno
+
+    # Get all speaker names
+    speakers = []
+    for line in new_annos:
+        path, speaker, text = line.split("|")
+        if speaker not in speakers:
+            speakers.append(speaker)
+    assert (len(speakers) != 0), "No audio file found. Please check your uploaded file structure."
+    # Source 3 (Optional): sampled audios as extra training helpers
+    if args.add_auxiliary_data:
+        with open("../sampled_audio4ft.txt", 'r', encoding='utf-8') as f:
+            old_annos = f.readlines()
+        # filter old_annos according to supported languages
+        filtered_old_annos = []
+        for line in old_annos:
+            for lang in langs:
+                if lang in line:
+                    filtered_old_annos.append(line)
+        old_annos = filtered_old_annos
+        for line in old_annos:
+            path, speaker, text = line.split("|")
+            if speaker not in speakers:
+                speakers.append(speaker)
+        num_old_voices = len(old_annos)
+        num_new_voices = len(new_annos)
+        # STEP 1: balance number of new & old voices
+        cc_duplicate = num_old_voices // num_new_voices
+        if cc_duplicate == 0:
+            cc_duplicate = 1
+
+
+        # STEP 2: modify config file
+        with open("../configs/finetune_speaker.json", 'r', encoding='utf-8') as f:
+            hps = json.load(f)
+
+        # assign ids to new speakers
+        speaker2id = {}
+        for i, speaker in enumerate(speakers):
+            speaker2id[speaker] = i
+        # modify n_speakers
+        hps['data']["n_speakers"] = len(speakers)
+        # overwrite speaker names
+        hps['speakers'] = speaker2id
+        hps['train']['log_interval'] = 100
+        hps['train']['eval_interval'] = 1000
+        hps['train']['batch_size'] = 16
+        hps['data']['training_files'] = "final_annotation_train.txt"
+        hps['data']['validation_files'] = "final_annotation_val.txt"
+        # save modified config
+        with open("../configs/modified_finetune_speaker.json", 'w', encoding='utf-8') as f:
+            json.dump(hps, f, indent=2)
+
+        # STEP 3: clean annotations, replace speaker names with assigned speaker IDs
+        import text
+        cleaned_new_annos = []
+        for i, line in enumerate(new_annos):
+            path, speaker, txt = line.split("|")
+            if len(txt) > 150:
+                continue
+            cleaned_text = text._clean_text(txt, hps['data']['text_cleaners'])
+            cleaned_text += "\n" if not cleaned_text.endswith("\n") else ""
+            cleaned_new_annos.append(path + "|" + str(speaker2id[speaker]) + "|" + cleaned_text)
+        cleaned_old_annos = []
+        for i, line in enumerate(old_annos):
+            path, speaker, txt = line.split("|")
+            if len(txt) > 150:
+                continue
+            cleaned_text = text._clean_text(txt, hps['data']['text_cleaners'])
+            cleaned_text += "\n" if not cleaned_text.endswith("\n") else ""
+            cleaned_old_annos.append(path + "|" + str(speaker2id[speaker]) + "|" + cleaned_text)
+        # merge with old annotation
+        final_annos = cleaned_old_annos + cc_duplicate * cleaned_new_annos
+        # save annotation file
+        with open("../final_annotation_train.txt", 'w', encoding='utf-8') as f:
+            for line in final_annos:
+                f.write(line)
+        # save annotation file for validation
+        with open("../final_annotation_val.txt", 'w', encoding='utf-8') as f:
+            for line in cleaned_new_annos:
+                f.write(line)
+        print("finished")
+    else:
+        # Do not add extra helper data
+        # STEP 1: modify config file
+        with open("../configs/finetune_speaker.json", 'r', encoding='utf-8') as f:
+            hps = json.load(f)
+
+        # assign ids to new speakers
+        speaker2id = {}
+        for i, speaker in enumerate(speakers):
+            speaker2id[speaker] = i
+        # modify n_speakers
+        hps['data']["n_speakers"] = len(speakers)
+        # overwrite speaker names
+        hps['speakers'] = speaker2id
+        hps['train']['log_interval'] = 10
+        hps['train']['eval_interval'] = 100
+        hps['train']['batch_size'] = 16
+        hps['data']['training_files'] = "final_annotation_train.txt"
+        hps['data']['validation_files'] = "final_annotation_val.txt"
+        # save modified config
+        with open("../configs/modified_finetune_speaker.json", 'w', encoding='utf-8') as f:
+            json.dump(hps, f, indent=2)
+
+        # STEP 2: clean annotations, replace speaker names with assigned speaker IDs
+        import text
+
+        cleaned_new_annos = []
+        for i, line in enumerate(new_annos):
+            path, speaker, txt = line.split("|")
+            if len(txt) > 150:
+                continue
+            cleaned_text = text._clean_text(txt, hps['data']['text_cleaners']).replace("[ZH]", "")
+            cleaned_text += "\n" if not cleaned_text.endswith("\n") else ""
+            cleaned_new_annos.append(path + "|" + str(speaker2id[speaker]) + "|" + cleaned_text)
+
+        final_annos = cleaned_new_annos
+        # save annotation file
+        with open("../final_annotation_train.txt", 'w', encoding='utf-8') as f:
+            for line in final_annos:
+                f.write(line)
+        # save annotation file for validation
+        with open("../final_annotation_val.txt", 'w', encoding='utf-8') as f:
+            for line in cleaned_new_annos:
+                f.write(line)
+        print("finished")

scripts/rearrange_speaker.py

@@ -21,16 +21,16 @@ if __name__ == "__main__":
             hps['speakers'][speaker] = i
         hps['data']['n_speakers'] = len(valid_speakers)
         model_sd['model']['emb_g.weight'] = new_emb_g
-        with open("./finetune_speaker.json", 'w', encoding='utf-8') as f:
+        with open("../finetune_speaker.json", 'w', encoding='utf-8') as f:
             json.dump(hps, f, indent=2)
-        torch.save(model_sd, "./G_latest.pth")
+        torch.save(model_sd, "../G_latest.pth")
     else:
-        with open("./finetune_speaker.json", 'w', encoding='utf-8') as f:
+        with open("../finetune_speaker.json", 'w', encoding='utf-8') as f:
             json.dump(hps, f, indent=2)
-        torch.save(model_sd, "./G_latest.pth")
+        torch.save(model_sd, "../G_latest.pth")
     # save another config file copy in MoeGoe format
     hps['speakers'] = valid_speakers
-    with open("./moegoe_config.json", 'w', encoding='utf-8') as f:
+    with open("../moegoe_config.json", 'w', encoding='utf-8') as f:
         json.dump(hps, f, indent=2)
 
 

scripts/short_audio_transcribe.py

@@ -51,12 +51,12 @@ if __name__ == "__main__":
         }
     assert (torch.cuda.is_available()), "Please enable GPU in order to run Whisper!"
     model = whisper.load_model(args.whisper_size)
-    parent_dir = "./custom_character_voice/"
+    parent_dir = "../custom_character_voice/"
     speaker_names = list(os.walk(parent_dir))[0][1]
     speaker_annos = []
     # resample audios
     # 2023/4/21: Get the target sampling rate
-    with open("./configs/finetune_speaker.json", 'r', encoding='utf-8') as f:
+    with open("../configs/finetune_speaker.json", 'r', encoding='utf-8') as f:
         hps = json.load(f)
     target_sr = hps['data']['sampling_rate']
     for speaker in speaker_names:

scripts/video2audio.py

@@ -3,8 +3,8 @@ from concurrent.futures import ThreadPoolExecutor
 
 from moviepy.editor import AudioFileClip
 
-video_dir = "./video_data/"
-audio_dir = "./raw_audio/"
+video_dir = "../video_data/"
+audio_dir = "../raw_audio/"
 filelist = list(os.walk(video_dir))[0][2]
 
 

scripts/voice_upload.py

@@ -12,17 +12,17 @@ if __name__ == "__main__":
     uploaded = files.upload() # 上传文件
     assert(file_type in ['zip', 'audio', 'video'])
     if file_type == "zip":
-        upload_path = "./custom_character_voice/"
+        upload_path = "../custom_character_voice/"
         for filename in uploaded.keys():
             #将上传的文件移动到指定的位置上
             shutil.move(os.path.join(basepath, filename), os.path.join(upload_path, "custom_character_voice.zip"))
     elif file_type == "audio":
-        upload_path = "./raw_audio/"
+        upload_path = "../raw_audio/"
         for filename in uploaded.keys():
             #将上传的文件移动到指定的位置上
             shutil.move(os.path.join(basepath, filename), os.path.join(upload_path, filename))
     elif file_type == "video":
-        upload_path = "./video_data/"
+        upload_path = "../video_data/"
         for filename in uploaded.keys():
             # 将上传的文件移动到指定的位置上
             shutil.move(os.path.join(basepath, filename), os.path.join(upload_path, filename))