rearranged repo

scripts/denoise_audio.py

@@ -0,0 +1,22 @@
+import os
+import json
+import torchaudio
+raw_audio_dir = "./raw_audio/"
+denoise_audio_dir = "./denoised_audio/"
+filelist = list(os.walk(raw_audio_dir))[0][2]
+# 2023/4/21: Get the target sampling rate
+with open("./configs/finetune_speaker.json", 'r', encoding='utf-8') as f:
+    hps = json.load(f)
+target_sr = hps['data']['sampling_rate']
+for file in filelist:
+    if file.endswith(".wav"):
+        os.system(f"demucs --two-stems=vocals {raw_audio_dir}{file}")
+for file in filelist:
+    file = file.replace(".wav", "")
+    wav, sr = torchaudio.load(f"./separated/htdemucs/{file}/vocals.wav", frame_offset=0, num_frames=-1, normalize=True,
+                              channels_first=True)
+    # merge two channels into one
+    wav = wav.mean(dim=0).unsqueeze(0)
+    if sr != target_sr:
+        wav = torchaudio.transforms.Resample(orig_freq=sr, new_freq=target_sr)(wav)
+    torchaudio.save(denoise_audio_dir + file + ".wav", wav, target_sr, channels_first=True)

scripts/download_model.py

@@ -0,0 +1,4 @@
+from google.colab import files
+files.download("./G_latest.pth")
+files.download("./finetune_speaker.json")
+files.download("./moegoe_config.json")

scripts/download_video.py

@@ -0,0 +1,37 @@
+import os
+import random
+import shutil
+from concurrent.futures import ThreadPoolExecutor
+from google.colab import files
+
+basepath = os.getcwd()
+uploaded = files.upload()  # 上传文件
+for filename in uploaded.keys():
+    assert (filename.endswith(".txt")), "speaker-videolink info could only be .txt file!"
+    shutil.move(os.path.join(basepath, filename), os.path.join("./speaker_links.txt"))
+
+
+def generate_infos():
+    infos = []
+    with open("./speaker_links.txt", 'r', encoding='utf-8') as f:
+        lines = f.readlines()
+    for line in lines:
+        line = line.replace("\n", "").replace(" ", "")
+        if line == "":
+            continue
+        speaker, link = line.split("|")
+        filename = speaker + "_" + str(random.randint(0, 1000000))
+        infos.append({"link": link, "filename": filename})
+    return infos
+
+
+def download_video(info):
+    link = info["link"]
+    filename = info["filename"]
+    os.system(f"youtube-dl -f 0 {link} -o ./video_data/{filename}.mp4 --no-check-certificate")
+
+
+if __name__ == "__main__":
+    infos = generate_infos()
+    with ThreadPoolExecutor(max_workers=os.cpu_count()) as executor:
+        executor.map(download_video, infos)

scripts/long_audio_transcribe.py

@@ -0,0 +1,75 @@
+from moviepy.editor import AudioFileClip
+import whisper
+import os
+import json
+import torchaudio
+import librosa
+import torch
+import argparse
+parent_dir = "./denoised_audio/"
+filelist = list(os.walk(parent_dir))[0][2]
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--languages", default="CJE")
+    parser.add_argument("--whisper_size", default="medium")
+    args = parser.parse_args()
+    if args.languages == "CJE":
+        lang2token = {
+            'zh': "[ZH]",
+            'ja': "[JA]",
+            "en": "[EN]",
+        }
+    elif args.languages == "CJ":
+        lang2token = {
+            'zh': "[ZH]",
+            'ja': "[JA]",
+        }
+    elif args.languages == "C":
+        lang2token = {
+            'zh': "[ZH]",
+        }
+    assert(torch.cuda.is_available()), "Please enable GPU in order to run Whisper!"
+    with open("./configs/finetune_speaker.json", 'r', encoding='utf-8') as f:
+        hps = json.load(f)
+    target_sr = hps['data']['sampling_rate']
+    model = whisper.load_model(args.whisper_size)
+    speaker_annos = []
+    for file in filelist:
+        print(f"transcribing {parent_dir + file}...\n")
+        options = dict(beam_size=5, best_of=5)
+        transcribe_options = dict(task="transcribe", **options)
+        result = model.transcribe(parent_dir + file, word_timestamps=True, **transcribe_options)
+        segments = result["segments"]
+        # result = model.transcribe(parent_dir + file)
+        lang = result['language']
+        if result['language'] not in list(lang2token.keys()):
+            print(f"{lang} not supported, ignoring...\n")
+            continue
+        # segment audio based on segment results
+        character_name = file.rstrip(".wav").split("_")[0]
+        code = file.rstrip(".wav").split("_")[1]
+        if not os.path.exists("./segmented_character_voice/" + character_name):
+            os.mkdir("./segmented_character_voice/" + character_name)
+        wav, sr = torchaudio.load(parent_dir + file, frame_offset=0, num_frames=-1, normalize=True,
+                                  channels_first=True)
+
+        for i, seg in enumerate(result['segments']):
+            start_time = seg['start']
+            end_time = seg['end']
+            text = seg['text']
+            text = lang2token[lang] + text.replace("\n", "") + lang2token[lang]
+            text = text + "\n"
+            wav_seg = wav[:, int(start_time*sr):int(end_time*sr)]
+            wav_seg_name = f"{character_name}_{code}_{i}.wav"
+            savepth = "./segmented_character_voice/" + character_name + "/" + wav_seg_name
+            speaker_annos.append(savepth + "|" + character_name + "|" + text)
+            print(f"Transcribed segment: {speaker_annos[-1]}")
+            # trimmed_wav_seg = librosa.effects.trim(wav_seg.squeeze().numpy())
+            # trimmed_wav_seg = torch.tensor(trimmed_wav_seg[0]).unsqueeze(0)
+            torchaudio.save(savepth, wav_seg, target_sr, channels_first=True)
+    if len(speaker_annos) == 0:
+        print("Warning: no long audios & videos found, this IS expected if you have only uploaded short audios")
+        print("this IS NOT expected if you have uploaded any long audios, videos or video links. Please check your file structure or make sure your audio/video language is supported.")
+    with open("long_character_anno.txt", 'w', encoding='utf-8') as f:
+        for line in speaker_annos:
+            f.write(line)

scripts/rearrange_speaker.py

@@ -0,0 +1,37 @@
+import torch
+import argparse
+import json
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model_dir", type=str, default="./OUTPUT_MODEL/G_latest.pth")
+    parser.add_argument("--config_dir", type=str, default="./configs/modified_finetune_speaker.json")
+    args = parser.parse_args()
+
+    model_sd = torch.load(args.model_dir, map_location='cpu')
+    with open(args.config_dir, 'r', encoding='utf-8') as f:
+        hps = json.load(f)
+
+    valid_speakers = list(hps['speakers'].keys())
+    if hps['data']['n_speakers'] > len(valid_speakers):
+        new_emb_g = torch.zeros([len(valid_speakers), 256])
+        old_emb_g = model_sd['model']['emb_g.weight']
+        for i, speaker in enumerate(valid_speakers):
+            new_emb_g[i, :] = old_emb_g[hps['speakers'][speaker], :]
+            hps['speakers'][speaker] = i
+        hps['data']['n_speakers'] = len(valid_speakers)
+        model_sd['model']['emb_g.weight'] = new_emb_g
+        with open("./finetune_speaker.json", 'w', encoding='utf-8') as f:
+            json.dump(hps, f, indent=2)
+        torch.save(model_sd, "./G_latest.pth")
+    else:
+        with open("./finetune_speaker.json", 'w', encoding='utf-8') as f:
+            json.dump(hps, f, indent=2)
+        torch.save(model_sd, "./G_latest.pth")
+    # save another config file copy in MoeGoe format
+    hps['speakers'] = valid_speakers
+    with open("./moegoe_config.json", 'w', encoding='utf-8') as f:
+        json.dump(hps, f, indent=2)
+
+
+

scripts/resample.py

@@ -0,0 +1,20 @@
+import os
+import json
+import argparse
+import torchaudio
+
+
+def main():
+    with open("./configs/finetune_speaker.json", 'r', encoding='utf-8') as f:
+        hps = json.load(f)
+    target_sr = hps['data']['sampling_rate']
+    filelist = list(os.walk("./sampled_audio4ft"))[0][2]
+    if target_sr != 22050:
+        for wavfile in filelist:
+            wav, sr = torchaudio.load("./sampled_audio4ft" + "/" + wavfile, frame_offset=0, num_frames=-1,
+                                      normalize=True, channels_first=True)
+            wav = torchaudio.transforms.Resample(orig_freq=sr, new_freq=target_sr)(wav)
+            torchaudio.save("./sampled_audio4ft" + "/" + wavfile, wav, target_sr, channels_first=True)
+
+if __name__ == "__main__":
+    main()

scripts/short_audio_transcribe.py

@@ -0,0 +1,116 @@
+import whisper
+import os
+import json
+import torchaudio
+import argparse
+import torch
+
+lang2token = {
+            'zh': "[ZH]",
+            'ja': "[JA]",
+            "en": "[EN]",
+        }
+def transcribe_one(audio_path):
+    # load audio and pad/trim it to fit 30 seconds
+    audio = whisper.load_audio(audio_path)
+    audio = whisper.pad_or_trim(audio)
+
+    # make log-Mel spectrogram and move to the same device as the model
+    mel = whisper.log_mel_spectrogram(audio).to(model.device)
+
+    # detect the spoken language
+    _, probs = model.detect_language(mel)
+    print(f"Detected language: {max(probs, key=probs.get)}")
+    lang = max(probs, key=probs.get)
+    # decode the audio
+    options = whisper.DecodingOptions()
+    result = whisper.decode(model, mel, options)
+
+    # print the recognized text
+    print(result.text)
+    return lang, result.text
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--languages", default="CJE")
+    parser.add_argument("--whisper_size", default="medium")
+    args = parser.parse_args()
+    if args.languages == "CJE":
+        lang2token = {
+            'zh': "[ZH]",
+            'ja': "[JA]",
+            "en": "[EN]",
+        }
+    elif args.languages == "CJ":
+        lang2token = {
+            'zh': "[ZH]",
+            'ja': "[JA]",
+        }
+    elif args.languages == "C":
+        lang2token = {
+            'zh': "[ZH]",
+        }
+    assert (torch.cuda.is_available()), "Please enable GPU in order to run Whisper!"
+    model = whisper.load_model(args.whisper_size)
+    parent_dir = "./custom_character_voice/"
+    speaker_names = list(os.walk(parent_dir))[0][1]
+    speaker_annos = []
+    # resample audios
+    # 2023/4/21: Get the target sampling rate
+    with open("./configs/finetune_speaker.json", 'r', encoding='utf-8') as f:
+        hps = json.load(f)
+    target_sr = hps['data']['sampling_rate']
+    for speaker in speaker_names:
+        for i, wavfile in enumerate(list(os.walk(parent_dir + speaker))[0][2]):
+            # try to load file as audio
+            if wavfile.startswith("processed_"):
+                continue
+            try:
+                wav, sr = torchaudio.load(parent_dir + speaker + "/" + wavfile, frame_offset=0, num_frames=-1, normalize=True,
+                                          channels_first=True)
+                wav = wav.mean(dim=0).unsqueeze(0)
+                if sr != target_sr:
+                    wav = torchaudio.transforms.Resample(orig_freq=sr, new_freq=target_sr)(wav)
+                if wav.shape[1] / sr > 20:
+                    print(f"{wavfile} too long, ignoring\n")
+                save_path = parent_dir + speaker + "/" + f"processed_{i}.wav"
+                torchaudio.save(save_path, wav, target_sr, channels_first=True)
+                # transcribe text
+                lang, text = transcribe_one(save_path)
+                if lang not in list(lang2token.keys()):
+                    print(f"{lang} not supported, ignoring\n")
+                    continue
+                text = lang2token[lang] + text + lang2token[lang] + "\n"
+                speaker_annos.append(save_path + "|" + speaker + "|" + text)
+            except:
+                continue
+
+    # # clean annotation
+    # import argparse
+    # import text
+    # from utils import load_filepaths_and_text
+    # for i, line in enumerate(speaker_annos):
+    #     path, sid, txt = line.split("|")
+    #     cleaned_text = text._clean_text(txt, ["cjke_cleaners2"])
+    #     cleaned_text += "\n" if not cleaned_text.endswith("\n") else ""
+    #     speaker_annos[i] = path + "|" + sid + "|" + cleaned_text
+    # write into annotation
+    if len(speaker_annos) == 0:
+        print("Warning: no short audios found, this IS expected if you have only uploaded long audios, videos or video links.")
+        print("this IS NOT expected if you have uploaded a zip file of short audios. Please check your file structure or make sure your audio language is supported.")
+    with open("short_character_anno.txt", 'w', encoding='utf-8') as f:
+        for line in speaker_annos:
+            f.write(line)
+
+    # import json
+    # # generate new config
+    # with open("./configs/finetune_speaker.json", 'r', encoding='utf-8') as f:
+    #     hps = json.load(f)
+    # # modify n_speakers
+    # hps['data']["n_speakers"] = 1000 + len(speaker2id)
+    # # add speaker names
+    # for speaker in speaker_names:
+    #     hps['speakers'][speaker] = speaker2id[speaker]
+    # # save modified config
+    # with open("./configs/modified_finetune_speaker.json", 'w', encoding='utf-8') as f:
+    #     json.dump(hps, f, indent=2)
+    # print("finished")

scripts/video2audio.py

@@ -0,0 +1,27 @@
+import os
+from concurrent.futures import ThreadPoolExecutor
+
+from moviepy.editor import AudioFileClip
+
+video_dir = "./video_data/"
+audio_dir = "./raw_audio/"
+filelist = list(os.walk(video_dir))[0][2]
+
+
+def generate_infos():
+    videos = []
+    for file in filelist:
+        if file.endswith(".mp4"):
+            videos.append(file)
+    return videos
+
+
+def clip_file(file):
+    my_audio_clip = AudioFileClip(video_dir + file)
+    my_audio_clip.write_audiofile(audio_dir + file.rstrip(".mp4") + ".wav")
+
+
+if __name__ == "__main__":
+    infos = generate_infos()
+    with ThreadPoolExecutor(max_workers=os.cpu_count()) as executor:
+        executor.map(clip_file, infos)

scripts/voice_upload.py

@@ -0,0 +1,28 @@
+from google.colab import files
+import shutil
+import os
+import argparse
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--type", type=str, required=True, help="type of file to upload")
+    args = parser.parse_args()
+    file_type = args.type
+
+    basepath = os.getcwd()
+    uploaded = files.upload() # 上传文件
+    assert(file_type in ['zip', 'audio', 'video'])
+    if file_type == "zip":
+        upload_path = "./custom_character_voice/"
+        for filename in uploaded.keys():
+            #将上传的文件移动到指定的位置上
+            shutil.move(os.path.join(basepath, filename), os.path.join(upload_path, "custom_character_voice.zip"))
+    elif file_type == "audio":
+        upload_path = "./raw_audio/"
+        for filename in uploaded.keys():
+            #将上传的文件移动到指定的位置上
+            shutil.move(os.path.join(basepath, filename), os.path.join(upload_path, filename))
+    elif file_type == "video":
+        upload_path = "./video_data/"
+        for filename in uploaded.keys():
+            # 将上传的文件移动到指定的位置上
+            shutil.move(os.path.join(basepath, filename), os.path.join(upload_path, filename))