From 7d5c196cc70f26d16300d1e972d497c838518fd8 Mon Sep 17 00:00:00 2001 From: Plachta Date: Sun, 26 Feb 2023 20:08:37 +0800 Subject: [PATCH] updated pipeline --- finetune_speaker.py | 320 ------------------------------ preprocess.py | 45 ----- user_voice/user_voice.txt | 93 --------- user_voice/user_voice.txt.cleaned | 93 --------- user_voice_collect.py | 72 ------- whisper_transcribe.py | 90 --------- 6 files changed, 713 deletions(-) delete mode 100644 finetune_speaker.py delete mode 100644 preprocess.py delete mode 100644 user_voice/user_voice.txt delete mode 100644 user_voice/user_voice.txt.cleaned delete mode 100644 user_voice_collect.py delete mode 100644 whisper_transcribe.py diff --git a/finetune_speaker.py b/finetune_speaker.py deleted file mode 100644 index 1679741..0000000 --- a/finetune_speaker.py +++ /dev/null @@ -1,320 +0,0 @@ -import os -import json -import argparse -import itertools -import math -import torch -from torch import nn, optim -from torch.nn import functional as F -from torch.utils.data import DataLoader -from torch.utils.tensorboard import SummaryWriter -import torch.multiprocessing as mp -import torch.distributed as dist -from torch.nn.parallel import DistributedDataParallel as DDP -from torch.cuda.amp import autocast, GradScaler -from tqdm import tqdm - -import librosa -import logging - -logging.getLogger('numba').setLevel(logging.WARNING) - -import commons -import utils -from data_utils import ( - TextAudioSpeakerLoader, - TextAudioSpeakerCollate, - DistributedBucketSampler -) -from models import ( - SynthesizerTrn, - MultiPeriodDiscriminator, -) -from losses import ( - generator_loss, - discriminator_loss, - feature_loss, - kl_loss -) -from mel_processing import mel_spectrogram_torch, spec_to_mel_torch - - -torch.backends.cudnn.benchmark = True -global_step = 0 - - -def main(): - """Assume Single Node Multi GPUs Training Only""" - assert torch.cuda.is_available(), "CPU training is not allowed." - - n_gpus = torch.cuda.device_count() - os.environ['MASTER_ADDR'] = 'localhost' - os.environ['MASTER_PORT'] = '8000' - - hps = utils.get_hparams() - mp.spawn(run, nprocs=n_gpus, args=(n_gpus, hps,)) - - -def run(rank, n_gpus, hps): - global global_step - symbols = hps['symbols'] - if rank == 0: - logger = utils.get_logger(hps.model_dir) - logger.info(hps) - utils.check_git_hash(hps.model_dir) - writer = SummaryWriter(log_dir=hps.model_dir) - writer_eval = SummaryWriter(log_dir=os.path.join(hps.model_dir, "eval")) - - dist.init_process_group(backend='nccl', init_method='env://', world_size=n_gpus, rank=rank) - torch.manual_seed(hps.train.seed) - torch.cuda.set_device(rank) - - train_dataset = TextAudioSpeakerLoader(hps.data.training_files, hps.data) - train_sampler = DistributedBucketSampler( - train_dataset, - hps.train.batch_size, - [32,300,400,500,600,700,800,900,1000], - num_replicas=n_gpus, - rank=rank, - shuffle=True) - collate_fn = TextAudioSpeakerCollate() - train_loader = DataLoader(train_dataset, num_workers=8, shuffle=False, pin_memory=True, - collate_fn=collate_fn, batch_sampler=train_sampler) - # train_loader = DataLoader(train_dataset, batch_size=hps.train.batch_size, num_workers=0, shuffle=False, pin_memory=True, - # collate_fn=collate_fn) - if rank == 0: - eval_dataset = TextAudioSpeakerLoader(hps.data.validation_files, hps.data) - eval_loader = DataLoader(eval_dataset, num_workers=0, shuffle=False, - batch_size=hps.train.batch_size, pin_memory=True, - drop_last=False, collate_fn=collate_fn) - - net_g = SynthesizerTrn( - len(symbols), - hps.data.filter_length // 2 + 1, - hps.train.segment_size // hps.data.hop_length, - n_speakers=hps.data.n_speakers, - **hps.model).cuda(rank) - net_d = MultiPeriodDiscriminator(hps.model.use_spectral_norm).cuda(rank) - - # load existing model - _, _, _, _ = utils.load_checkpoint("./pretrained_models/G_trilingual.pth", net_g, None) - _, _, _, _ = utils.load_checkpoint("./pretrained_models/D_trilingual.pth", net_d, None) - epoch_str = 1 - global_step = 0 - # freeze all other layers except speaker embedding - for p in net_g.parameters(): - p.requires_grad = True - for p in net_d.parameters(): - p.requires_grad = True - # for p in net_d.parameters(): - # p.requires_grad = False - # net_g.emb_g.weight.requires_grad = True - optim_g = torch.optim.AdamW( - net_g.parameters(), - hps.train.learning_rate, - betas=hps.train.betas, - eps=hps.train.eps) - optim_d = torch.optim.AdamW( - net_d.parameters(), - hps.train.learning_rate, - betas=hps.train.betas, - eps=hps.train.eps) - # optim_d = None - net_g = DDP(net_g, device_ids=[rank]) - net_d = DDP(net_d, device_ids=[rank]) - - scheduler_g = torch.optim.lr_scheduler.ExponentialLR(optim_g, gamma=hps.train.lr_decay) - scheduler_d = torch.optim.lr_scheduler.ExponentialLR(optim_d, gamma=hps.train.lr_decay) - - scaler = GradScaler(enabled=hps.train.fp16_run) - - for epoch in range(epoch_str, hps.train.epochs + 1): - if rank==0: - train_and_evaluate(rank, epoch, hps, [net_g, net_d], [optim_g, optim_d], [scheduler_g, scheduler_d], scaler, [train_loader, eval_loader], logger, [writer, writer_eval]) - else: - train_and_evaluate(rank, epoch, hps, [net_g, net_d], [optim_g, optim_d], [scheduler_g, scheduler_d], scaler, [train_loader, None], None, None) - scheduler_g.step() - scheduler_d.step() - - -def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler, loaders, logger, writers): - net_g, net_d = nets - optim_g, optim_d = optims - scheduler_g, scheduler_d = schedulers - train_loader, eval_loader = loaders - if writers is not None: - writer, writer_eval = writers - - # train_loader.batch_sampler.set_epoch(epoch) - global global_step - - net_g.train() - net_d.train() - for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths, speakers) in enumerate(tqdm(train_loader)): - x, x_lengths = x.cuda(rank, non_blocking=True), x_lengths.cuda(rank, non_blocking=True) - spec, spec_lengths = spec.cuda(rank, non_blocking=True), spec_lengths.cuda(rank, non_blocking=True) - y, y_lengths = y.cuda(rank, non_blocking=True), y_lengths.cuda(rank, non_blocking=True) - speakers = speakers.cuda(rank, non_blocking=True) - - with autocast(enabled=hps.train.fp16_run): - y_hat, l_length, attn, ids_slice, x_mask, z_mask,\ - (z, z_p, m_p, logs_p, m_q, logs_q) = net_g(x, x_lengths, spec, spec_lengths, speakers) - - mel = spec_to_mel_torch( - spec, - hps.data.filter_length, - hps.data.n_mel_channels, - hps.data.sampling_rate, - hps.data.mel_fmin, - hps.data.mel_fmax) - y_mel = commons.slice_segments(mel, ids_slice, hps.train.segment_size // hps.data.hop_length) - y_hat_mel = mel_spectrogram_torch( - y_hat.squeeze(1), - hps.data.filter_length, - hps.data.n_mel_channels, - hps.data.sampling_rate, - hps.data.hop_length, - hps.data.win_length, - hps.data.mel_fmin, - hps.data.mel_fmax - ) - - y = commons.slice_segments(y, ids_slice * hps.data.hop_length, hps.train.segment_size) # slice - - # Discriminator - y_d_hat_r, y_d_hat_g, _, _ = net_d(y, y_hat.detach()) - with autocast(enabled=False): - loss_disc, losses_disc_r, losses_disc_g = discriminator_loss(y_d_hat_r, y_d_hat_g) - loss_disc_all = loss_disc - optim_d.zero_grad() - scaler.scale(loss_disc_all).backward() - scaler.unscale_(optim_d) - grad_norm_d = commons.clip_grad_value_(net_d.parameters(), None) - scaler.step(optim_d) - - with autocast(enabled=hps.train.fp16_run): - # Generator - y_d_hat_r, y_d_hat_g, fmap_r, fmap_g = net_d(y, y_hat) - with autocast(enabled=False): - loss_dur = torch.sum(l_length.float()) - loss_mel = F.l1_loss(y_mel, y_hat_mel) * hps.train.c_mel - loss_kl = kl_loss(z_p, logs_q, m_p, logs_p, z_mask) * hps.train.c_kl - - loss_fm = feature_loss(fmap_r, fmap_g) - loss_gen, losses_gen = generator_loss(y_d_hat_g) - loss_gen_all = loss_gen + loss_fm + loss_mel + loss_dur + loss_kl - optim_g.zero_grad() - scaler.scale(loss_gen_all).backward() - scaler.unscale_(optim_g) - grad_norm_g = commons.clip_grad_value_(net_g.parameters(), None) - scaler.step(optim_g) - scaler.update() - - if rank==0: - if global_step % hps.train.log_interval == 0: - lr = optim_g.param_groups[0]['lr'] - losses = [loss_disc, loss_gen, loss_fm, loss_mel, loss_dur, loss_kl] - logger.info('Train Epoch: {} [{:.0f}%]'.format( - epoch, - 100. * batch_idx / len(train_loader))) - logger.info([x.item() for x in losses] + [global_step, lr]) - - scalar_dict = {"loss/g/total": loss_gen_all, "loss/d/total": loss_disc_all, "learning_rate": lr, "grad_norm_g": grad_norm_g} - scalar_dict.update({"loss/g/fm": loss_fm, "loss/g/mel": loss_mel, "loss/g/dur": loss_dur, "loss/g/kl": loss_kl}) - - scalar_dict.update({"loss/g/{}".format(i): v for i, v in enumerate(losses_gen)}) - scalar_dict.update({"loss/d_r/{}".format(i): v for i, v in enumerate(losses_disc_r)}) - scalar_dict.update({"loss/d_g/{}".format(i): v for i, v in enumerate(losses_disc_g)}) - image_dict = { - "slice/mel_org": utils.plot_spectrogram_to_numpy(y_mel[0].data.cpu().numpy()), - "slice/mel_gen": utils.plot_spectrogram_to_numpy(y_hat_mel[0].data.cpu().numpy()), - "all/mel": utils.plot_spectrogram_to_numpy(mel[0].data.cpu().numpy()), - "all/attn": utils.plot_alignment_to_numpy(attn[0,0].data.cpu().numpy()) - } - utils.summarize( - writer=writer, - global_step=global_step, - images=image_dict, - scalars=scalar_dict) - - if global_step % hps.train.eval_interval == 0: - evaluate(hps, net_g, eval_loader, writer_eval) - utils.save_checkpoint(net_g, None, hps.train.learning_rate, epoch, os.path.join(hps.model_dir, "G_{}.pth".format(global_step))) - utils.save_checkpoint(net_g, None, hps.train.learning_rate, epoch, - os.path.join(hps.model_dir, "G_latest.pth".format(global_step))) - # utils.save_checkpoint(net_d, optim_d, hps.train.learning_rate, epoch, os.path.join(hps.model_dir, "D_{}.pth".format(global_step))) - old_g=os.path.join(hps.model_dir, "G_{}.pth".format(global_step-4000)) - # old_d=os.path.join(hps.model_dir, "D_{}.pth".format(global_step-400)) - if os.path.exists(old_g): - os.remove(old_g) - # if os.path.exists(old_d): - # os.remove(old_d) - global_step += 1 - if epoch > hps.max_epochs: - print("Maximum epoch reached, closing training...") - exit() - - if rank == 0: - logger.info('====> Epoch: {}'.format(epoch)) - - -def evaluate(hps, generator, eval_loader, writer_eval): - generator.eval() - with torch.no_grad(): - for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths, speakers) in enumerate(eval_loader): - x, x_lengths = x.cuda(0), x_lengths.cuda(0) - spec, spec_lengths = spec.cuda(0), spec_lengths.cuda(0) - y, y_lengths = y.cuda(0), y_lengths.cuda(0) - speakers = speakers.cuda(0) - - # remove else - x = x[:1] - x_lengths = x_lengths[:1] - spec = spec[:1] - spec_lengths = spec_lengths[:1] - y = y[:1] - y_lengths = y_lengths[:1] - speakers = speakers[:1] - break - y_hat, attn, mask, *_ = generator.module.infer(x, x_lengths, speakers, max_len=1000) - y_hat_lengths = mask.sum([1,2]).long() * hps.data.hop_length - - mel = spec_to_mel_torch( - spec, - hps.data.filter_length, - hps.data.n_mel_channels, - hps.data.sampling_rate, - hps.data.mel_fmin, - hps.data.mel_fmax) - y_hat_mel = mel_spectrogram_torch( - y_hat.squeeze(1).float(), - hps.data.filter_length, - hps.data.n_mel_channels, - hps.data.sampling_rate, - hps.data.hop_length, - hps.data.win_length, - hps.data.mel_fmin, - hps.data.mel_fmax - ) - image_dict = { - "gen/mel": utils.plot_spectrogram_to_numpy(y_hat_mel[0].cpu().numpy()) - } - audio_dict = { - "gen/audio": y_hat[0,:,:y_hat_lengths[0]] - } - if global_step == 0: - image_dict.update({"gt/mel": utils.plot_spectrogram_to_numpy(mel[0].cpu().numpy())}) - audio_dict.update({"gt/audio": y[0,:,:y_lengths[0]]}) - - utils.summarize( - writer=writer_eval, - global_step=global_step, - images=image_dict, - audios=audio_dict, - audio_sampling_rate=hps.data.sampling_rate - ) - generator.train() - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/preprocess.py b/preprocess.py deleted file mode 100644 index 595cf73..0000000 --- a/preprocess.py +++ /dev/null @@ -1,45 +0,0 @@ -import os -if __name__ == "__main__": - # load sampled_audio4ft - with open("sampled_audio4ft.txt", 'r', encoding='utf-8') as f: - old_annos = f.readlines() - num_old_voices = len(old_annos) - # load user text - with open("./user_voice/user_voice.txt.cleaned", 'r', encoding='utf-8') as f: - user_annos = f.readlines() - - # check how many voices are recorded - wavfiles = [file for file in list(os.walk("./user_voice"))[0][2] if file.endswith(".wav")] - num_user_voices = len(wavfiles) - # user voices need to occupy 1/4 of the total dataset - if num_user_voices: - user_duplicate = num_old_voices // num_user_voices // 3 - else: - user_duplicate = 0 - - # find corresponding existing annotation lines - actual_user_annos = ["./user_voice/" + line for line in user_annos if line.split("|")[0] in wavfiles] - final_annos = old_annos + actual_user_annos * user_duplicate - - # load custom characters - if os.path.exists("custom_character_anno.txt"): - with open("custom_character_anno.txt", 'r', encoding='utf-8') as f: - custom_character_anno = f.readlines() - if len(custom_character_anno): - # custom character voices need to be at least equal to number of sample_audio4ft - num_character_voices = len(custom_character_anno) - cc_duplicate = num_old_voices // num_character_voices - if cc_duplicate == 0: - cc_duplicate = 1 - final_annos = final_annos + custom_character_anno * cc_duplicate - # save annotation file - with open("final_annotation_train.txt", 'w', encoding='utf-8') as f: - for line in final_annos: - f.write(line) - # save annotation file for validation - with open("final_annotation_val.txt", 'w', encoding='utf-8') as f: - for line in actual_user_annos: - f.write(line) - if os.path.exists("custom_character_anno.txt"): - for line in custom_character_anno: - f.write(line) diff --git a/user_voice/user_voice.txt b/user_voice/user_voice.txt deleted file mode 100644 index 59bfce3..0000000 --- a/user_voice/user_voice.txt +++ /dev/null @@ -1,93 +0,0 @@ -0.wav|999|[ZH]所以，人的内在拥有对于人的幸福才是最关键的。[ZH] -1.wav|999|[ZH]正因为在大多数情形下人的自身内在相当贫乏，[ZH] -2.wav|999|[ZH]所以，那些再也用不着与生活的匮乏作斗争的人，[ZH] -3.wav|999|[ZH]他们之中的大多数从根本上还是感觉闷闷不乐。[ZH] -4.wav|999|[ZH]情形就跟那些还在生活的困苦中搏斗的人一般无异。[ZH] -5.wav|999|[ZH]他们内在空虚、感觉意识呆滞、思想匮乏，[ZH] -6.wav|999|[ZH]这些就驱使他们投入社交人群中。[ZH] -7.wav|999|[ZH]组成那些社交圈子的人也正是他们这一类的人。[ZH] -8.wav|999|[ZH]“因为相同羽毛的鸟聚在一块”。[ZH] -9.wav|999|[ZH]他们聚在一块追逐消遣、娱乐。[ZH] -10.wav|999|[ZH]他们以放纵感官的欢娱、极尽声色的享受开始，[ZH] -11.wav|999|[ZH]以荒唐、无度而告终。[ZH] -12.wav|999|[ZH]众多刚刚踏入生活的纨绔子弟穷奢极欲，[ZH] -13.wav|999|[ZH]在令人难以置信的极短时间内就把大部分家财挥霍殆尽。[ZH] -14.wav|999|[ZH]这种做派，其根源确实不是别的，正是无聊[ZH] -18.wav|999|[ZH]它源自上述的精神贫乏和空虚。[ZH] -16.wav|999|[ZH]一个外在富有、但内在贫乏的富家子弟来到这个世界，[ZH] -17.wav|999|[ZH]会徒劳地用外在的财富去补偿内在的不足；[ZH] -18.wav|999|[ZH]他渴望从外部得到一切，[ZH] -19.wav|999|[ZH]这情形就好比试图以少女的汗水去强健自己体魄的老朽之人。[ZH] -20.wav|999|[ZH]人自身内在的贫乏由此导致了外在财富的贫乏。[ZH] -21.wav|999|[ZH]至于另外两项人生好处的重要性，[ZH] -22.wav|999|[ZH]不需要我特别强调。[ZH] -23.wav|999|[ZH]财产的价值在当今是人所公认的，[ZH] -24.wav|999|[ZH]用不着为其宣传介绍。[ZH] -25.wav|999|[ZH]比起第二项的好处，[ZH] -26.wav|999|[ZH]第三项的好处具有一种相当飘渺的成分，[ZH] -27.wav|999|[ZH]因为名誉、名望、地位等[ZH] -28.wav|999|[ZH]全由他人的意见构成。[ZH] -29.wav|999|[ZH]每人都可以争取得到名誉，[ZH] -30.wav|999|[ZH]亦即清白的名声；[ZH] -31.wav|999|[ZH]但社会地位，则只有月盼国家政府的人才能染指；[ZH] -32.wav|999|[ZH]至于显赫的名望就只有极少数人才会得到。[ZH] -33.wav|999|[ZH]在所有这些当中，[ZH] -34.wav|999|[ZH]名誉是弥足珍贵的；[ZH] -35.wav|999|[ZH]显赫的名望则是人所希望得到的价值至昂的东西，[ZH] -36.wav|999|[ZH]那是天之骄子才能得到的金羊毛。[ZH] -37.wav|999|[ZH]另一方面，[ZH] -38.wav|999|[ZH]只有傻瓜才会把社会地位放置在财产之前。[ZH] -39.wav|999|[ZH]另外，人拥有的财产、物品和名誉、声望，[ZH] -40.wav|999|[ZH]是处于一种所谓的互为影响、促进的关系。[ZH] -41.wav|999|[ZH]彼得尼斯说过：“一个人所拥有的财产决定了这个人在他人眼中的价值”。[ZH] -42.wav|999|[ZH]如果这句话是正确的话，[ZH] -43.wav|999|[ZH]那么，反过来，他人对自己的良好评价，[ZH] -44.wav|999|[ZH]能以各种形式帮助自己获取财产。[ZH] -45.wav|999|[ZH]能以各种形式帮助自己获取财产。[ZH] -46.wav|999|[EN]So far, the problems have occurred in the US.[EN] -47.wav|999|[EN]The statement contained no surprises.[EN] -48.wav|999|[EN]The performance continues to improve.[EN] -49.wav|999|[EN]That would be a danger.[EN] -50.wav|999|[EN]Frankly, we were lucky to get second.[EN] -51.wav|999|[EN]However, no further action was taken by police.[EN] -52.wav|999|[EN]It depends on Labour, not on us.[EN] -53.wav|999|[EN]I had to be into it.[EN] -54.wav|999|[EN]My view has always been the same.[EN] -55.wav|999|[EN]In fact, he is not even in the squad for the game.[EN] -56.wav|999|[EN]It is set in Paris.[EN] -57.wav|999|[EN]Do you think we're a top nation ?[EN] -58.wav|999|[EN]Is our children learning ?[EN] -59.wav|999|[EN]How independent is that ?[EN] -60.wav|999|[EN]What form did that take ?[EN] -61.wav|999|[EN]Are they free ?[EN] -62.wav|999|[EN]The party has never fully recovered.[EN] -63.wav|999|[EN]In many ways, that is as important.[EN] -64.wav|999|[EN]His leader wanted to celebrate.[EN] -65.wav|999|[EN]It is not an option, but a policy requirement.[EN] -66.wav|999|[EN]Of further privacy, he had no need.[EN] -67.wav|999|[EN]He was good, but not that good.[EN] -68.wav|999|[EN]The results are expected within days.[EN] -69.wav|999|[EN]The company still had the confidence of its bankers, he said.[EN] -70.wav|999|[EN]Mr Crawford is no stranger to Scottish Enterprise.[EN] -71.wav|999|[EN]We were well worth the win.[EN] -72.wav|999|[EN]And it was this one.[EN] -73.wav|999|[EN]But that's another story for another day.[EN] -74.wav|999|[EN]But it was also hard.[EN] -75.wav|999|[EN]If so, it is a funny time to introduce it.[EN] -76.wav|999|[EN]We're looking for unity in the council.[EN] -77.wav|999|[EN]By then, a massive legal battle is likely to have started.[EN] -78.wav|999|[EN]They are not going the length of the pitch.[EN] -79.wav|999|[EN]However, he lasted only five days before going on the run.[EN] -80.wav|999|[EN]The concerns are the same.[EN] -81.wav|999|[EN]That was a bonus, but it was not the main objective.[EN] -82.wav|999|[EN]Scotland won by six wickets.[EN] -83.wav|999|[EN]It has to be enforced.[EN] -84.wav|999|[EN]And we can do it.[EN] -85.wav|999|[EN]Every one is a winner.[EN] -86.wav|999|[EN]Drugs are used a lot at the fishing, not just cannabis.[EN] -87.wav|999|[EN]They have set the standard.[EN] -88.wav|999|[EN]This is a major property in Edinburgh.[EN] -89.wav|999|[EN]It is a crisis of human rights, a crisis of leadership.[EN] -90.wav|999|[EN]The existing structure is in trouble.[EN] -91.wav|999|[EN]It all started with a visiting rugby club.[EN] -92.wav|999|[EN]He refused to reveal the reasons for the split.[EN] diff --git a/user_voice/user_voice.txt.cleaned b/user_voice/user_voice.txt.cleaned deleted file mode 100644 index 86da462..0000000 --- a/user_voice/user_voice.txt.cleaned +++ /dev/null @@ -1,93 +0,0 @@ -0.wav|999|swo↓↑i↓↑, ɹ`ən↑ t⁼ə neɪ↓ts⁼aɪ↓ jʊŋ→joʊ↓↑ t⁼weɪ↓ɥ↑ ɹ`ən↑ t⁼ə ʃiŋ↓fu↑ tsʰaɪ↑ s`ɹ`↓ ts⁼weɪ↓ k⁼wan→tʃ⁼jɛn↓ t⁼ə. -1.wav|999|ts`⁼əŋ↓ in→weɪ↓ ts⁼aɪ↓ t⁼a↓t⁼wo→s`u↓ tʃʰiŋ↑ʃiŋ↑ ʃja↓ɹ`ən↑ t⁼ə ts⁼ɹ↓s`ən→ neɪ↓ts⁼aɪ↓ ʃiɑŋ→t⁼ɑŋ→ pʰin↑fa↑, -2.wav|999|swo↓↑i↓↑, na↓ʃiɛ→ ts⁼aɪ↓iɛ↓↑ jʊŋ↓p⁼u↓ts`⁼ə ɥ↓↑ s`əŋ→xwo↑ t⁼ə kʰweɪ↓fa↑ ts⁼wo↓ t⁼oʊ↓ts`⁼əŋ→ t⁼ə ɹ`ən↑, -3.wav|999|tʰa→mən ts`⁼ɹ`→ts`⁼ʊŋ→ t⁼ə t⁼a↓t⁼wo→s`u↓ tsʰʊŋ↑k⁼ən→p⁼ən↓↑s`ɑŋ↓ xaɪ↑s`ɹ`↓ k⁼an↓↑tʃ⁼ɥɛ↑ mən↓mən↓p⁼u↓lə↓. -4.wav|999|tʃʰiŋ↑ʃiŋ↑ tʃ⁼joʊ↓ k⁼ən→ na↓ʃiɛ→ xaɪ↑ ts⁼aɪ↓ s`əŋ→xwo↑ t⁼ə kʰwən↓kʰu↓↑ ts`⁼ʊŋ→ p⁼wo↑t⁼oʊ↓ t⁼ə ɹ`ən↑ i↓p⁼an→ u↑i↓. -5.wav|999|tʰa→mən neɪ↓ts⁼aɪ↓ kʰʊŋ→ʃɥ→, k⁼an↓↑tʃ⁼ɥɛ↑ i↓s`ɹ`↑ t⁼aɪ→ts`⁼ɹ`↓, sɹ→ʃiɑŋ↓↑ kʰweɪ↓fa↑, -6.wav|999|ts`⁼ə↓ʃiɛ→ tʃ⁼joʊ↓ tʃʰɥ→s`ɹ`↓↑ tʰa→mən tʰoʊ↑ɹ`u↓ s`ə↓tʃ⁼iɑʊ→ ɹ`ən↑tʃʰɥn↑ ts`⁼ʊŋ→. -7.wav|999|ts⁼u↓↑ts`ʰəŋ↑ na↓ʃiɛ→ s`ə↓tʃ⁼iɑʊ→tʃʰɥæn→ts⁼ɹ t⁼ə ɹ`ən↑ iɛ↓↑ ts`⁼əŋ↓s`ɹ`↓ tʰa→mən ts`⁼ə↓ i→leɪ↓ t⁼ə ɹ`ən↑. -8.wav|999|“ in→weɪ↓ ʃiɑŋ→tʰʊŋ↑ ɥ↓↑mɑʊ↑ t⁼ə niɑʊ↓↑ tʃ⁼ɥ↓ ts⁼aɪ↓ i→kʰwaɪ↓”. -9.wav|999|tʰa→mən tʃ⁼ɥ↓ts⁼aɪ↓ i→kʰwaɪ↓ ts`⁼weɪ→ts`⁼u↑ ʃiɑʊ→tʃʰjɛn↓↑, ɥ↑lə↓. -10.wav|999|tʰa→mən i↓↑ fɑŋ↓ts⁼ʊŋ↓ k⁼an↓↑k⁼wan→ t⁼ə xwan→ɥ↑, tʃ⁼i↑tʃ⁼in↓↑ s`əŋ→sə↓ t⁼ə ʃiɑŋ↓↑s`oʊ↓ kʰaɪ→s`ɹ`↓↑, -11.wav|999|i↓↑ xuɑŋ→tʰɑŋ↑, u↑t⁼u↓ əɹ`↑ k⁼ɑʊ↓ts`⁼ʊŋ→. -12.wav|999|ts`⁼ʊŋ↓t⁼wo→ k⁼ɑŋ→k⁼ɑŋ→ tʰa↓ɹ`u↓ s`əŋ→xwo↑ t⁼ə wan↑kʰu↓ts⁼ɹ↓↑t⁼i↓ tʃʰjʊŋ↑s`ə→tʃ⁼i↑ɥ↓, -13.wav|999|ts⁼aɪ↓ liŋ↓ɹ`ən↑ nan↑i↓↑ts`⁼ɹ`↓ʃin↓ t⁼ə tʃ⁼i↑ t⁼wan↓↑s`ɹ`↑tʃ⁼jɛn→ neɪ↓ tʃ⁼joʊ↓ p⁼a↓↑ t⁼a↓p⁼u↓fən↓ tʃ⁼ja→tsʰaɪ↑ xweɪ→xwo↓ t⁼aɪ↓tʃ⁼in↓. -14.wav|999|ts`⁼ə↓ts`⁼ʊŋ↓↑ ts⁼wo↓pʰaɪ↓, tʃʰi↑ k⁼ən→ɥæn↑ tʃʰɥɛ↓s`ɹ`↑ p⁼u↑s`ɹ`↓ p⁼iɛ↑t⁼ə, ts`⁼əŋ↓s`ɹ`↓ u↑liɑʊ↑. -18.wav|999|tʰa→ ɥæn↑ts⁼ɹ↓ s`ɑŋ↓s`u↓ t⁼ə tʃ⁼iŋ→s`ən↑ pʰin↑fa↑ xə↑ kʰʊŋ→ʃɥ→. -16.wav|999|i↑k⁼ə↓ waɪ↓ ts⁼aɪ↓ fu↓joʊ↓↑, t⁼an↓ neɪ↓ts⁼aɪ↓ pʰin↑fa↑ t⁼ə fu↓tʃ⁼ja→ts⁼ɹ↓↑t⁼i↓ laɪ↑t⁼ɑʊ↓ ts`⁼ə↓k⁼ə↓ s`ɹ`↓tʃ⁼iɛ↓, -17.wav|999|xweɪ↓ tʰu↑lɑʊ↑t⁼i↓ jʊŋ↓waɪ↓ ts⁼aɪ↓ t⁼ə tsʰaɪ↑fu↓ tʃʰɥ↓ p⁼u↓↑ts`ʰɑŋ↑ neɪ↓ts⁼aɪ↓ t⁼ə p⁼u↓ts⁼u↑, -18.wav|999|tʰa→ kʰə↓↑uɑŋ↓ tsʰʊŋ↑ waɪ↓p⁼u↓ t⁼ə↑t⁼ɑʊ↓ i→tʃʰiɛ↓, -19.wav|999|ts`⁼ə↓ tʃʰiŋ↑ʃiŋ↑ tʃ⁼joʊ↓ xɑʊ↓↑p⁼i↓↑ s`ɹ`↓tʰu↑ i↓↑ s`ɑʊ↓nɥ↓↑ t⁼ə xan↓s`weɪ↓↑ tʃʰɥ↓ tʃʰiɑŋ↑tʃ⁼jɛn↓ ts⁼ɹ↓tʃ⁼i↓↑ tʰi↓↑pʰwo↓ t⁼ə lɑʊ↓↑ʃjoʊ↓↑ ts`⁼ɹ`→ ɹ`ən↑. -20.wav|999|ɹ`ən↑ ts⁼ɹ↓s`ən→ neɪ↓ts⁼aɪ↓ t⁼ə pʰin↑fa↑ joʊ↑tsʰɹ↓↑ t⁼ɑʊ↓↑ts`⁼ɹ`↓ lə waɪ↓ ts⁼aɪ↓ tsʰaɪ↑fu↓ t⁼ə pʰin↑fa↑. -21.wav|999|ts`⁼ɹ`↓ɥ↑ liŋ↓waɪ↓ liɑŋ↓↑ʃiɑŋ↓ ɹ`ən↑s`əŋ→ xɑʊ↓↑ts`ʰu↓ t⁼ə ts`⁼ʊŋ↓iɑʊ↓ʃiŋ↓, -22.wav|999|p⁼u↓ ʃɥ→iɑʊ↓ wo↓↑ tʰə↓p⁼iɛ↑tʃʰiɑŋ↑t⁼iɑʊ↓. -23.wav|999|tsʰaɪ↑ts`ʰan↓↑ t⁼ə tʃ⁼ja↓ts`⁼ɹ`↑ ts⁼aɪ↓ t⁼ɑŋ→tʃ⁼in→ s`ɹ`↓ ɹ`ən↑ swo↓↑ k⁼ʊŋ→ɹ`ən↓ t⁼ə, -24.wav|999|jʊŋ↓p⁼u↓ts`⁼ə weɪ↓ tʃʰi↑ ʃɥæn→ts`ʰwan↑ tʃ⁼iɛ↓s`ɑʊ↓. -25.wav|999|p⁼i↓↑tʃʰi↓↑ t⁼i↓əɹ`↓ʃiɑŋ↓ t⁼ə xɑʊ↓↑ts`ʰu↓, -26.wav|999|t⁼i↓san→ʃiɑŋ↓ t⁼ə xɑʊ↓↑ts`ʰu↓ tʃ⁼ɥ↓joʊ↓↑ i→ts`⁼ʊŋ↓↑ ʃiɑŋ→t⁼ɑŋ→ pʰiɑʊ→miɑʊ↓↑ t⁼ə ts`ʰəŋ↑fən↓, -27.wav|999|in→weɪ↓ miŋ↑ɥ↓, miŋ↑uɑŋ↓, t⁼i↓weɪ↓ t⁼əŋ↓↑. -28.wav|999|tʃʰɥæn↑ joʊ↑ tʰa→ɹ`ən↑ t⁼ə i↓tʃ⁼jɛn↓ k⁼oʊ↓ts`ʰəŋ↑. -29.wav|999|meɪ↓↑ɹ`ən↑ t⁼oʊ→ kʰə↓↑i↓↑ ts`⁼əŋ→tʃʰɥ↓↑ t⁼ə↑t⁼ɑʊ↓ miŋ↑ɥ↓, -30.wav|999|i↓ tʃ⁼i↑ tʃʰiŋ→p⁼aɪ↑ t⁼ə miŋ↑s`əŋ→, -31.wav|999|t⁼an↓ s`ə↓xweɪ↓ t⁼i↓weɪ↓, ts⁼ə↑ ts`⁼ɹ`↓↑joʊ↓↑ ɥɛ↓ pʰan↓ k⁼wo↑tʃ⁼ja→ ts`⁼əŋ↓fu↓↑ t⁼ə ɹ`ən↑tsʰaɪ↑ nəŋ↑ ɹ`an↓↑ts`⁼ɹ`↓↑, -32.wav|999|ts`⁼ɹ`↓ɥ↑ ʃjɛn↓↑xə↓ t⁼ə miŋ↑uɑŋ↓ tʃ⁼joʊ↓ ts`⁼ɹ`↓↑joʊ↓↑ tʃ⁼i↑s`ɑʊ↓↑s`u↓ ɹ`ən↑tsʰaɪ↑ xweɪ↓ t⁼ə↑t⁼ɑʊ↓. -33.wav|999|ts⁼aɪ↓ swo↓↑joʊ↓↑ ts`⁼ə↓ʃiɛ→ t⁼ɑŋ→ts`⁼ʊŋ→, -34.wav|999|miŋ↑ɥ↓ s`ɹ`↓ mi↑ts⁼u↑ts`⁼ən→k⁼weɪ↓ t⁼ə, -35.wav|999|ʃjɛn↓↑xə↓ t⁼ə miŋ↑uɑŋ↓ ts⁼ə↑ s`ɹ`↓ ɹ`ən↑ swo↓↑ ʃi→uɑŋ↓ t⁼ə↑t⁼ɑʊ↓ t⁼ə tʃ⁼ja↓ts`⁼ɹ`↑ ts`⁼ɹ`↓ɑŋ↑ t⁼ə t⁼ʊŋ→ʃi→, -36.wav|999|na↓ s`ɹ`↓ tʰjɛn→ts`⁼ɹ`→tʃ⁼iɑʊ→ts⁼ɹ tsʰaɪ↑nəŋ↑ t⁼ə↑t⁼ɑʊ↓ t⁼ə tʃ⁼in→ iɑŋ↑mɑʊ↑. -37.wav|999|liŋ↓i↓fɑŋ→mjɛn↓, -38.wav|999|ts`⁼ɹ`↓↑joʊ↓↑ s`a↓↑k⁼wa→ tsʰaɪ↑ xweɪ↓ p⁼a↓↑ s`ə↓xweɪ↓ t⁼i↓weɪ↓ fɑŋ↓ts`⁼ɹ`↓ ts⁼aɪ↓ tsʰaɪ↑ts`ʰan↓↑ ts`⁼ɹ`→tʃʰjɛn↑. -39.wav|999|liŋ↓waɪ↓, ɹ`ən↑ jʊŋ→joʊ↓↑ t⁼ə tsʰaɪ↑ts`ʰan↓↑, u↓pʰin↓↑ xə↑ miŋ↑ɥ↓, s`əŋ→uɑŋ↓, -40.wav|999|s`ɹ`↓ ts`ʰu↓↑ɥ↑ i→ts`⁼ʊŋ↓↑ swo↓↑weɪ↓ t⁼ə xu↓weɪ↓ iŋ↓↑ʃiɑŋ↓↑, tsʰu↓tʃ⁼in↓ t⁼ə k⁼wan→ʃi↓. -41.wav|999|p⁼i↓↑t⁼ə↑ ni↑sɹ→ s`wo→ k⁼wo↓,“ i↑k⁼ə↓ ɹ`ən↑ swo↓↑ jʊŋ→joʊ↓↑ t⁼ə tsʰaɪ↑ts`ʰan↓↑ tʃ⁼ɥɛ↑t⁼iŋ↓ lə ts`⁼ə↓k⁼ə↓ ɹ`ən↑ ts⁼aɪ↓ tʰa→ɹ`ən↑ jɛn↓↑ts`⁼ʊŋ→ t⁼ə tʃ⁼ja↓ts`⁼ɹ`↑”. -42.wav|999|ɹ`u↑k⁼wo↓↑ ts`⁼ə↓tʃ⁼ɥ↓ xwa↓ s`ɹ`↓ ts`⁼əŋ↓tʃʰɥɛ↓ t⁼əxwa↓, -43.wav|999|na↓mə, fan↓↑k⁼wo↓laɪ↑, tʰa→ɹ`ən↑ t⁼weɪ↓ ts⁼ɹ↓tʃ⁼i↓↑ t⁼ə liɑŋ↑xɑʊ↓↑ pʰiŋ↑tʃ⁼ja↓, -44.wav|999|nəŋ↑i↓↑ k⁼ə↓ts`⁼ʊŋ↓↑ ʃiŋ↑s`ɹ`↓ p⁼ɑŋ→ts`⁼u↓ ts⁼ɹ↓tʃ⁼i↓↑ xwo↓tʃʰɥ↓↑ tsʰaɪ↑ts`ʰan↓↑. -45.wav|999|nəŋ↑i↓↑ k⁼ə↓ts`⁼ʊŋ↓↑ ʃiŋ↑s`ɹ`↓ p⁼ɑŋ→ts`⁼u↓ ts⁼ɹ↓tʃ⁼i↓↑ xwo↓tʃʰɥ↓↑ tsʰaɪ↑ts`ʰan↓↑. -46.wav|999|soʊ fɑɹ, ðə ˈpɹɑbləmz hæv əˈkəɹd ɪn ðə ˈjuˈɛs. -47.wav|999|ðə ˈsteɪtmənt kənˈteɪnd noʊ səˈpɹaɪzɪz. -48.wav|999|ðə pəɹˈfɔɹməns kənˈtɪnjuz tɪ ˌɪmˈpɹuv. -49.wav|999|ðət wʊd bi ə ˈdeɪndʒəɹ. -50.wav|999|ˈfɹæŋkli, wi wəɹ ˈləki tɪ gɪt ˈsɛkənd. -51.wav|999|ˌhaʊˈɛvəɹ, noʊ ˈfəɹðəɹ ˈækʃən wɑz ˈteɪkən baɪ pəˈlis. -52.wav|999|ɪt dɪˈpɛndz ɔn ˈleɪbəɹ, nɑt ɔn ˈjuˈɛs. -53.wav|999|aɪ hæd tɪ bi ˈɪntu ɪt. -54.wav|999|maɪ vju həz ˈɔlˌweɪz bɪn ðə seɪm. -55.wav|999|ɪn fækt, hi ɪz nɑt ˈivɪn ɪn ðə skwɑd fəɹ ðə geɪm. -56.wav|999|ɪt ɪz sɛt ɪn ˈpɛɹɪs. -57.wav|999|du ju θɪŋk wɪɹ ə tɔp ˈneɪʃən. -58.wav|999|ɪz ɑɹ ˈtʃɪldɹən ˈləɹnɪŋ. -59.wav|999|haʊ ˌɪndɪˈpɛndənt ɪz ðət. -60.wav|999|wət fɔɹm dɪd ðət teɪk. -61.wav|999|əɹ ðeɪ fɹi. -62.wav|999|ðə ˈpɑɹti həz ˈnɛvəɹ ˈfʊli ɹɪˈkəvəɹd. -63.wav|999|ɪn ˈmɛni weɪz, ðət ɪz ɛz ˌɪmˈpɔɹtənt. -64.wav|999|hɪz ˈlidəɹ ˈwɔntɪd tɪ ˈsɛləˌbɹeɪt. -65.wav|999|ɪt ɪz nɑt ən ˈɔpʃən, bət ə ˈpɑləsi ɹɪkˈwaɪɹmənt. -66.wav|999|əv ˈfəɹðəɹ ˈpɹaɪvəsi, hi hæd noʊ nid. -67.wav|999|hi wɑz gʊd, bət nɑt ðət gʊd. -68.wav|999|ðə ɹɪˈzəɫts əɹ ɪkˈspɛktɪd wɪˈθɪn deɪz. -69.wav|999|ðə ˈkəmpəˌni stɪɫ hæd ðə ˈkɑnfədɛns əv ɪts ˈbæŋkəɹz, hi sɛd. -70.wav|999|ˈmɪstəɹ ˈkɹɔfəɹd ɪz noʊ ˈstɹeɪndʒəɹ tɪ ˈskɑtɪʃ ˈɛnəɹˌpɹaɪz. -71.wav|999|wi wəɹ wɛɫ wəɹθ ðə wɪn. -72.wav|999|ənd ɪt wɑz ðɪs wən. -73.wav|999|bət ðæts əˈnəðəɹ ˈstɔɹi fəɹ əˈnəðəɹ deɪ. -74.wav|999|bət ɪt wɑz ˈɔlsoʊ hɑɹd. -75.wav|999|ɪf soʊ, ɪt ɪz ə ˈfəni taɪm tɪ ˌɪntɹəˈdus ɪt. -76.wav|999|wɪɹ ˈlʊkɪŋ fəɹ ˈjunɪti ɪn ðə ˈkaʊnsəɫ. -77.wav|999|baɪ ðɛn, ə ˈmæsɪv ˈligəɫ ˈbætəɫ ɪz ˈlaɪkli tɪ hæv ˈstɑɹtɪd. -78.wav|999|ðeɪ əɹ nɑt goʊɪŋ ðə lɛŋθ əv ðə pɪtʃ. -79.wav|999|ˌhaʊˈɛvəɹ, hi ˈlæstɪd ˈoʊnli faɪv deɪz ˌbiˈfɔɹ goʊɪŋ ɔn ðə ɹən. -80.wav|999|ðə kənˈsəɹnz əɹ ðə seɪm. -81.wav|999|ðət wɑz ə ˈboʊnəs, bət ɪt wɑz nɑt ðə meɪn əˈbdʒɛktɪv. -82.wav|999|ˈskɑtlənd wən baɪ sɪks ˈwɪkəts. -83.wav|999|ɪt həz tɪ bi ɛnˈfɔɹst. -84.wav|999|ənd wi kən du ɪt. -85.wav|999|ˈɛvəɹi wən ɪz ə ˈwɪnəɹ. -86.wav|999|dɹəgz əɹ juzd ə lɔt æt ðə ˈfɪʃɪŋ, nɑt dʒɪst ˈkænəbəs. -87.wav|999|ðeɪ hæv sɛt ðə ˈstændəɹd. -88.wav|999|ðɪs ɪz ə ˈmeɪdʒəɹ ˈpɹɑpəɹti ɪn ˈɛdənbəɹoʊ. -89.wav|999|ɪt ɪz ə ˈkɹaɪsəs əv ˈjumən ɹaɪts, ə ˈkɹaɪsəs əv ˈlidəɹˌʃɪp. -90.wav|999|ðə ɪgˈzɪstɪŋ ˈstɹəktʃəɹ ɪz ɪn ˈtɹəbəɫ. -91.wav|999|ɪt ɔɫ ˈstɑɹtɪd wɪθ ə ˈvɪzɪtɪŋ ˈɹəgbi kləb. -92.wav|999|hi ɹɪfˈjuzd tɪ ɹɪˈviɫ ðə ˈɹizənz fəɹ ðə splɪt. diff --git a/user_voice_collect.py b/user_voice_collect.py deleted file mode 100644 index 4785dd1..0000000 --- a/user_voice_collect.py +++ /dev/null @@ -1,72 +0,0 @@ -import numpy as np -import torch -import torchaudio -import gradio as gr -import os - -anno_lines = [] -with open("./user_voice/user_voice.txt", 'r', encoding='utf-8') as f: - for line in f.readlines(): - anno_lines.append(line.strip("\n")) - -text_index = 0 - -def display_text(index): - index = int(index) - global text_index - text_index = index - return f"{text_index}: " + anno_lines[index].split("|")[2].strip("[ZH]").strip("[EN]") - -def display_prev_text(): - global text_index - if text_index != 0: - text_index -= 1 - return f"{text_index}: " + anno_lines[text_index].split("|")[2].strip("[ZH]").strip("[EN]") - -def display_next_text(): - global text_index - if text_index != len(anno_lines)-1: - text_index += 1 - return f"{text_index}: " + anno_lines[text_index].split("|")[2].strip("[ZH]").strip("[EN]") - -def save_audio(audio): - global text_index - if audio: - sr, wav = audio - wav = torch.tensor(wav).type(torch.float32) / max(wav.max(), -wav.min()) - wav = wav.unsqueeze(0) if len(wav.shape) == 1 else wav - if sr != 22050: - res_wav = torchaudio.transforms.Resample(orig_freq=sr, new_freq=22050)(wav) - else: - res_wav = wav - torchaudio.save(f"./user_voice/{str(text_index)}.wav", res_wav, 22050, channels_first=True) - return f"Audio saved to ./user_voice/{str(text_index)}.wav successfully!" - else: - return "Error: Please record your audio!" - - -if __name__ == "__main__": - app = gr.Blocks() - with app: - with gr.Row(): - text = gr.Textbox(value="0: " + anno_lines[0].split("|")[2].strip("[ZH]"), label="Please read the text here") - with gr.Row(): - audio_to_collect = gr.Audio(source="microphone") - with gr.Row(): - with gr.Column(): - prev_btn = gr.Button(value="Previous") - with gr.Column(): - next_btn = gr.Button(value="Next") - with gr.Row(): - index_dropdown = gr.Dropdown(choices=[str(i) for i in range(len(anno_lines))], value="0", - label="No. of text", interactive=True) - with gr.Row(): - with gr.Column(): - save_btn = gr.Button(value="Save Audio") - with gr.Column(): - audio_save_message = gr.Textbox(label="Message") - index_dropdown.change(display_text, inputs=index_dropdown, outputs=text) - prev_btn.click(display_prev_text, inputs=None, outputs=text) - next_btn.click(display_next_text, inputs=None, outputs=text) - save_btn.click(save_audio, inputs=audio_to_collect, outputs=audio_save_message) - app.launch() \ No newline at end of file diff --git a/whisper_transcribe.py b/whisper_transcribe.py deleted file mode 100644 index 3bd1f1d..0000000 --- a/whisper_transcribe.py +++ /dev/null @@ -1,90 +0,0 @@ -import whisper -import os -import torchaudio -import json - -lang2token = { - 'zh': "[ZH]", - 'ja': "[JA]", - "en": "[EN]", -} - - -def transcribe_one(audio_path): - # load audio and pad/trim it to fit 30 seconds - audio = whisper.load_audio(audio_path) - audio = whisper.pad_or_trim(audio) - - # make log-Mel spectrogram and move to the same device as the model - mel = whisper.log_mel_spectrogram(audio).to(model.device) - - # detect the spoken language - _, probs = model.detect_language(mel) - print(f"Detected language: {max(probs, key=probs.get)}") - lang = max(probs, key=probs.get) - # decode the audio - options = whisper.DecodingOptions() - result = whisper.decode(model, mel, options) - - # print the recognized text - print(result.text) - return lang, result.text -if __name__ == "__main__": - - model = whisper.load_model("medium") - parent_dir = "./custom_character_voice/" - speaker_names = list(os.walk(parent_dir))[0][1] - speaker2id = {} - speaker_annos = [] - # resample audios - for speaker in speaker_names: - speaker2id[speaker] = 1000 + len(speaker2id) - for i, wavfile in enumerate(list(os.walk(parent_dir + speaker))[0][2]): - # try to load file as audio - if wavfile.startswith("processed_"): - continue - try: - wav, sr = torchaudio.load(parent_dir + speaker + "/" + wavfile, frame_offset=0, num_frames=-1, normalize=True, - channels_first=True) - wav = wav.mean(dim=0).unsqueeze(0) - if sr != 22050: - wav = torchaudio.transforms.Resample(orig_freq=sr, new_freq=22050)(wav) - if wav.shape[1] / sr > 20: - print(f"{wavfile} too long, ignoring\n") - save_path = parent_dir + speaker + "/" + f"processed_{i}.wav" - torchaudio.save(save_path, wav, 22050, channels_first=True) - # transcribe text - lang, text = transcribe_one(save_path) - if lang not in ['zh', 'en', 'ja']: - print(f"{lang} not supported, ignoring\n") - text = lang2token[lang] + text + lang2token[lang] + "\n" - speaker_annos.append(save_path + "|" + str(speaker2id[speaker]) + "|" + text) - except: - continue - - # clean annotation - import text - cleaned_speaker_annos = [] - for i, line in enumerate(speaker_annos): - path, sid, txt = line.split("|") - if len(txt) > 100: - continue - cleaned_text = text._clean_text(txt, ["cjke_cleaners2"]) - cleaned_text += "\n" if not cleaned_text.endswith("\n") else "" - cleaned_speaker_annos.append(path + "|" + sid + "|" + cleaned_text) - with open("custom_character_anno.txt", 'w', encoding='utf-8') as f: - for line in cleaned_speaker_annos: - f.write(line) - # generate new config - with open("./configs/finetune_speaker.json", 'r', encoding='utf-8') as f: - hps = json.load(f) - - # modify n_speakers - hps['data']["n_speakers"] = 1000 + len(speaker2id) - # add speaker names - for speaker in speaker_names: - hps['speakers'][speaker] = speaker2id[speaker] - # save modified config - with open("./configs/modified_finetune_speaker.json", 'w', encoding='utf-8') as f: - json.dump(hps, f, indent=2) - print("finished") \ No newline at end of file