From c39ce766afc023b01115e020a7ca9d79de799da0 Mon Sep 17 00:00:00 2001 From: "Azalea (on HyDEV-Daisy)" Date: Sat, 1 Oct 2022 13:41:07 -0400 Subject: [PATCH] [U] Backup unfinished changes --- experiment/accuracy_calculation.py | 2 + experiment/ina_main.py | 49 +++++++++++++++- experiment/test_fft.py | 72 +++++++++++++++++++++++ experiment/validate_cn_celeb.py | 91 ++++++++++++++++++------------ experiment/vox1.py | 78 +++++++++++++++++++++++++ 5 files changed, 255 insertions(+), 37 deletions(-) create mode 100644 experiment/test_fft.py create mode 100644 experiment/vox1.py diff --git a/experiment/accuracy_calculation.py b/experiment/accuracy_calculation.py index 35a322f..9535c06 100644 --- a/experiment/accuracy_calculation.py +++ b/experiment/accuracy_calculation.py @@ -8,6 +8,8 @@ if __name__ == '__main__': with open(r'C:\Datasets\CN-Celeb_flac\ina_pf_map.json', 'r', encoding='UTF-8') as f: pf = json.load(f) + print(len(labels)) + correct_f = [] correct_m = [] incorrect_f = [] diff --git a/experiment/ina_main.py b/experiment/ina_main.py index 5032e82..6ca2d07 100644 --- a/experiment/ina_main.py +++ b/experiment/ina_main.py @@ -38,8 +38,53 @@ class Result(NamedTuple): file: str -def segment(file) -> list[ResultFrame]: - return [ResultFrame(*s) for s in seg(file)] +class BatchResults(NamedTuple): + results: list[Result] + time_full: float + time_avg: float + successes: int + messages: list[tuple[str, int]] + + +def _process_helper(args: tuple[Segmenter, str]) -> Result: + seg, f = args + lseg = seg(f) + return Result([ResultFrame(*s) for s in lseg], f) + + +def process(self: Segmenter, inp: list[str], tmpdir=None, verbose=False, skip_if_exist=False, + nbtry=1, try_delay=2.) -> list[Result]: + from tqdm.contrib.concurrent import process_map + import tqdm + results = [] + for i in tqdm.tqdm(inp): + results.append(_process_helper((self, i))) + return results + + # return process_map(_process_helper, [(self, i) for i in inp], max_workers=2) + + # + # t_batch_start = time.time() + # + # results: list[Result] = [] + # lmsg = [] + # fg = featGenerator(inp.copy(), inp.copy(), tmpdir, self.ffmpeg, skip_if_exist, nbtry, try_delay) + # i = 0 + # for feats, msg in fg: + # lmsg += msg + # i += len(msg) + # if verbose: + # print('%d/%d' % (i, len(inp)), msg) + # if feats is None: + # break + # mspec, loge, diff_len = feats + # lseg = self.segment_feats(mspec, loge, diff_len, 0) + # results.append() + # + # t_batch_dur = time.time() - t_batch_start + # nb_processed = len([e for e in lmsg if e[1] == 0]) + # avg = t_batch_dur / nb_processed if nb_processed else -1 + # return BatchResults(results, t_batch_dur, avg, nb_processed, lmsg) def to_wav(file: str, callback: Callable, start_sec: float = 0, stop_sec: float = 0): diff --git a/experiment/test_fft.py b/experiment/test_fft.py new file mode 100644 index 0000000..4ff27a3 --- /dev/null +++ b/experiment/test_fft.py @@ -0,0 +1,72 @@ +import librosa +import librosa.display +import matplotlib.pyplot as plt +import numpy as np +import parselmouth + +import sgs + +if __name__ == '__main__': + librosa.filters.mel(sr=16000, n_fft=1024, htk=True) + + f = 'Z:/EECS 6414/voice_cnn/test.wav' + y, sr = librosa.load(f, sr=16000) + + # Plot waveform + # plt.plot(y) + # plt.title('Signal') + # plt.xlabel('Time (samples)') + # plt.ylabel('Amplitude') + # plt.show() + # plt.clf() + + # Plot frequency domain graph at a single time + n_fft = 2048 + ft = np.abs(librosa.stft(y[:n_fft], hop_length=n_fft + 1)) + + # plt.plot(ft) + # plt.title('Spectrum') + # plt.xlabel('Frequency Bin') + # plt.ylabel('Amplitude') + # plt.show() + # plt.clf() + + # Plot spectrogram + spec = np.abs(librosa.stft(y, n_fft=1024, hop_length=512)) + # spec = librosa.amplitude_to_db(spec, ref=np.max) + + # librosa.display.specshow(spec, sr=sr, x_axis='time', y_axis='log') + # plt.colorbar(format='%+2.0f dB') + # plt.title('Spectrogram') + # plt.show() + # plt.clf() + + # Mel transform + mel_spect = librosa.feature.melspectrogram(y=y, sr=sr, n_fft=2048, hop_length=512, htk=True) + mel_spect = librosa.power_to_db(mel_spect, ref=np.max) + + print(len(mel_spect)) + librosa.display.specshow(mel_spect, y_axis='mel', fmax=8000, x_axis='time', n_fft=1024, hop_length=512) + result, freq_array = sgs.api.calculate_feature_classification(parselmouth.Sound(f)) + + pitch_array = freq_array[:, 0] + # x_len = len(pitch_array) / len(mel_spect) + # x = np.arange(len(mel_spect)) + # y = [] + # for x in range(len(mel_spect) // 2): + # y.append(float(np.mean(pitch_array[int(x_len * x):int(x_len * (x + 1))]))) + # print(len(y)) + x = np.linspace(0, 4.1) + print(x) + x_len = len(pitch_array) / len(x) + y = [] + for a in range(len(x)): + y.append(np.mean(pitch_array[int(x_len * a):int(x_len * (a + 1))])) + + plt.plot(x, y, color='#7bff4f') + plt.plot(x, [100] * len(x), color='#7bff4f') + plt.yticks([0,100,200,300,400,500,600,700,800,900,1000,1200,1400,1600]) + plt.title('Mel Spectrogram') + plt.colorbar(format='%+2.0f dB') + plt.show() + plt.clf() \ No newline at end of file diff --git a/experiment/validate_cn_celeb.py b/experiment/validate_cn_celeb.py index 3fa8a52..b4358e2 100644 --- a/experiment/validate_cn_celeb.py +++ b/experiment/validate_cn_celeb.py @@ -1,69 +1,81 @@ import json import os +# os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' import warnings from pathlib import Path import matplotlib.pyplot as plt import numpy as np -import pygame +import tensorflow as tf from inaSpeechSegmenter import Segmenter from ina_main import process, get_result_percentages from utils import color, printc +gpu_devices = tf.config.experimental.list_physical_devices('GPU') +for device in gpu_devices: + tf.config.experimental.set_memory_growth(device, True) + + def segment_all(): # Create segmenter seg = Segmenter() np.seterr(invalid='ignore') # Loop through all celebrities - for id in ids: - id_dir = data_dir.joinpath(id) + for id in ids[559:]: + id_dir = data_dir / id + + if (id_dir / 'total.json').is_file(): + continue # Loop through all recordings (Exclude singing for now) - utters = [r for r in os.listdir(id_dir) if r.endswith('.flac') - and not r.startswith('singing')] + utters = audio_files[id] # Exclude existing - utters = [id_dir.joinpath(u) for u in utters] - utters = [u for u in utters if not u.with_suffix('.json').exists()] + utters = [id_dir.joinpath(u) for u in utters if u.endswith('.wav')] + # utters = [u for u in utters if not u.with_suffix('.json').exists()] + if len(utters) == 0: continue # Analyze + print(f'Processing {id}') results = process(seg, [str(u) for u in utters], verbose=True) # Write results - total = [0, 0, 0, 0, 0] - type_totals = {} - for result in results.results: + # total = [0, 0, 0, 0, 0] + # type_totals = {} + total = [] + for result in results: file = Path(result.file).with_suffix('.json') # Get results # f: Frames, r: Ratios - ratios = [round(r, 3) for r in get_result_percentages(result)] - stored = {'f': result.frames, 'r': ratios} + _, _, _, pf = get_result_percentages(result) + total.append(pf) # Count type total (type_totals[utter_type][-1] is the count) - file_name = file.name - utter_type = file_name[:file_name.index('-')] - type_totals.setdefault(utter_type, [0, 0, 0, 0, 0]) - for i in range(4): - type_totals[utter_type][i] += ratios[i] - total[i] += ratios[i] - type_totals[utter_type][-1] += 1 - total[-1] += 1 + # file_name = file.name + # utter_type = file_name[:file_name.index('-')] + # type_totals.setdefault(utter_type, [0, 0, 0, 0, 0]) + # for i in range(4): + # type_totals[utter_type][i] += ratios[i] + # total[i] += ratios[i] + # type_totals[utter_type][-1] += 1 + # total[-1] += 1 # Write result - file.write_text(json.dumps(stored)) + # file.write_text(json.dumps(ratios)) # Write type averages - type_averages = {t: [r / type_totals[t][-1] for r in type_totals[t][:-1]] for t in type_totals} - total_average = [r / total[-1] for r in total[:-1]] - obj = {'type_averages': type_averages, 'total_averages': total_average} - id_dir.joinpath('total.json').write_text(json.dumps(obj)) + # type_averages = {t: [r / type_totals[t][-1] for r in type_totals[t][:-1]] for t in type_totals} + # total_average = [r / total[-1] for r in total[:-1]] + # obj = {'type_averages': type_averages, 'total_averages': total_average} + # id_dir.joinpath('total.json').write_text(json.dumps(obj)) + id_dir.joinpath('total.json').write_text(json.dumps({'ratio': np.nanmean(total)})) def graph_histogram(): @@ -107,7 +119,7 @@ def manually_label_data(): Since CN-Celeb isn't labelled with the speaker's gender, this script is used to manually label them. """ - pygame.mixer.init() + # pygame.mixer.init() # Load existing labels labels_json = data_dir.joinpath('id_labels.json') @@ -132,13 +144,13 @@ def manually_label_data(): tracks = [f for f in os.listdir(id_dir) if f.endswith('.flac')] for track_i, audio in enumerate(tracks): # Play track - sound = pygame.mixer.Sound(id_dir.joinpath(audio)) - sound.play() + # sound = pygame.mixer.Sound(id_dir.joinpath(audio)) + # sound.play() i = input(color( f'\n&7Playing speaker {id[-3:]}/{len(ids)} - track {track_i}/{len(tracks)} - {audio}&r' - f'\n- Press f / m, or anything else to play next track: '))\ + f'\n- Press f / m, or anything else to play next track: ')) \ .lower().strip() - sound.stop() + # sound.stop() # Skip if i == 's': @@ -159,10 +171,19 @@ def manually_label_data(): if __name__ == '__main__': - cn_celeb_root = Path('C:/Users/me/Workspace/Data/CN-Celeb_flac') - data_dir = cn_celeb_root.joinpath('data') - ids = [id for id in os.listdir(data_dir) if id.startswith('id0')] + cn_celeb_root = Path(r'C:\Datasets\VoxCeleb1\wav') - # segment_all() + data_dir = cn_celeb_root + ids = [id for id in os.listdir(data_dir) if id.startswith('id1')] + + # Get all audio files for each id + audio_files = {} + for id in ids[559:]: + audio_files[id] = [] + for dirpath, dirnames, filenames in os.walk(data_dir / id): + audio_files[id] += [os.path.join(dirpath, file) for file in filenames if file.endswith('.wav')] + # print(audio_files.keys()) + + segment_all() # graph_histogram() - manually_label_data() + # manually_label_data() diff --git a/experiment/vox1.py b/experiment/vox1.py new file mode 100644 index 0000000..fe2a3e1 --- /dev/null +++ b/experiment/vox1.py @@ -0,0 +1,78 @@ + +from pathlib import Path +import os +import json + +import numpy as np +import pandas as pd + +if __name__ == '__main__': + cn_celeb_root = Path(r'C:\Datasets\vox1_test_wav\wav') + data_dir = cn_celeb_root + ids = [id for id in os.listdir(data_dir) if id.startswith('id1')] + # ids=[data_dir / id for id in os.listdir(data_dir) if id.startswith('id1')] + + appendix= Path(r'C:\Datasets\VoxCeleb1\wav') + ids += [id for id in os.listdir(appendix) if id.startswith('id1')] + # ids += [appendix / id for id in os.listdir(appendix) if id.startswith('id1')] + with open(r"C:\Datasets\vox1_label.csv") as f: + txt = f.read().strip() + map = {} + for row in txt.split('\n'): + id, gender = row.split(',') + map[id] = gender + # female-> positive, male -> negative + f_correct = 0 #tp + f_incorrect = 0 #fp + m_correct = 0 #tn + m_incorrect = 0 #fn + for id in ids[:40]: + obj = json.loads((data_dir / id / 'total.json').read_text()) + + label = map[id] #ground truth + if label == 'f': + if obj['ratio'] >= 0.5: + f_correct += 1 + else: + # f_incorrect += 1 #fn + m_incorrect += 1 + if label == 'm': + if obj['ratio'] < 0.5: + m_correct += 1 + else: + # m_incorrect += 1 #fp + f_incorrect += 1 + for id in ids[40:]: + obj = json.loads((appendix / id / 'total.json').read_text()) + + label = map[id] #ground truth + if label == 'f': + if obj['ratio'] >= 0.5: + f_correct += 1 + else: + # f_incorrect += 1 #fn + m_incorrect += 1 + if label == 'm': + if obj['ratio'] < 0.5: + m_correct += 1 + else: + # m_incorrect += 1 #fp + f_incorrect += 1 + + + # print(f_incorrect) + # print(m_incorrect) + + f_precision = f_correct / (f_correct + f_incorrect) + f_recall = f_correct / (f_correct + m_incorrect) + + m_precision = m_correct / (m_correct + m_incorrect) + m_recall = m_correct / (m_correct + f_incorrect) + + print('Precision_f', f_precision) + print('Recall_f', f_recall) + + print('Precision_m', m_precision) + print('Recall_m', m_recall) + + print('total number: ', f_incorrect+f_correct+m_incorrect+m_correct)