From ea491a457f46502dea041450413ec0d29aef0e00 Mon Sep 17 00:00:00 2001 From: lrioxh Date: Tue, 7 Mar 2023 09:39:56 +0800 Subject: [PATCH 1/3] Add tags via re --- utils.py | 132 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/utils.py b/utils.py index c9ba316..eee178a 100644 --- a/utils.py +++ b/utils.py @@ -8,6 +8,7 @@ import subprocess import numpy as np from scipy.io.wavfile import read import torch +import regex as re MATPLOTLIB_FLAG = False @@ -15,6 +16,137 @@ logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) logger = logging + +zh_pattern = re.compile(r'[\u4e00-\u9fa5]') +en_pattern = re.compile(r'[a-zA-Z]') +jp_pattern = re.compile(r'[\u3040-\u30ff\u31f0-\u31ff]') +kr_pattern = re.compile(r'[\uac00-\ud7af\u1100-\u11ff\u3130-\u318f\ua960-\ua97f]') +num_pattern=re.compile(r'[0-9]') +comma=r"(?<=[.。!！?？；;，,、:：'\"‘“”’()（）《》「」~——])" #向前匹配但固定长度 +tags={'ZH':'[ZH]','EN':'[EN]','JP':'[JA]','KR':'[KR]'} + +def restart_program(): + python = sys.executable + os.execl(python, python, * sys.argv) + +def tag_cjke(text): + '''为中英日韩加tag,中日正则分不开，故先分句分离中日再识别，以应对大部分情况''' + sentences = re.split(r"([.。!！?？；;，,、:：'\"‘“”’()（）【】《》「」~——]+ *(?![0-9]))", text) #分句排除小数点 + sentences.append("") + sentences = ["".join(i) for i in zip(sentences[0::2],sentences[1::2])] + # print(sentences) + prev_lang=None + tagged_text = "" + for s in sentences: + nu = re.sub(r'[\s\p{P}]+', '', s, flags=re.U).strip() + if len(nu)==0: + continue + s = re.sub(r'[()（）《》「」【】‘“”’]+', '', s) + jp=re.findall(jp_pattern, s) + if len(jp)>0: + prev_lang,tagged_jke=tag_jke(s,prev_lang) + tagged_text +=tagged_jke + else: + prev_lang,tagged_cke=tag_cke(s,prev_lang) + tagged_text +=tagged_cke + return tagged_text + +def tag_jke(text,prev_sentence=None): + '''为英日韩加tag''' + # 初始化标记变量 + tagged_text = "" + prev_lang = None + tagged=0 + # 遍历文本 + for char in text: + # 判断当前字符属于哪种语言 + if jp_pattern.match(char): + lang = "JP" + elif zh_pattern.match(char): + lang = "JP" + elif kr_pattern.match(char): + lang = "KR" + elif en_pattern.match(char): + lang = "EN" + # elif num_pattern.match(char): + # lang = prev_sentence + else: + lang = None + tagged_text += char + continue + # 如果当前语言与上一个语言不同，就添加标记 + if lang != prev_lang: + tagged=1 + if prev_lang==None: # 开头 + tagged_text =tags[lang]+tagged_text + else: + tagged_text =tagged_text+tags[prev_lang]+tags[lang] + + # 重置标记变量 + prev_lang = lang + + # 添加当前字符到标记文本中 + tagged_text += char + + # 在最后一个语言的结尾添加对应的标记 + if prev_lang: + tagged_text += tags[prev_lang] + if not tagged: + prev_lang=prev_sentence + tagged_text =tags[prev_lang]+tagged_text+tags[prev_lang] + + return prev_lang,tagged_text + +def tag_cke(text,prev_sentence=None): + '''为中英韩加tag''' + # 初始化标记变量 + tagged_text = "" + prev_lang = None + # 是否全略过未标签 + tagged=0 + + # 遍历文本 + for char in text: + # 判断当前字符属于哪种语言 + if zh_pattern.match(char): + lang = "ZH" + elif kr_pattern.match(char): + lang = "KR" + elif en_pattern.match(char): + lang = "EN" + # elif num_pattern.match(char): + # lang = prev_sentence + else: + # 略过 + lang = None + tagged_text += char + continue + + # 如果当前语言与上一个语言不同，添加标记 + if lang != prev_lang: + tagged=1 + if prev_lang==None: # 开头 + tagged_text =tags[lang]+tagged_text + else: + tagged_text =tagged_text+tags[prev_lang]+tags[lang] + + # 重置标记变量 + prev_lang = lang + + # 添加当前字符到标记文本中 + tagged_text += char + + # 在最后一个语言的结尾添加对应的标记 + if prev_lang: + tagged_text += tags[prev_lang] + # 未标签则继承上一句标签 + if tagged==0: + prev_lang=prev_sentence + tagged_text =tags[prev_lang]+tagged_text+tags[prev_lang] + return prev_lang,tagged_text + + + def load_checkpoint(checkpoint_path, model, optimizer=None, drop_speaker_emb=False): assert os.path.isfile(checkpoint_path) checkpoint_dict = torch.load(checkpoint_path, map_location='cpu') From 5aea258e26179e240f022ce95fc2046801ae43e7 Mon Sep 17 00:00:00 2001 From: lrioxh Date: Tue, 7 Mar 2023 09:44:43 +0800 Subject: [PATCH 2/3] Add tags via re --- utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.py b/utils.py index eee178a..f433b07 100644 --- a/utils.py +++ b/utils.py @@ -38,7 +38,7 @@ def tag_cjke(text): prev_lang=None tagged_text = "" for s in sentences: - nu = re.sub(r'[\s\p{P}]+', '', s, flags=re.U).strip() + nu = re.sub(r'[\s\p{P}]+', '', s, flags=re.U).strip() #全为字符跳过 if len(nu)==0: continue s = re.sub(r'[()（）《》「」【】‘“”’]+', '', s) From e7f0574fc8c5dc6d3653dc29778fcaf8ea7bdc4a Mon Sep 17 00:00:00 2001 From: lrioxh Date: Tue, 7 Mar 2023 09:58:05 +0800 Subject: [PATCH 3/3] Add tags via re --- utils.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/utils.py b/utils.py index f433b07..d72ea0f 100644 --- a/utils.py +++ b/utils.py @@ -25,24 +25,22 @@ num_pattern=re.compile(r'[0-9]') comma=r"(?<=[.。!！?？；;，,、:：'\"‘“”’()（）《》「」~——])" #向前匹配但固定长度 tags={'ZH':'[ZH]','EN':'[EN]','JP':'[JA]','KR':'[KR]'} -def restart_program(): - python = sys.executable - os.execl(python, python, * sys.argv) - def tag_cjke(text): '''为中英日韩加tag,中日正则分不开，故先分句分离中日再识别，以应对大部分情况''' - sentences = re.split(r"([.。!！?？；;，,、:：'\"‘“”’()（）【】《》「」~——]+ *(?![0-9]))", text) #分句排除小数点 + sentences = re.split(r"([.。!！?？；;，,、:：'\"‘“”’()（）【】《》「」~——]+ *(?![0-9]))", text) #分句，排除小数点 sentences.append("") sentences = ["".join(i) for i in zip(sentences[0::2],sentences[1::2])] # print(sentences) prev_lang=None tagged_text = "" for s in sentences: - nu = re.sub(r'[\s\p{P}]+', '', s, flags=re.U).strip() #全为字符跳过 + #全为符号跳过 + nu = re.sub(r'[\s\p{P}]+', '', s, flags=re.U).strip() if len(nu)==0: continue s = re.sub(r'[()（）《》「」【】‘“”’]+', '', s) jp=re.findall(jp_pattern, s) + #本句含日语字符判断为日语 if len(jp)>0: prev_lang,tagged_jke=tag_jke(s,prev_lang) tagged_text +=tagged_jke