dataset for experiments in check_novoapi is updated.
This commit is contained in:
parent
2004399179
commit
97486e5599
Binary file not shown.
@ -4,7 +4,7 @@
|
|||||||
<SchemaVersion>2.0</SchemaVersion>
|
<SchemaVersion>2.0</SchemaVersion>
|
||||||
<ProjectGuid>4d8c8573-32f0-4a62-9e62-3ce5cc680390</ProjectGuid>
|
<ProjectGuid>4d8c8573-32f0-4a62-9e62-3ce5cc680390</ProjectGuid>
|
||||||
<ProjectHome>.</ProjectHome>
|
<ProjectHome>.</ProjectHome>
|
||||||
<StartupFile>fame_hmm.py</StartupFile>
|
<StartupFile>check_novoapi.py</StartupFile>
|
||||||
<SearchPath>
|
<SearchPath>
|
||||||
</SearchPath>
|
</SearchPath>
|
||||||
<WorkingDirectory>.</WorkingDirectory>
|
<WorkingDirectory>.</WorkingDirectory>
|
||||||
|
@ -29,48 +29,47 @@ forced_alignment_novo70 = True
|
|||||||
|
|
||||||
|
|
||||||
## ===== load novo phoneset =====
|
## ===== load novo phoneset =====
|
||||||
phoneset_ipa, phoneset_novo70, translation_key_ipa2novo70, translation_key_novo702ipa = novoapi_functions.load_phonset()
|
phoneset_ipa, phoneset_novo70, translation_key_ipa2novo70, translation_key_novo702ipa = novoapi_functions.load_novo70_phoneset()
|
||||||
|
|
||||||
|
|
||||||
## ===== extract pronunciations written in novo70 only (not_in_novo70) =====
|
## ===== extract pronunciations written in novo70 only (not_in_novo70) =====
|
||||||
|
|
||||||
# As per Nederlandse phoneset_aki.xlsx recieved from David
|
|
||||||
# [ɔː] oh / ohr
|
|
||||||
# [ɪː] ih / ihr
|
|
||||||
# [iː] iy
|
|
||||||
# [œː] uh
|
|
||||||
# [ɛː] eh
|
|
||||||
# [w] wv in IPA written as ʋ.
|
|
||||||
david_suggestion = ['ɔː', 'ɪː', 'iː', 'œː', 'ɛː', 'w']
|
|
||||||
|
|
||||||
## read pronunciation variants.
|
## read pronunciation variants.
|
||||||
stimmen_transcription_ = pd.ExcelFile(default.stimmen_transcription_xlsx)
|
#stimmen_transcription_ = pd.ExcelFile(default.stimmen_transcription_xlsx)
|
||||||
df = pd.read_excel(stimmen_transcription_, 'frequency')
|
#df = pd.read_excel(stimmen_transcription_, 'frequency')
|
||||||
transcription_ipa = list(df['IPA'])
|
#transcription_ipa = list(df['IPA'])
|
||||||
|
|
||||||
# transcription mistake?
|
|
||||||
transcription_ipa = [ipa.replace(';', 'ː') for ipa in transcription_ipa if not ipa=='pypɪl' and not pd.isnull(ipa)]
|
|
||||||
transcription_ipa = [ipa.replace('ˑ', '') for ipa in transcription_ipa] # only one case.
|
|
||||||
|
|
||||||
not_in_novo70 = []
|
stimmen_test_dir = r'c:\OneDrive\Research\rug\_data\stimmen_test'
|
||||||
all_in_novo70 = []
|
df = stimmen_functions.load_transcriptions_novo70(stimmen_test_dir)
|
||||||
for ipa in transcription_ipa:
|
|
||||||
ipa = ipa.replace(':', 'ː')
|
|
||||||
ipa = convert_phone_set.split_ipa(ipa)
|
|
||||||
|
|
||||||
# list of phones not in novo70 phoneset.
|
|
||||||
not_in_novo70_ = [phone for phone in ipa
|
|
||||||
if not phone in phoneset_ipa and not phone in david_suggestion]
|
|
||||||
not_in_novo70_ = [phone.replace('sp', '') for phone in not_in_novo70_]
|
|
||||||
not_in_novo70_ = [phone.replace(':', '') for phone in not_in_novo70_]
|
|
||||||
not_in_novo70_ = [phone.replace('ː', '') for phone in not_in_novo70_]
|
|
||||||
|
|
||||||
if len(not_in_novo70_) == 0:
|
## transcription mistake?
|
||||||
all_in_novo70.append(''.join(ipa))
|
#transcription_ipa = [ipa.replace(';', 'ː') for ipa in transcription_ipa if not ipa=='pypɪl' and not pd.isnull(ipa)]
|
||||||
|
#transcription_ipa = [ipa.replace('ˑ', '') for ipa in transcription_ipa] # only one case.
|
||||||
|
|
||||||
#translation_key.get(phone, phone)
|
#not_in_novo70 = []
|
||||||
not_in_novo70.extend(not_in_novo70_)
|
#all_in_novo70 = []
|
||||||
not_in_novo70_list = list(set(not_in_novo70))
|
#for ipa in transcription_ipa:
|
||||||
|
# ipa = ipa.replace(':', 'ː')
|
||||||
|
# ipa = convert_phone_set.split_ipa(ipa)
|
||||||
|
|
||||||
|
# # list of phones not in novo70 phoneset.
|
||||||
|
# not_in_novo70_ = [phone for phone in ipa
|
||||||
|
# if not phone in phoneset_ipa and not phone in david_suggestion]
|
||||||
|
# not_in_novo70_ = [phone.replace('sp', '') for phone in not_in_novo70_]
|
||||||
|
# not_in_novo70_ = [phone.replace(':', '') for phone in not_in_novo70_]
|
||||||
|
# not_in_novo70_ = [phone.replace('ː', '') for phone in not_in_novo70_]
|
||||||
|
|
||||||
|
# if len(not_in_novo70_) == 0:
|
||||||
|
# all_in_novo70.append(''.join(ipa))
|
||||||
|
|
||||||
|
# #translation_key.get(phone, phone)
|
||||||
|
# not_in_novo70.extend(not_in_novo70_)
|
||||||
|
#not_in_novo70_list = list(set(not_in_novo70))
|
||||||
|
|
||||||
|
|
||||||
## check which phones used in stimmen but not in novo70
|
## check which phones used in stimmen but not in novo70
|
||||||
@ -85,41 +84,43 @@ not_in_novo70_list = list(set(not_in_novo70))
|
|||||||
# [ʊ] 'ʊ'(1) --> can be ʏ (uh)??
|
# [ʊ] 'ʊ'(1) --> can be ʏ (uh)??
|
||||||
# [χ] --> can be x??
|
# [χ] --> can be x??
|
||||||
|
|
||||||
def search_phone_ipa(x, phone_list):
|
#def search_phone_ipa(x, phone_list):
|
||||||
x_in_item = []
|
# x_in_item = []
|
||||||
for ipa in phone_list:
|
# for ipa in phone_list:
|
||||||
ipa_original = ipa
|
# ipa_original = ipa
|
||||||
ipa = ipa.replace(':', 'ː')
|
# ipa = ipa.replace(':', 'ː')
|
||||||
ipa = convert_phone_set.split_ipa(ipa)
|
# ipa = convert_phone_set.split_ipa(ipa)
|
||||||
if x in ipa and not x+':' in ipa:
|
# if x in ipa and not x+':' in ipa:
|
||||||
x_in_item.append(ipa_original)
|
# x_in_item.append(ipa_original)
|
||||||
return x_in_item
|
# return x_in_item
|
||||||
#search_phone_ipa('ø', transcription_ipa)
|
#search_phone_ipa('ø', transcription_ipa)
|
||||||
|
|
||||||
|
|
||||||
## ===== load all transcriptions (df) =====
|
## ===== load all transcriptions (df) =====
|
||||||
df = stimmen_functions.load_transcriptions()
|
#df = stimmen_functions.load_transcriptions()
|
||||||
word_list = [i for i in list(set(df['word'])) if not pd.isnull(i)]
|
word_list = [i for i in list(set(df['word'])) if not pd.isnull(i)]
|
||||||
word_list = sorted(word_list)
|
word_list = sorted(word_list)
|
||||||
|
|
||||||
|
|
||||||
## check frequency of each pronunciation variants
|
## check frequency of each pronunciation variants
|
||||||
cols = ['word', 'ipa', 'frequency']
|
#cols = ['word', 'ipa', 'frequency']
|
||||||
df_samples = pd.DataFrame(index=[], columns=cols)
|
#df_samples = pd.DataFrame(index=[], columns=cols)
|
||||||
for ipa in all_in_novo70:
|
#for ipa in all_in_novo70:
|
||||||
ipa = ipa.replace('ː', ':')
|
# ipa = ipa.replace('ː', ':')
|
||||||
samples = df[df['ipa'] == ipa]
|
# samples = df[df['ipa'] == ipa]
|
||||||
word = list(set(samples['word']))[0]
|
# word = list(set(samples['word']))[0]
|
||||||
samples_Series = pd.Series([word, ipa, len(samples)], index=df_samples.columns)
|
# samples_Series = pd.Series([word, ipa, len(samples)], index=df_samples.columns)
|
||||||
df_samples = df_samples.append(samples_Series, ignore_index=True)
|
# df_samples = df_samples.append(samples_Series, ignore_index=True)
|
||||||
|
|
||||||
# each word
|
# each word
|
||||||
df_per_word = pd.DataFrame(index=[], columns=df_samples.keys())
|
#df_per_word = pd.DataFrame(index=[], columns=df_samples.keys())
|
||||||
|
|
||||||
for word in word_list:
|
#for word in word_list:
|
||||||
df_samples_ = df_samples[df_samples['word']==word]
|
word = word_list[2]
|
||||||
df_samples_ = df_samples_[df_samples_['frequency']>2]
|
df_ = df[df['word']==word]
|
||||||
df_per_word = df_per_word.append(df_samples_, ignore_index=True)
|
np.unique(list(df_['ipa']))
|
||||||
|
#df_samples_ = df_samples_[df_samples_['frequency']>2]
|
||||||
|
#df_per_word = df_per_word.append(df_samples_, ignore_index=True)
|
||||||
#df_per_word.to_excel(os.path.join(default.stimmen_dir, 'pronunciation_variants_novo70.xlsx'), encoding="utf-8")
|
#df_per_word.to_excel(os.path.join(default.stimmen_dir, 'pronunciation_variants_novo70.xlsx'), encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
@ -352,9 +352,6 @@ def fix_lexicon(lexicon_file):
|
|||||||
return
|
return
|
||||||
|
|
||||||
|
|
||||||
#def add_sp_to_lexicon(lexicon_file):
|
|
||||||
|
|
||||||
|
|
||||||
def word2htk(word):
|
def word2htk(word):
|
||||||
return ''.join([fame_asr.translation_key_word2htk.get(i, i) for i in word])
|
return ''.join([fame_asr.translation_key_word2htk.get(i, i) for i in word])
|
||||||
|
|
||||||
|
@ -174,6 +174,8 @@ def forced_alignment(wav_file, word, pronunciation_ipa):
|
|||||||
p.add_argument("--user", default='martijn.wieling')
|
p.add_argument("--user", default='martijn.wieling')
|
||||||
p.add_argument("--password", default='xxxxxx')
|
p.add_argument("--password", default='xxxxxx')
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
args = p.parse_args()
|
args = p.parse_args()
|
||||||
|
|
||||||
rec = session.Recognizer(grammar_version="1.0", lang="nl", snodeid=101, user=args.user, password=args.password, keepopen=True) # , modeldir=modeldir)
|
rec = session.Recognizer(grammar_version="1.0", lang="nl", snodeid=101, user=args.user, password=args.password, keepopen=True) # , modeldir=modeldir)
|
||||||
@ -194,6 +196,24 @@ def result2pronunciation(result, word):
|
|||||||
return pronunciation_ipa, pronunciation_novo70, llh
|
return pronunciation_ipa, pronunciation_novo70, llh
|
||||||
|
|
||||||
|
|
||||||
|
def phones_not_in_novo70(ipa):
|
||||||
|
""" extract phones which is not in novo70 phoneset. """
|
||||||
|
|
||||||
|
phoneset_ipa, _, _, _ = load_novo70_phoneset()
|
||||||
|
|
||||||
|
# As per Nederlandse phoneset_aki.xlsx recieved from David
|
||||||
|
# [ɔː] oh / ohr
|
||||||
|
# [ɪː] ih / ihr
|
||||||
|
# [iː] iy
|
||||||
|
# [œː] uh
|
||||||
|
# [ɛː] eh
|
||||||
|
# [w] wv in IPA written as ʋ.
|
||||||
|
david_suggestion = ['ɔː', 'ɪː', 'iː', 'œː', 'ɛː', 'w']
|
||||||
|
|
||||||
|
return [phone for phone in split_ipa(ipa)
|
||||||
|
if not phone in phoneset_ipa and not phone in david_suggestion]
|
||||||
|
|
||||||
|
|
||||||
if __name__ == 'main':
|
if __name__ == 'main':
|
||||||
pronunciation_ipa = ['rø:s', 'mɑn', 'mɑntsjə']
|
pronunciation_ipa = ['rø:s', 'mɑn', 'mɑntsjə']
|
||||||
#grammar = make_grammar('reus', pronunciation_ipa)
|
#grammar = make_grammar('reus', pronunciation_ipa)
|
||||||
|
@ -7,6 +7,7 @@ import pandas as pd
|
|||||||
import convert_xsampa2ipa
|
import convert_xsampa2ipa
|
||||||
import defaultfiles as default
|
import defaultfiles as default
|
||||||
import fame_functions
|
import fame_functions
|
||||||
|
import novoapi_functions
|
||||||
|
|
||||||
|
|
||||||
def _load_transcriptions():
|
def _load_transcriptions():
|
||||||
@ -67,6 +68,19 @@ def load_transcriptions_clean(clean_wav_dir):
|
|||||||
return df_clean
|
return df_clean
|
||||||
|
|
||||||
|
|
||||||
|
def load_transcriptions_novo70(clean_wav_dir):
|
||||||
|
""" extract rows of which ipa is written in novo70 phonset. """
|
||||||
|
df = load_transcriptions_clean(clean_wav_dir)
|
||||||
|
|
||||||
|
df_novo70 = pd.DataFrame(index=[], columns=list(df.keys()))
|
||||||
|
for index, row in df.iterrows():
|
||||||
|
not_in_novo70 = novoapi_functions.phones_not_in_novo70(row['ipa'])
|
||||||
|
if len(not_in_novo70) == 0:
|
||||||
|
df_novo70 = df_novo70.append(row, ignore_index=True)
|
||||||
|
|
||||||
|
return df_novo70
|
||||||
|
|
||||||
|
|
||||||
def add_row_htk(df):
|
def add_row_htk(df):
|
||||||
""" df['htk'] is made from df['ipa'] and added. """
|
""" df['htk'] is made from df['ipa'] and added. """
|
||||||
htk = []
|
htk = []
|
||||||
|
Loading…
Reference in New Issue
Block a user