2018-12-26 23:49:28 +01:00
|
|
|
|
import os
|
|
|
|
|
os.chdir(r'C:\Users\Aki\source\repos\acoustic_model\acoustic_model')
|
|
|
|
|
|
|
|
|
|
import sys
|
|
|
|
|
import csv
|
|
|
|
|
#import subprocess
|
|
|
|
|
#from collections import Counter
|
|
|
|
|
#import re
|
|
|
|
|
|
|
|
|
|
import numpy as np
|
|
|
|
|
import pandas as pd
|
|
|
|
|
#import matplotlib.pyplot as plt
|
|
|
|
|
#from sklearn.metrics import confusion_matrix
|
|
|
|
|
|
|
|
|
|
import acoustic_model_functions as am_func
|
|
|
|
|
import convert_xsampa2ipa
|
|
|
|
|
import defaultfiles as default
|
|
|
|
|
|
2018-12-31 13:04:33 +01:00
|
|
|
|
from forced_alignment import pyhtk, convert_phone_set
|
2018-12-26 23:49:28 +01:00
|
|
|
|
|
|
|
|
|
import novoapi
|
|
|
|
|
|
|
|
|
|
|
2018-12-30 23:47:55 +01:00
|
|
|
|
## ======================= novo phoneset ======================
|
2018-12-26 23:49:28 +01:00
|
|
|
|
translation_key = dict()
|
|
|
|
|
|
2018-12-30 23:30:33 +01:00
|
|
|
|
#phonelist_novo70_ = pd.ExcelFile(default.phonelist_novo70_xlsx)
|
|
|
|
|
#df = pd.read_excel(phonelist_novo70_, 'list')
|
|
|
|
|
## *_simple includes columns which has only one phone in.
|
|
|
|
|
#for ipa, novo70 in zip(df['IPA_simple'], df['novo70_simple']):
|
|
|
|
|
# if not pd.isnull(ipa):
|
|
|
|
|
# print('{0}:{1}'.format(ipa, novo70))
|
|
|
|
|
# translation_key[ipa] = novo70
|
|
|
|
|
#phonelist_novo70 = np.unique(list(df['novo70_simple']))
|
|
|
|
|
|
|
|
|
|
phoneset_ipa = []
|
|
|
|
|
phoneset_novo70 = []
|
2018-12-31 13:04:33 +01:00
|
|
|
|
with open(default.novo70_phoneset, "rt", encoding="utf-8") as fin:
|
2018-12-30 23:30:33 +01:00
|
|
|
|
lines = fin.read()
|
|
|
|
|
lines = lines.split('\n')
|
|
|
|
|
for line in lines:
|
|
|
|
|
words = line.split('\t')
|
|
|
|
|
if len(words) > 1:
|
|
|
|
|
novo70 = words[0]
|
|
|
|
|
ipa = words[1]
|
|
|
|
|
phoneset_ipa.append(ipa)
|
|
|
|
|
phoneset_novo70.append(novo70)
|
|
|
|
|
translation_key[ipa] = novo70
|
|
|
|
|
phoneset_ipa = np.unique(phoneset_ipa)
|
|
|
|
|
phoneset_novo70 = np.unique(phoneset_novo70)
|
2018-12-26 23:49:28 +01:00
|
|
|
|
|
2018-12-31 13:04:33 +01:00
|
|
|
|
# As per Nederlandse phoneset_aki.xlsx recieved from David
|
|
|
|
|
# [ɔː] oh / ohr
|
|
|
|
|
# [ɪː] ih / ihr
|
|
|
|
|
# [iː] iy
|
|
|
|
|
# [œː] uh
|
|
|
|
|
# [ɛː] eh
|
2019-01-07 11:50:24 +01:00
|
|
|
|
# [w] wv in IPA written as ʋ.
|
|
|
|
|
david_suggestion = ['ɔː', 'ɪː', 'iː', 'œː', 'ɛː', 'w']
|
2018-12-31 13:04:33 +01:00
|
|
|
|
|
2018-12-30 23:47:55 +01:00
|
|
|
|
|
2019-01-07 11:50:24 +01:00
|
|
|
|
## ======================= extract words which is written only with novo70 ======================
|
2018-12-30 23:47:55 +01:00
|
|
|
|
mapping = convert_xsampa2ipa.load_converter('xsampa', 'ipa', default.ipa_xsampa_converter_dir)
|
|
|
|
|
|
|
|
|
|
stimmen_transcription_ = pd.ExcelFile(default.stimmen_transcription_xlsx)
|
2019-01-07 11:50:24 +01:00
|
|
|
|
df = pd.read_excel(stimmen_transcription_, 'frequency')
|
2018-12-30 23:47:55 +01:00
|
|
|
|
#for xsampa, ipa in zip(df['X-SAMPA'], df['IPA']):
|
|
|
|
|
# ipa_converted = convert_xsampa2ipa.xsampa2ipa(mapping, xsampa)
|
|
|
|
|
# if not ipa_converted == ipa:
|
2018-12-31 13:04:33 +01:00
|
|
|
|
# print('{0}: {1} - {2}'.format(xsampa, ipa_converted, ipa))
|
|
|
|
|
transcription_ipa = list(df['IPA'])
|
|
|
|
|
|
|
|
|
|
# transcription mistake?
|
2019-01-07 11:50:24 +01:00
|
|
|
|
transcription_ipa = [ipa.replace(';', 'ː') for ipa in transcription_ipa if not ipa=='pypɪl' and not pd.isnull(ipa)]
|
2018-12-31 13:04:33 +01:00
|
|
|
|
transcription_ipa = [ipa.replace('ˑ', '') for ipa in transcription_ipa] # only one case.
|
|
|
|
|
|
|
|
|
|
not_in_novo70 = []
|
2019-01-07 11:50:24 +01:00
|
|
|
|
all_in_novo70 = []
|
2018-12-31 13:04:33 +01:00
|
|
|
|
for ipa in transcription_ipa:
|
2019-01-07 11:50:24 +01:00
|
|
|
|
ipa = ipa.replace(':', 'ː')
|
2018-12-31 13:04:33 +01:00
|
|
|
|
ipa = convert_phone_set.split_ipa(ipa)
|
|
|
|
|
|
|
|
|
|
not_in_novo70_ = [phone for phone in ipa
|
|
|
|
|
if not phone in phoneset_ipa and not phone in david_suggestion]
|
|
|
|
|
not_in_novo70_ = [phone.replace('sp', '') for phone in not_in_novo70_]
|
|
|
|
|
not_in_novo70_ = [phone.replace(':', '') for phone in not_in_novo70_]
|
|
|
|
|
not_in_novo70_ = [phone.replace('ː', '') for phone in not_in_novo70_]
|
|
|
|
|
|
2019-01-07 11:50:24 +01:00
|
|
|
|
if len(not_in_novo70_) == 0:
|
|
|
|
|
all_in_novo70.append(''.join(ipa))
|
|
|
|
|
|
2018-12-31 13:04:33 +01:00
|
|
|
|
#translation_key.get(phone, phone)
|
|
|
|
|
not_in_novo70.extend(not_in_novo70_)
|
|
|
|
|
not_in_novo70_list = list(set(not_in_novo70))
|
|
|
|
|
|
2019-01-07 11:50:24 +01:00
|
|
|
|
## check which phone is used in stimmen but not in novo70
|
|
|
|
|
# 'ʀ', 'ʁ',
|
|
|
|
|
# 'ɒ', 'ɐ',
|
|
|
|
|
# 'o', 'a' (o:, a:?)
|
2018-12-31 13:04:33 +01:00
|
|
|
|
# [e] 'nyːver mɑntsjə' (1)
|
2019-01-07 11:50:24 +01:00
|
|
|
|
# [ɾ] 'ɪːɾ'(1)
|
|
|
|
|
# [ɹ] 'iːjəɹ' (1), 'ɪ:ɹ' (1)
|
|
|
|
|
# [ø] 'gʀøtəpi:r'(1), 'grøtəpi:r'(1)
|
|
|
|
|
# [æ] 'røːzəʀæt'(2), 'røːzəræt'(1)
|
|
|
|
|
# [ʊ] 'ʊ'(1) --> can be ʏ (uh)??
|
|
|
|
|
# [χ] --> can be x??
|
2018-12-31 13:04:33 +01:00
|
|
|
|
|
2019-01-07 11:50:24 +01:00
|
|
|
|
def search_phone_ipa(x, phone_list):
|
|
|
|
|
x_in_item = []
|
|
|
|
|
for ipa in phone_list:
|
|
|
|
|
ipa_original = ipa
|
|
|
|
|
ipa = ipa.replace(':', 'ː')
|
|
|
|
|
ipa = convert_phone_set.split_ipa(ipa)
|
|
|
|
|
if x in ipa and not x+':' in ipa:
|
|
|
|
|
x_in_item.append(ipa_original)
|
|
|
|
|
return x_in_item
|
|
|
|
|
#search_phone_ipa('ø', transcription_ipa)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
df = pd.read_excel(stimmen_transcription_, 'original')
|
|
|
|
|
|
|
|
|
|
ipas = []
|
|
|
|
|
famehtks = []
|
|
|
|
|
for xsampa in df['Self Xsampa']:
|
|
|
|
|
if not isinstance(xsampa, float): # 'NaN'
|
|
|
|
|
# typo?
|
|
|
|
|
xsampa = xsampa.replace('r2:z@rA:\\t', 'r2:z@rA:t')
|
|
|
|
|
xsampa = xsampa.replace(';', ':')
|
|
|
|
|
|
|
|
|
|
ipa = convert_xsampa2ipa.xsampa2ipa(mapping, xsampa)
|
|
|
|
|
ipa = ipa.replace('ː', ':')
|
|
|
|
|
ipa = ipa.replace(' ', '')
|
|
|
|
|
ipas.append(ipa)
|
|
|
|
|
else:
|
|
|
|
|
ipas.append('')
|
|
|
|
|
|
|
|
|
|
# extract interesting cols.
|
|
|
|
|
df = pd.DataFrame({'filename': df['Filename'],
|
|
|
|
|
'word': df['Word'],
|
|
|
|
|
'xsampa': df['Self Xsampa'],
|
|
|
|
|
'ipa': pd.Series(ipas)})
|
|
|
|
|
|
|
|
|
|
# find options which all phones are in novo70.
|
|
|
|
|
#word_list = list(set(df['word']))
|
|
|
|
|
#word_list = [word for word in word_list if not pd.isnull(word)]
|
|
|
|
|
#word = word_list[1]
|
|
|
|
|
|
|
|
|
|
## pronunciation variants of 'word'
|
|
|
|
|
#df_ = df[df['word'] == word]['xsampa']
|
|
|
|
|
##pronunciation_variant = list(set(df_))
|
|
|
|
|
|
|
|
|
|
cols = ['word', 'ipa', 'frequency']
|
|
|
|
|
df_samples = pd.DataFrame(index=[], columns=cols)
|
|
|
|
|
for ipa in all_in_novo70:
|
|
|
|
|
ipa = ipa.replace('ː', ':')
|
|
|
|
|
samples = df[df['ipa'] == ipa]
|
|
|
|
|
word = list(set(samples['word']))[0]
|
|
|
|
|
samples_Series = pd.Series([word, ipa, len(samples)], index=df_samples.columns)
|
|
|
|
|
df_samples = df_samples.append(samples_Series, ignore_index=True)
|