From 41d4fa5ff9338bae38240e4ce8b909eb4dfffc20 Mon Sep 17 00:00:00 2001 From: yemaozi88 <428968@gmail.com> Date: Tue, 5 Mar 2019 00:11:38 +0100 Subject: [PATCH] sp is added to the model. --- .vs/acoustic_model/v15/.suo | Bin 102400 -> 100864 bytes acoustic_model/fame_functions.py | 13 ++-- acoustic_model/fame_hmm.py | 106 +++++++++++++------------------ 3 files changed, 52 insertions(+), 67 deletions(-) diff --git a/.vs/acoustic_model/v15/.suo b/.vs/acoustic_model/v15/.suo index 712c255714ef6ffda4e8bb7262540e56cc859336..96899cb98ddd6e7708d34b29a0a50a9133e53c15 100644 GIT binary patch delta 6346 zcmd^D4OEm>8vfpo8RTa`K)?ZE91s){5d;)aUWG1l~BHmU8pR;%kVt+OrS)*h{O&06)SrR;NO1|(zmZ0&aToZWlQGtd41 zdEfWG@BPNG%h|BSxhaNSUFWLyk|cLYk`%z*&dyGdM1bcjxdUX^oWSz0+E1S$J*$$} z(sS&(yq=0!wbLq(R`h8P7riCPg^{A)S}k&zptB#?*OABM(jY+TDBnT*VC2<601yNW z0D^&0fY?PTFc63b!~)SkKHvu|0=$7?z+*rkz#qW0Ydc3EqXN=^p}-j60U#6*N1Fv6 z4fF>hfH)u)a08-%WMBxO15$uMAO?s8FuXGXoDTE_Mgk84nSf~9CAOA50C@o0+WF$z*Jy5Ao}Dvc({W_T@vz-0tLWqK+~K%EN>z*S%B!U*umVv zqE4hLM|nk$`jw9SDsT<3wny93$Q$J{*QVq>V};tw?pM3tPW0$fyM2w$P7j;#w`_(SmyV5VvZy?a6_G09P+C}>Yuu!|mH-LrOMP8Vf(E2zb8&Pq;+C^QR zqx1c04@3PtK&V~x6X|}n&q13|dwFwizu~UPv;acQ_U`uIo@y2bE|O1k?trzz(nX=r z!6jg4NB(>IMBNNPSoKU`0U!)^K6oy$5SRuCqZ9LqB#cjRg@c!XmjacJ@^VQUZtL(k zDykios~s!`inIo0gQL#q=Bs!LH=n895<|r5&!E78t-!N@sN3$~yDTu`%O zcerWE@GG zruP8>o6+@S+kku0T93Mo&>_=?Cc27)Y(+(4bM8Q@ZO-i<{411DX-hBk$iImEtLUYE z4H*mYI`9VYCh!)(E{6swk0Nuox#(}k=t>%Eu8Mw=jDevnRC|^_Ww*57ItN2&r-_YU?WJLp*f{T4RV})T{flH8tT44IjH*h-aMZFDstk6>N2Q1p zg%5ldD7J`9qDPG4X^|=OsMORUvl3VZh(oRhuW{rJWWGB>Rv0TX6v9lzs>+dQTg`6y z4P_>6sEe>9A#B$8#|YoQJ4ml}l`!RBk}q2J@|iKF+2LnVU+;$Ly2p zDKoAAK85f&qOYc|c*t<1;tf0}K6}AMXu7XPuQr-^E1V60BHBmsw;Q2Vm@S+mMnHg<2qdgAsXalMZG zh8gPnf8cvX|3rBD!M&~v9uV%K+k{8LFBUstdhl0RA#LirDebrsW6>wwN0KICvO;hy z_$6>V>JPzx3&nm0K8*4vKL|sXr1!ufEM?kJ(;bWC?69EBkt!SN$bR7;1l3}uJ$PX5%?37FS<$6 zCG?wyFm7{f;6I`KI^;f(c=JTkn;xOmbN%6w(ER}7iSSHU!M{U!Q=lZ(DJAJN_(jx@ z3dZ@c^poHjV0Re7Cz3gJ>=80emYb(~nZ0TfX;kU^Z|*H?3Vo$*&)YdZL&smlOx-)r z#m+)tFXKF6o$%czUr8E_@{j=Yn$@+kap^G^5kzwVp*Q1kh54q{r4O7_t4tIWiAZ%o zD@i3-y&k4^1G}<$8T_#J%CNPPjEO(O=!g4CQWbbAxD@OT-L1l?r=hn!C~pUE1-}h_ zy^2{{prYZ41*;&JE-*VNCoGNqQN5 z4oBd);6X6`cJzG&{3gobQIgc?1kVe;fO2dMelTvrNIaIn-J)r_5ylRNXOqlA}eQ5ewcl03ju5(=gpO z)NKcZuXzrf2-E_*0kNzJya(6|JP-U55cT`O`vJChQxqL*&P`LkhZ5ZVUrc*tW3AOE z#+Fvs#6-o^WEPbzD=uas4H`E4jtei;k{3Hx9pN=9Dk0w9xDtcIr!n1ng9neizDi@i09m2P%3Tc0uz%c<75 z#m01vU=h6t-IYx%%dc*$Ar;NcJk&sJjRK(!m4?RH*K@Yl`k4e5f)D`8cwED z3gdU*v1jji+WK>yl00~glCn%~GQX^$=lEu#CEO52k*xgN2+QFRS|}^wfTTc+HJTnG zZ!cVMdy23IBOkyuF?3QnZd`Kyw6?Qns22n0yVaD$3U4{{^M16!Up%DlPZ>w~?|rF> zH>sg)Y&V(jP*axggP8Rn+>QhA%e~!&R%cR}t|*L`tb452dKEtR?n^`XPA3}8uMVg8 zEkpa!IcK4c)*yJQe^np8_w*s2sl6znl~;OG!}m2%LL7Q%;7v(1`uke2Knu%=&Hi?@ z5c*>>g>eY{1scI$=}Uuc7eJ1T?k-I9KGZG72&*R);cI~+JjgD>#};*7U3K${%kqonF0Nv#F+oGi z@qtny72$UQKI|)ClVf4J6)@s*d@g&~zQx5%#XQ%YV%e2T&Q6(93`+~2*?0VF`;V=X zjzwi(T9aCGK+E*0ewI7&a#+(K8#I4%UM&f=kh_D%SpidJ6mkEaqymB+Wtlb?^+Zb&UPb2@xRqu;QsP|ZG$uUshemMfa!|pRtm4ngp7;26S>C7H@rL*1-T@RM7tQ5{c#2>l->JRY z--t+mDf7$``aIyok=QS?c~d;8d83AUYf@4L&9Kl|I#2xca?<|aot_njt>D@;a^@W= zv`>D<79cDL5JAKbE~b_2cA+nHtLFOYw1|i4DU)XM3n`SqPZi?bQWx%viR4dw-@Qy6N( zr%K6-*A$Vqhma!e`O(}kha#v|oOC`{%%KcgdGDlr$6N#@+hz+Z5qTOP4dgjLTr4^B=rmD5k8R-tS|cvTN*+HSAtSVewnAB5`uAn^ zqg5841vH1cbP5SXivdBra5`o4h*BEGD;H9>T|xv%C`n1<_?LmmgqprylA_Fc8umG=5_3bbjKHKwRwj#)6rCbncu zF29;Z&RskEsn&+DX4{7~54Qgwu&+0NR%5Xl`~QX}EjKq%773eM%5|IYNWl$la7vbJ b;UVCM;X#t}pm>LG!L7tCN>M%YBw7Cs2B(ej delta 5943 zcmd5=3sjWXwVpi;Pk{oC78P+oG|0=D0S2T-2Sr2$10))fn8;HR2_gc;i15cm&4U=6 zljf+2iW9GS*4X*mj!6_p^Kjj^*CuJ2+N>lO?{c-ts%c1SR-;Knzx~gToh0qL*1Otu z4}NR!bN+MoIcLAl9+gKzl!1_bYm(8Rw-|#V3aknEYGh=DM}vTvA5}~;h7WwlWnS^d zTT~-DjJ=dCJ~Q4!En;KPrl`G`vu_&DHWto;d)83Kqf++#A z0UIy{V1d``^fsMN)TvXaGeB!~J`Qv~5CSX!{(|+zposv>S^_!;CN5nobkxZ(HJr`gi8HA`QwS^6w2-P=2AWtPir43_Zi`H-WHZO-heV_$&9l&Pe1>D$t-_!eS zJ^$#)$Xz=B1ZX#~8R*r=egNuyq@C1V=JP(-Ra6k#D(R=bH+CAzYTVweep z_(fJqFfSVspJmPU&&!`_^)t;g+Y@>6U1Tt90+`c@6g>1sK z6tjAH+?k*l_BxJ|(F?0*C!`u($)D46;!^S#6Ed;rbA0A@H?Re`2e=P`BP@ye<0Lw} zfX4t<5w`(+%sqes7Rv2E^pClqPXbQ?PXo^Z{uv`9KLr2NofT8fUNA?2mjXVD3yBF) zuLcafCSo7^gQw3)QnAu4F_V`D9XvZNesAn2uVigI((o&BvU{8ud}y9~y2+I?Kb+W! zID}X^qCMr8Vt%Ymv`@PsoYr8J1BAZ>Z^3OJ7MEf-kzG@qO?(!24{1t^+=fYs9M@Nnjgwq)R3F7_(%p29ghY0DDMs-ftDe}y*z-h)0kvqfao^2FLWU`uE{62YJxfnEN zGS-gpLC7Gfn6hR6DvDN}rM_Tyyipp9$tj%8K3B7OMtOAS)Ru;i&YyVaFCT7+uy7u_ zvfC%fP7OhDTV$^7;`%#Zc6HvKboy7}f2#S|bmsoqr;g1W_wk~gW`hG60> zEee-si1xLx4$F(uzX$hYht2Ax+eVD^p15?!NN6^`ps@<_R7dsC!rd*}?d|gD{gkMd zw;dsOtI?HHWMVI8slgAi40!eo;a*)CvJAcZ4dQanQq%W&hUOi)`o3NINaS|R+Kvu4 zm-0#&hYzPQo~~Y0B1yItQh{2~{u;Tr7)8Zk`WU-CBd5E#rg=j{uc+8HQyxjDsOY+N zEu9T*?JHXw*R?ck5Xvr}v7sH)GlH@+?Bb!_MPuq4R@HPgx35~$+#udKo?*6TSp8CZ zu-&v}8yecm+uZjWJ>4Y~v$NuK(IiXlslCHp2VaQWm;skJ$GwAhU@?sH0348w6P=*; zY{r>}x5lV3+lI-t&nIFBo6XsI8Fq(Xv@$AMIQJ{jtWtiorUEr}!nr#swiypcOIGW~ z8F=>#tOc8L+%J+SJU(9TPp3rraR`Np*j;mjYgvv^UQ%FWi)xuTM4n?#5^wxwq8Q#6 zCJqngMB4N7MlZDKidw|Vk7C5am*OVh4FOlu3;3=|%KkM#iC6kW4<_*=iEjUE_l7 z4o8MnEZ*hNj3&xl{9S?MG9BPQW{!69F-pxN@pN{W!y)d<-KZt5hCSG!DHLvb>(l2Sxt z(OIX}nrRJao4f5fjtpDA-(u`mR_o?&n=N-#oqV0YwU$t!EKr2~;e+bI{uLyXN@<#y zw){4n#%Fo|h17Y6i{Cm$=O_H^l!+j<0$bLH9gz&qGkVs@3X)1LP^WNG48E>gT7^t}WIhv{=W< z5M^K8e%T-Il&xO+ub-Q~6@Q`P!ge#OJow51MZYLfO9uKvqxhNO#&m&t?#U@e&!M;I z>%#)WF@`RK6KBRh4SU4N>8)PHNY4|w1l_br&MBsNQF?fpID0sikWHSS>wca@{qqST z!IPr;JWldF6=aNhTNl8BszqUUaoRf6FU=^T*I|n+L#tsU(1u=Xf}7&*O^EI8eA zIj@-FaJ(gl=iqprv+z;pZxm7KN$@5_(D@+OyFQ`(RuNsE+2YH0@AbUxqi=~7%6p1y zC?~Q7MOcHO1@*xS+;@7sbS|JqXyfFg9ddRBMF`JPvv_MLV@wL(MLZ&?Mqd`TFZ$&$ zAtIi=kSd=3-~{cJ2OQLd1P~<_U%pN@Cea}=>(f5(%Bi%M=rRA-pC2D1e*I}K@%k$F zS>q3g@~?=oMH}X2-gpy|57JL|eqLRzEx$g~Vav_V%(3MGk4&67N%c#-D#TK`suRDVrAb93tY0ITW#Zdu~BZbek(MTYrrBDY+>mNv++?KEBtTf=0`0va#P z^^_p{>!?*W7g3z?QO(W0NC62%x5{%%Xn}myN@kp-Lhjr~Q>3+?N@dkTRAW2j`Vz|J z98n}LSDW!86@@Tzpg6gEitGuc2j$UOWd3{hJvJs#k9QYLzUIrx|5<|FUtK;OwdoDm z{V<>DA+hvX3r-v)?<}Do%Jw?Eyq+?Oms#bo%f5o?;&;hxaa_6|!|Lb&E4yk+NqAg~&(B$ih#44y!v- zI_>29Urqk+G89y$8A`M6)#&cWWmF^|*#<|^jK8#uish*V)FuZ?DM6fB5h905ap$2W z)GXV}$wi%h&w#p%d0pF;x+g8bX~zFE%B;W2{X_fk{}y;@Wkm(@kISF5hbyT^vybev z!5FVs;;`qNaIB#X^x~+O2b;+Z&D;!Mc-(K>p&K#pLLI8Ogqz7ihvX;4v|83zQ4!X_ z&GJOviDbChL=KD&zcO~*K2g4Y3;g3Qf9(9RiVFO?Lg^RgEx&`_B$-k}yZnK7q>LuY zW7X8i4&(5@@r~6exZzR>Qv1s9zSx=<T&oD?a>B+KtZo`K)A`F~an iZijnl%4}Ya5Ztj5Ddv9iq4amkwzWv+Q2CqJwEVBUko>j) diff --git a/acoustic_model/fame_functions.py b/acoustic_model/fame_functions.py index 9ca7e0d..9f4e127 100644 --- a/acoustic_model/fame_functions.py +++ b/acoustic_model/fame_functions.py @@ -378,17 +378,22 @@ def ipa2htk(ipa): return ''.join(htk_splitted) -def performance_on_stimmen(stimmen_dir, hmmdefs): - #hmmdefs = r'c:\OneDrive\Research\rug\experiments\acoustic_model\fame\htk\model_\hmm1\iter20\hmmdefs' - #stimmen_dir = r'c:\OneDrive\Research\rug\experiments\acoustic_model\fame\htk\stimmen' +def performance_on_stimmen(config_dir, stimmen_dir, hmmdefs): lattice_file = os.path.join(stimmen_dir, 'word_lattice.ltc') hvite_scp = os.path.join(stimmen_dir, 'hvite.scp') #fh.make_filelist(os.path.join(stimmen_dir, 'mfc'), hvite_scp, file_type='mfc') hresult_scp = os.path.join(stimmen_dir, 'hresult.scp') #fh.make_filelist(os.path.join(stimmen_dir, 'mfc'), hresult_scp, file_type='rec') lexicon_file = os.path.join(stimmen_dir, 'lexicon_recognition.dic') - chtk = pyhtk.HTK(config_dir, fame_asr.phoneset_htk, lexicon_file) + # get feature_size from hmmdefs. + with open(hmmdefs) as f: + line = f.readline() + line = f.readline().strip() + feature_size = int(line.split(' ')[2]) + + chtk = pyhtk.HTK(config_dir, fame_asr.phoneset_htk, lexicon_file, feature_size) + result = chtk.recognition( lattice_file, hmmdefs, diff --git a/acoustic_model/fame_hmm.py b/acoustic_model/fame_hmm.py index 4e2f430..5555919 100644 --- a/acoustic_model/fame_hmm.py +++ b/acoustic_model/fame_hmm.py @@ -26,7 +26,7 @@ make_mlf = 0 extract_features = 0 flat_start = 0 train_model_without_sp = 0 -add_sp = 0 +add_sp = 1 train_model_with_sp = 0 train_model_with_sp_align_mlf = 0 train_triphone = 0 @@ -35,6 +35,9 @@ train_triphone = 0 # pre-defined values. dataset_list = ['devel', 'test', 'train'] +feature_size = 39 +improvement_threshold = 0.5 + hmmdefs_name = 'hmmdefs' proto_name = 'proto' @@ -47,7 +50,9 @@ sil_hed = os.path.join(config_dir, 'sil.hed') prototype = os.path.join(config_dir, proto_name) model_dir = os.path.join(default.htk_dir, 'model') - +model0_dir = os.path.join(model_dir, 'hmm0') +model1_dir = os.path.join(model_dir, 'hmm1') +model1sp_dir = os.path.join(model_dir, 'hmm1sp') # directories / files to be made. lexicon_dir = os.path.join(default.htk_dir, 'lexicon') @@ -55,9 +60,6 @@ lexicon_htk_asr = os.path.join(lexicon_dir, 'lex.htk_asr') lexicon_htk_oov = os.path.join(lexicon_dir, 'lex.htk_oov') lexicon_htk = os.path.join(lexicon_dir, 'lex.htk') - -#model1_dir = os.path.join(model_dir, 'hmm1') - feature_dir = os.path.join(default.htk_dir, 'mfc') fh.make_new_directory(feature_dir, existing_dir='leave') tmp_dir = os.path.join(default.htk_dir, 'tmp') @@ -65,11 +67,17 @@ fh.make_new_directory(tmp_dir, existing_dir='leave') label_dir = os.path.join(default.htk_dir, 'label') fh.make_new_directory(label_dir, existing_dir='leave') + + ## training hcompv_scp_train = os.path.join(tmp_dir, 'train.scp') mlf_file_train = os.path.join(label_dir, 'train_phone.mlf') mlf_file_train_aligned = os.path.join(label_dir, 'train_phone_aligned.mlf') +## testing +htk_stimmen_dir = os.path.join(default.htk_dir, 'stimmen') + + ## train without sp niter_max = 10 @@ -100,7 +108,7 @@ if make_lexicon: ## intialize the instance for HTK. -chtk = pyhtk.HTK(config_dir, fame_asr.phoneset_htk, lexicon_htk) +chtk = pyhtk.HTK(config_dir, fame_asr.phoneset_htk, lexicon_htk, feature_size) ## ======================= make label files ======================= @@ -223,11 +231,14 @@ if extract_features: if flat_start: timer_start = time.time() print('==== flat start ====') - feature_size = 39 - model0_dir = os.path.join(model_dir, 'hmm0') fh.make_new_directory(model0_dir, existing_dir='leave') - chtk.flat_start(hcompv_scp_train, model0_dir, feature_size) + chtk.flat_start(hcompv_scp_train, model0_dir) + + # create macros. + vFloors = os.path.join(model0_dir, 'vFloors') + if os.path.exists(vFloors): + chtk.create_macros(vFloors) # allocate mean & variance to all phones in the phone list print('>>> allocating mean & variance to all phones in the phone list...') @@ -241,69 +252,38 @@ if flat_start: ## ======================= train model without short pause ======================= if train_model_without_sp: - fh.make_new_directory(model1_dir) - print('==== train model without sp ====') - if not os.path.exists(os.path.join(model1_dir, 'iter0')): - shutil.copytree(model0_dir, os.path.join(model1_dir, 'iter0')) - for niter in range(1, niter_max): - timer_start = time.time() - hmm_n = 'iter' + str(niter) - hmm_n_pre = 'iter' + str(niter-1) - modeln_dir = os.path.join(model1_dir, hmm_n) - modeln_dir_pre = os.path.join(model1_dir, hmm_n_pre) - - # re-estimation - fh.make_new_directory(modeln_dir) - pyhtk.re_estimation( - config_train, - os.path.join(modeln_dir_pre, hmmdefs_name), - modeln_dir, - hcompv_scp_train, phonelist_txt, - mlf_file=mlf_file_train, - macros=os.path.join(modeln_dir_pre, 'macros')) - print("elapsed time: {}".format(time.time() - timer_start)) + + timer_start = time.time() + niter = chtk.re_estimation_until_saturated( + model1_dir, + model0_dir, improvement_threshold, hcompv_scp_train, + os.path.join(htk_stimmen_dir, 'mfc'), + 'mfc', + os.path.join(htk_stimmen_dir, 'word_lattice.ltc'), + mlf_file=mlf_file_train, + lexicon_file=os.path.join(htk_stimmen_dir, 'lexicon_recognition.dic') + ) + + print("elapsed time: {}".format(time.time() - timer_start)) ## ======================= adding sp to the model ======================= if add_sp: print('==== adding sp to the model ====') + # reference: + # http://www.f.waseda.jp/yusukekondo/htk.html#flat_start_estimation # make model with sp. - print('>>> modifying the last model in the previous step...') - modeln_dir_pre = os.path.join(model1_dir, 'iter'+str(niter_max-1)) - modeln_dir = modeln_dir_pre.replace('iter' + str(niter_max-1), 'iter' + str(niter_max)) - fh.make_new_directory(modeln_dir) - shutil.copy( - os.path.join(modeln_dir_pre, 'macros'), - os.path.join(modeln_dir, 'macros')) - shutil.copy( - os.path.join(modeln_dir_pre, hmmdefs_name), - os.path.join(modeln_dir, hmmdefs_name)) + niter = 7 + print('>>> adding sp state to the last model in the previous step...') + fh.make_new_directory(model1sp_dir, existing_dir='leave') + modeln_dir_pre = os.path.join(model1_dir, 'iter'+str(niter)) - ## ======================= - ## manually make changes to modeln_dir/hmmdefs - ## ======================= - # add states 'sil'. - # http://www.f.waseda.jp/yusukekondo/htk.html#flat_start_estimation - #shutil.copy( - # os.path.join(model_dir, 'hmmdefs.txt'), - # os.path.join(modeln_dir, hmmdefs_name)) - - #hmmdefs_file_pre = os.path.join(modeln_dir_pre, hmmdefs_name) - hmmdefs_file = os.path.join(modeln_dir, hmmdefs_name) - macros_file = os.path.join(modeln_dir, 'macros') - #with open(hmmdefs_file_pre) as f: - # lines = f.read() - #lines_ = lines.split('~h ') - #sil_model = [line for line in lines_ if line.split('\n')[0].replace('"', '') == 'sil'][0] - - # update hmmdefs and macros. - print('>>> updating hmmdefs and macros...') - modeln_dir_pre = modeln_dir - modeln_dir = modeln_dir.replace('iter' + str(niter_max), 'iter' + str(niter_max+1)) - fh.make_new_directory(modeln_dir) - pyhtk.include_sil_in_hmmdefs(macros_file, hmmdefs_file, modeln_dir, sil_hed, phonelist_txt) + ## update hmmdefs and macros. + print('>>> adding sp to the model...') + modeln_dir = os.path.join(model1sp_dir, 'iter0') + chtk.add_sp(modeln_dir_pre, modeln_dir) ## ======================= train model with short pause =======================