33 KiB
33 KiB
In [1]:
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import numpy as np
from keras.models import load_model
import sys
import pickle
from prepare_data import *
np.random.seed(7)Using Theano backend.
In [2]:
data = Data('l', shuffle_all_inputs=False)
content = data._read_content('data/SlovarIJS_BESEDE_utf8.lex')
dictionary, max_word, max_num_vowels, vowels, accented_vowels = data._create_dict(content)
feature_dictionary = data._create_slovene_feature_dictionary()
syllable_dictionary = data._create_syllables_dictionary(content, vowels)
accented_vowels = ['ŕ', 'á', 'ä', 'é', 'ë', 'ě', 'í', 'î', 'ó', 'ô', 'ö', 'ú', 'ü']
In [15]:
environment = {}
environment['dictionary'] = dictionary
environment['max_word'] = max_word
environment['max_num_vowels'] = max_num_vowels
environment['vowels'] = vowels
environment['accented_vowels'] = accented_vowels
environment['feature_dictionary'] = feature_dictionary
environment['eng_feature_dictionary'] = feature_dictionary
environment['syllable_dictionary'] = syllable_dictionary
output = open('environment.pkl', 'wb')
pickle.dump(environment, output)
output.close()In [12]:
i = 0
for el in syllable_dictionary:
if el == "da":
print(i)
i += 1407
In [98]:
feature__en_dictionary = data._create_feature_dictionary()
feature__slo_dictionary = data._create_slovene_feature_dictionary()In [3]:
letter_location_model, syllable_location_model, syllabled_letters_location_model = data.load_location_models(
'cnn/word_accetuation/cnn_dictionary/v3_10/20_test_epoch.h5',
'cnn/word_accetuation/syllables/v2_4/20_test_epoch.h5',
'cnn/word_accetuation/syllabled_letters/v2_5_3/20_test_epoch.h5')
letter_type_model, syllable_type_model, syllabled_letter_type_model = data.load_type_models(
'cnn/accent_classification/letters/v2_1/20_test_epoch.h5',
'cnn/accent_classification/syllables/v1_0/20_test_epoch.h5',
'cnn/accent_classification/syllabled_letters/v1_0/20_test_epoch.h5')In [6]:
test_input = [['uradni', '', 'Agpmpn', 'uradni'], ['podatki', '', 'Ncmpn', 'podatki'], ['policije', '', 'Ncfsg', 'policije'], ['kažejo', '', 'Vmpr3p', 'kažejo'], ['na', '', 'Sa', 'na'], ['precej', '', 'Rgp', 'precej'], ['napete', '', 'Appfpa', 'napete'], ['razmere', '', 'Ncfpa', 'razmere'], ['v', '', 'Sl', 'v'], ['piranskem', '', 'Agpmsl', 'piranskem'], ['zalivu', '', 'Ncmsl', 'zalivu'], ['je', '', 'Va-r3s-n', 'je'], ['danes', '', 'Rgp', 'danes'], ['poročala', '', 'Vmpp-sf', 'poročala'], ['oddaja', '', 'Ncfsn', 'oddaja'], ['do', '', 'Sg', 'do'], ['danes', '', 'Rgp', 'danes'], ['se', '', 'Px------y', 'se'], ['je', '', 'Va-r3s-n', 'je'], ['zgodilo', '', 'Vmep-sn', 'zgodilo']]In [16]:
%run prepare_data.py
data = Data('l', shuffle_all_inputs=False)
location_accented_words, accented_words = data.accentuate_word(test_input, letter_location_model, syllable_location_model, syllabled_letters_location_model,
letter_type_model, syllable_type_model, syllabled_letter_type_model,
dictionary, max_word, max_num_vowels, vowels, accented_vowels, feature_dictionary, syllable_dictionary)In [19]:
print(location_accented_words)
print(accented_words)['uradní', 'podatkí', 'policíje', 'kažéjo', 'ná', 'precéj', 'napeté', 'razmeré', 'v', 'piranském', 'zalivú', 'jé', 'danés', 'poročála', 'oddajá', 'dó', 'danés', 'sé', 'jé', 'zgodílo'] ['uradnî', 'podatkî', 'policíje', 'kažëjo', 'ná', 'precëj', 'napetë', 'razmerë', 'v', 'piranskëm', 'zalivú', 'jë', 'danës', 'poročála', 'oddajá', 'dó', 'danës', 'së', 'jë', 'zgodílo']
In [67]:
def predict_word(word_acentuation_model, accent_type_model, word, msd, dictionary, max_word, max_num_vowels, vowels, accented_vowels, feature_dictionary):
eye_input_accent = np.eye(10, dtype=int)
english_msd = msd
fake_content = [[word, '-', msd, '-']]
x, x_other_features, fake_y = data._generate_x_and_y(dictionary, max_word, max_num_vowels, fake_content, vowels, accented_vowels, feature_dictionary, 'who cares')
# print(x)
accent_loc = word_acentuation_model.predict([x, x_other_features])
j = 0
word=list(word)[::-1]
# print(word)
# print(accent_loc)
for i in range(len(word)):
if data._is_vowel(word, i, vowels):
if accent_loc[0][j] >= 0.5:
# print(x_other_features[0])
# print(eye_input_accent[i])
new_x_other_features = np.array([np.concatenate((x_other_features[0], eye_input_accent[j]))])
# print(x_other_features)
# print(new_x_other_features)
final_accent = accent_type_model.predict([x, new_x_other_features])
# print(accented_vowels[final_accent[0].argmax(axis=0)])
word[i] = accented_vowels[final_accent[0].argmax(axis=0)]
# print(final_accent)
j += 1
return ''.join(word[::-1])
predict_word(word_acentuation_model, accent_type_model, 'nadnaravnih', 'Afpfdg', dictionary, max_word, max_num_vowels, vowels, accented_vowels, feature_dictionary)Out [67]:
CREATING OTHER FEATURES... OTHER FEATURES CREATED!
'nädnarávnih'
In [4]:
from lxml import etree
def xml_words_generator(xml_path):
for event, element in etree.iterparse(xml_path, tag="LexicalEntry", encoding="UTF-8"):
words = []
for child in element:
if child.tag == 'WordForm':
msd = None
word = None
for wf in child:
if 'att' in wf.attrib and wf.attrib['att'] == 'msd':
msd = wf.attrib['val']
elif wf.tag == 'FormRepresentation':
for form_rep in wf:
if form_rep.attrib['att'] == 'zapis_oblike':
word = form_rep.attrib['val']
#if msd is not None and word is not None:
# pass
#else:
# print('NOOOOO')
words.append([word, '', msd, word])
yield words
gen = xml_words_generator('data/Sloleks_v1.2.xml')In [7]:
# SPLIT ALL TEXT!!!
NUM_OF_LINES=16660466
filename = 'data/Sloleks_v1.2.xml'
with open(filename) as fin:
fout = open('data/Sloleks_v1.2_p2.xml',"a")
for i,line in enumerate(fin):
if NUM_OF_LINES < i:
fout.write(line)
fout.close()
fout = open('data/Sloleks_v1.2_p2.xml',"a")
fout.close()In [5]:
words = []
while len(words) < 50000:
words.extend(next(gen))
print(len(words))50017
In [99]:
print(feature__en_dictionary)
print(feature__slo_dictionary)[[21, 'A', ['g', 's'], ['p', 'c', 's'], ['m', 'f', 'n'], ['s', 'd', 'p'], ['n', 'g', 'd', 'a', 'l', 'i'], ['-', 'n', 'y']], [3, 'C', ['c', 's']], [1, 'I'], [21, 'M', ['l'], ['-', 'c', 'o', 's'], ['m', 'f', 'n'], ['s', 'd', 'p'], ['n', 'g', 'd', 'a', 'l', 'i'], ['-', 'n', 'y']], [17, 'N', ['c'], ['m', 'f', 'n'], ['s', 'd', 'p'], ['n', 'g', 'd', 'a', 'l', 'i'], ['-', 'n', 'y']], [40, 'P', ['p', 's', 'd', 'r', 'x', 'g', 'q', 'i', 'z'], ['-', '1', '2', '3'], ['-', 'm', 'f', 'n'], ['-', 's', 'd', 'p'], ['-', 'n', 'g', 'd', 'a', 'l', 'i'], ['-', 's', 'd', 'p'], ['-', 'm', 'f', 'n'], ['-', 'y', 'b']], [1, 'Q'], [5, 'R', ['g'], ['p', 'c', 's']], [7, 'S', ['-', 'g', 'd', 'a', 'l', 'i']], [24, 'V', ['m'], ['-'], ['n', 'u', 'p', 'r', 'f', 'c'], ['-', '1', '2', '3'], ['-', 's', 'p', 'd'], ['-', 'm', 'f', 'n'], ['-', 'n', 'y']]] [[21, 'P', ['p', 's'], ['n', 'p', 's'], ['m', 'z', 's'], ['e', 'd', 'm'], ['i', 'r', 'd', 't', 'm', 'o'], ['-', 'n', 'd']], [3, 'V', ['p', 'd']], [1, 'M'], [21, 'K', ['b'], ['-', 'g', 'v', 'd'], ['m', 'z', 's'], ['e', 'd', 'm'], ['i', 'r', 'd', 't', 'm', 'o'], ['-', 'n', 'd']], [17, 'S', ['o'], ['m', 'z', 's'], ['e', 'd', 'm'], ['i', 'r', 'd', 't', 'm', 'o'], ['-', 'n', 'd']], [40, 'Z', ['o', 's', 'k', 'z', 'p', 'c', 'v', 'n', 'l'], ['-', 'p', 'd', 't'], ['-', 'm', 'z', 's'], ['-', 'e', 'd', 'm'], ['-', 'i', 'r', 'd', 't', 'm', 'o'], ['-', 'e', 'd', 'm'], ['-', 'm', 'z', 's'], ['-', 'k', 'z']], [1, 'L'], [5, 'R', ['s'], ['n', 'r', 's']], [7, 'D', ['-', 'r', 'd', 't', 'm', 'o']], [24, 'G', ['g'], ['-'], ['n', 'm', 'd', 's', 'p', 'g'], ['-', 'p', 'd', 't'], ['-', 'e', 'm', 'd'], ['-', 'm', 'z', 's'], ['-', 'n', 'd']]]
In [5]:
accented_vowels = ['ŕ', 'á', 'ä', 'é', 'ë', 'ě', 'í', 'î', 'ó', 'ô', 'ö', 'ú', 'ü']
words = [["Gorejevemu", "", "Psnsed", "Gorejevemu"]]In [29]:
%run prepare_data.py
data = Data('l', shuffle_all_inputs=False)
location_accented_words, accented_words = data.accentuate_word(words, letter_location_model, syllable_location_model, syllabled_letters_location_model,
letter_type_model, syllable_type_model, syllabled_letter_type_model,
dictionary, max_word, max_num_vowels, vowels, accented_vowels, feature_dictionary, syllable_dictionary)In [159]:
pos = 4282
print(location_accented_words)
print(accented_words)
print(words)['Gorejévemu'] ['Gorejěvemu'] [['Gorejevemu', '', 'Psnsed', 'Gorejevemu']]
In [165]:
import time
hello
[0;31m---------------------------------------------------------------------------[0m [0;31mNameError[0m Traceback (most recent call last) [0;32m<ipython-input-165-0a2d5e185f36>[0m in [0;36m<module>[0;34m()[0m [1;32m 3[0m [0mstart[0m [0;34m=[0m [0mtime[0m[0;34m.[0m[0mtime[0m[0;34m([0m[0;34m)[0m[0;34m[0m[0m [1;32m 4[0m [0mprint[0m[0;34m([0m[0;34m"hello"[0m[0;34m)[0m[0;34m[0m[0m [0;32m----> 5[0;31m [0mwait[0m[0;34m([0m[0;36m2[0m[0;34m)[0m[0;34m[0m[0m [0m[1;32m 6[0m [0mend[0m [0;34m=[0m [0mtime[0m[0;34m.[0m[0mtime[0m[0;34m([0m[0;34m)[0m[0;34m[0m[0m [1;32m 7[0m [0mprint[0m[0;34m([0m[0mend[0m [0;34m-[0m [0mstart[0m[0;34m)[0m[0;34m[0m[0m [0;31mNameError[0m: name 'wait' is not defined
In [5]:
from lxml import etree
import time
gen = xml_words_generator('data/Sloleks_v1.2.xml')
word_glob_num = 0
word_limit = 0
iter_num = 50000
word_index = 0
start_timer = time.time()
iter_index = 0
myfile = open('data/new_sloleks/test' + str(iter_index) + '.xml', 'ab')
#with open("data/new_sloleks/new_sloleks.xml", "ab") as myfile:
enable_print = False
for event, element in etree.iterparse('data/Sloleks_v1.2.xml', tag="LexicalEntry", encoding="UTF-8", remove_blank_text=True):
# LOAD NEW WORDS AND ACCENTUATE THEM
if word_glob_num >= word_limit:
myfile.close()
myfile = open('data/new_sloleks/test' + str(iter_index) + '.xml', 'ab')
#if iter_index == 5:
# break
iter_index += 1
print("Words proccesed: " + str(word_glob_num))
print("Word indeks: " + str(word_index))
print("Word number: " + str(len(words)))
end_timer = time.time()
print("Elapsed time: " + "{0:.2f}".format((end_timer - start_timer)/60.0) + " minutes")
word_index = 0
words = []
#print("HERE!!!")
while len(words) < iter_num:
try:
words.extend(next(gen))
except:
break
#print("HERE!!!")
#if word_glob_num > 1:
# break
word_limit += len(words)
#print("HERE!!!")
for child in element:
if child.tag == 'WordForm':
#msd = None
#word = None
for wf in child:
if wf.tag == 'FormRepresentation':
sloleks_word = None
for form_rep in wf:
if form_rep.attrib['att'] == 'zapis_oblike':
sloleks_word = form_rep.attrib['val']
if sloleks_word != words[word_index][0]:
print(sloleks_word)
print(words[word_index][0])
print(word_index)
#if sloleks_word ==
new_element = etree.Element('feat')
new_element.attrib['att']='naglasna_mesta_oblike'
new_element.attrib['val']=words[word_index][0]
wf.append(new_element)
new_element = etree.Element('feat')
new_element.attrib['att']='naglašena_oblika'
new_element.attrib['val']=words[word_index][0]
wf.append(new_element)
word_glob_num += 1
word_index += 1
#myfile.write(etree.tostring(element, encoding="UTF-8", pretty_print=True))
element.clear()
Words proccesed: 0 Word indeks: 0
[0;31m---------------------------------------------------------------------------[0m [0;31mNameError[0m Traceback (most recent call last) [0;32m<ipython-input-5-0e24b34aba55>[0m in [0;36m<module>[0;34m()[0m [1;32m 26[0m [0;34m[0m[0m [1;32m 27[0m [0mprint[0m[0;34m([0m[0;34m"Word indeks: "[0m [0;34m+[0m [0mstr[0m[0;34m([0m[0mword_index[0m[0;34m)[0m[0;34m)[0m[0;34m[0m[0m [0;32m---> 28[0;31m [0mprint[0m[0;34m([0m[0;34m"Word number: "[0m [0;34m+[0m [0mstr[0m[0;34m([0m[0mlen[0m[0;34m([0m[0mwords[0m[0;34m)[0m[0;34m)[0m[0;34m)[0m[0;34m[0m[0m [0m[1;32m 29[0m [0;34m[0m[0m [1;32m 30[0m [0mend_timer[0m [0;34m=[0m [0mtime[0m[0;34m.[0m[0mtime[0m[0;34m([0m[0;34m)[0m[0;34m[0m[0m [0;31mNameError[0m: name 'words' is not defined
In [6]:
problem_words = []In [6]:
#Words proccesed: 650250
#Word indeks: 50023
#Word number: 50023
from lxml import etree
import time
gen = xml_words_generator('data/Sloleks_v1.2_p2.xml')
word_glob_num = 0
word_limit = 0
iter_num = 50000
word_index = 0
start_timer = time.time()
iter_index = 0
words = []
lexical_entries_load_number = 0
lexical_entries_save_number = 0
# INSIDE
#word_glob_num = 1500686
word_glob_num = 1550705
#word_limit = 1500686
word_limit = 1550705
iter_index = 31
#done_lexical_entries = 33522
with open("data/new_sloleks/new_sloleks.xml", "ab") as myfile:
myfile2 = open('data/new_sloleks/p' + str(iter_index) + '.xml', 'ab')
for event, element in etree.iterparse('data/Sloleks_v1.2_p2.xml', tag="LexicalEntry", encoding="UTF-8", remove_blank_text=True):
# LOAD NEW WORDS AND ACCENTUATE THEM
#print("HERE")
# if lexical_entries_save_number < done_lexical_entries:
# next(gen)
# #print(lexical_entries_save_number)
# lexical_entries_save_number += 1
# lexical_entries_load_number += 1
# continue
if word_glob_num >= word_limit:
myfile2.close()
myfile2 = open('data/new_sloleks/p' + str(iter_index) + '.xml', 'ab')
iter_index += 1
print("Words proccesed: " + str(word_glob_num))
print("Word indeks: " + str(word_index))
print("Word number: " + str(len(words)))
#print("lexical_entries_load_number: " + str(lexical_entries_load_number))
#print("lexical_entries_save_number: " + str(lexical_entries_save_number))
end_timer = time.time()
print("Elapsed time: " + "{0:.2f}".format((end_timer - start_timer)/60.0) + " minutes")
word_index = 0
words = []
while len(words) < iter_num:
try:
words.extend(next(gen))
lexical_entries_load_number += 1
except:
break
#if word_glob_num > 1:
# break
#problem_words = words
#break
data = Data('l', shuffle_all_inputs=False)
location_accented_words, accented_words = data.accentuate_word(words, letter_location_model, syllable_location_model, syllabled_letters_location_model,
letter_type_model, syllable_type_model, syllabled_letter_type_model,
dictionary, max_word, max_num_vowels, vowels, accented_vowels, feature_dictionary, syllable_dictionary)
word_limit += len(words)
# READ DATA
for child in element:
if child.tag == 'WordForm':
msd = None
word = None
for wf in child:
if wf.tag == 'FormRepresentation':
new_element = etree.Element('feat')
new_element.attrib['att']='naglasna_mesta_oblike'
new_element.attrib['val']=location_accented_words[word_index]
wf.append(new_element)
new_element = etree.Element('feat')
new_element.attrib['att']='naglašena_oblika'
new_element.attrib['val']=accented_words[word_index]
wf.append(new_element)
word_glob_num += 1
word_index += 1
# print(etree.tostring(element, encoding="UTF-8"))
myfile2.write(etree.tostring(element, encoding="UTF-8", pretty_print=True))
myfile.write(etree.tostring(element, encoding="UTF-8", pretty_print=True))
element.clear()
lexical_entries_save_number += 1
Words proccesed: 1550705 Word indeks: 0 Word number: 0 Elapsed time: 0.00 minutes Words proccesed: 1600757 Word indeks: 50052 Word number: 50052 Elapsed time: 9.39 minutes Words proccesed: 1650762 Word indeks: 50005 Word number: 50005 Elapsed time: 18.22 minutes Words proccesed: 1700781 Word indeks: 50019 Word number: 50019 Elapsed time: 27.47 minutes Words proccesed: 1750833 Word indeks: 50052 Word number: 50052 Elapsed time: 36.58 minutes Words proccesed: 1800864 Word indeks: 50031 Word number: 50031 Elapsed time: 45.39 minutes Words proccesed: 1850886 Word indeks: 50022 Word number: 50022 Elapsed time: 54.31 minutes Words proccesed: 1900898 Word indeks: 50012 Word number: 50012 Elapsed time: 62.81 minutes Words proccesed: 1950911 Word indeks: 50013 Word number: 50013 Elapsed time: 70.84 minutes Words proccesed: 2000920 Word indeks: 50009 Word number: 50009 Elapsed time: 79.08 minutes Words proccesed: 2050927 Word indeks: 50007 Word number: 50007 Elapsed time: 87.50 minutes Words proccesed: 2100944 Word indeks: 50017 Word number: 50017 Elapsed time: 95.62 minutes Words proccesed: 2150949 Word indeks: 50005 Word number: 50005 Elapsed time: 104.08 minutes Words proccesed: 2200958 Word indeks: 50009 Word number: 50009 Elapsed time: 112.44 minutes Words proccesed: 2250969 Word indeks: 50011 Word number: 50011 Elapsed time: 120.68 minutes Words proccesed: 2300978 Word indeks: 50009 Word number: 50009 Elapsed time: 129.58 minutes Words proccesed: 2350986 Word indeks: 50008 Word number: 50008 Elapsed time: 139.40 minutes Words proccesed: 2400993 Word indeks: 50007 Word number: 50007 Elapsed time: 148.05 minutes Words proccesed: 2451000 Word indeks: 50007 Word number: 50007 Elapsed time: 156.79 minutes Words proccesed: 2501005 Word indeks: 50005 Word number: 50005 Elapsed time: 165.57 minutes Words proccesed: 2551016 Word indeks: 50011 Word number: 50011 Elapsed time: 174.29 minutes Words proccesed: 2601024 Word indeks: 50008 Word number: 50008 Elapsed time: 183.20 minutes Words proccesed: 2651037 Word indeks: 50013 Word number: 50013 Elapsed time: 191.94 minutes Words proccesed: 2701046 Word indeks: 50009 Word number: 50009 Elapsed time: 200.67 minutes Words proccesed: 2751050 Word indeks: 50004 Word number: 50004 Elapsed time: 209.40 minutes
In [10]:
len(problem_words)Out [10]:
50052
In [ ]:
%run prepare_data.py
data = Data('l', shuffle_all_inputs=False)
location_accented_words, accented_words = data.accentuate_word(problem_words[:], letter_location_model, syllable_location_model, syllabled_letters_location_model,
letter_type_model, syllable_type_model, syllabled_letter_type_model,
dictionary, max_word, max_num_vowels, vowels, accented_vowels, feature_dictionary, syllable_dictionary)In [21]:
# CALCULATE INDEX NUMBER:
previous_file_len = [622061, 618306, 618266, 618483, 619342]
word_nums = [50017, 50007, 50017, 50012, 50024]
def calculate_index(previous_files_len, word_nums):
return sum(previous_files_len) - 2 * sum(word_nums) + 11
calculate_index(previous_file_len[:3], word_nums[:3])Out [21]:
1558562
In [12]:
print(word_glob_num)
print(word_limit)
print(iter_num)
print(word_index)250002 250000 50000 0
In [22]:
max_wordOut [22]:
23
In [7]:
words[0]Out [7]:
['zapirati', '', 'Ggnn', 'zapirati']
In [ ]:
with open("new_sloleks.xml", "ab") as myfile:
for event, element in etree.iterparse('data/Sloleks_v1.2.xml', tag="LexicalEntry", encoding="UTF-8", remove_blank_text=True):
# READ DATA
for child in element:
if child.tag == 'WordForm':
msd = None
word = None
for wf in child:
# print(wf.attrib['att'])
if 'att' in wf.attrib and wf.attrib['att'] == 'msd':
msd = wf.attrib['val']
elif wf.tag == 'FormRepresentation':
for form_rep in wf:
if form_rep.attrib['att'] == 'zapis_oblike':
word = form_rep.attrib['val']
new_element = etree.Element('feat')
new_element.attrib['att']='naglasna_mesta_oblike'
new_element.attrib['val']='test'
wf.append(new_element)
new_element = etree.Element('feat')
new_element.attrib['att']='naglašena_oblika'
new_element.attrib['val']='test'
wf.append(new_element)
if msd is not None and word is not None:
print(msd)
print(word)
else:
print('NOOOOO')
print(etree.tostring(element, encoding="UTF-8"))
myfile.write(etree.tostring(element, encoding="UTF-8", pretty_print=True))
element.clear()
break