Files
stress_asignment/sloleks_accetuation.ipynb
T

33 KiB

In [1]:
# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import numpy as np
from keras.models import load_model
import sys
import pickle

from prepare_data import *

np.random.seed(7)
Using Theano backend.
In [2]:
data = Data('l', shuffle_all_inputs=False)
content = data._read_content('data/SlovarIJS_BESEDE_utf8.lex')
dictionary, max_word, max_num_vowels, vowels, accented_vowels = data._create_dict(content)
feature_dictionary = data._create_slovene_feature_dictionary()
syllable_dictionary = data._create_syllables_dictionary(content, vowels)
accented_vowels = ['ŕ', 'á', 'ä', 'é', 'ë', 'ě', 'í', 'î', 'ó', 'ô', 'ö', 'ú', 'ü']

In [15]:
environment = {}
environment['dictionary'] = dictionary
environment['max_word'] = max_word
environment['max_num_vowels'] = max_num_vowels
environment['vowels'] = vowels
environment['accented_vowels'] = accented_vowels
environment['feature_dictionary'] = feature_dictionary
environment['eng_feature_dictionary'] = feature_dictionary
environment['syllable_dictionary'] = syllable_dictionary
output = open('environment.pkl', 'wb')
pickle.dump(environment, output)
output.close()
In [12]:
i = 0
for el in syllable_dictionary:
    if el == "da":
        print(i)
    i += 1
407
In [98]:
feature__en_dictionary = data._create_feature_dictionary()
feature__slo_dictionary = data._create_slovene_feature_dictionary()
In [3]:
letter_location_model, syllable_location_model, syllabled_letters_location_model = data.load_location_models(
    'cnn/word_accetuation/cnn_dictionary/v3_10/20_test_epoch.h5',
    'cnn/word_accetuation/syllables/v2_4/20_test_epoch.h5',
    'cnn/word_accetuation/syllabled_letters/v2_5_3/20_test_epoch.h5')

letter_type_model, syllable_type_model, syllabled_letter_type_model = data.load_type_models(
    'cnn/accent_classification/letters/v2_1/20_test_epoch.h5',
    'cnn/accent_classification/syllables/v1_0/20_test_epoch.h5',
    'cnn/accent_classification/syllabled_letters/v1_0/20_test_epoch.h5')
In [6]:
test_input = [['uradni', '', 'Agpmpn', 'uradni'], ['podatki', '', 'Ncmpn', 'podatki'], ['policije', '', 'Ncfsg', 'policije'], ['kažejo', '', 'Vmpr3p', 'kažejo'], ['na', '', 'Sa', 'na'], ['precej', '', 'Rgp', 'precej'], ['napete', '', 'Appfpa', 'napete'], ['razmere', '', 'Ncfpa', 'razmere'], ['v', '', 'Sl', 'v'], ['piranskem', '', 'Agpmsl', 'piranskem'], ['zalivu', '', 'Ncmsl', 'zalivu'], ['je', '', 'Va-r3s-n', 'je'], ['danes', '', 'Rgp', 'danes'], ['poročala', '', 'Vmpp-sf', 'poročala'], ['oddaja', '', 'Ncfsn', 'oddaja'], ['do', '', 'Sg', 'do'], ['danes', '', 'Rgp', 'danes'], ['se', '', 'Px------y', 'se'], ['je', '', 'Va-r3s-n', 'je'], ['zgodilo', '', 'Vmep-sn', 'zgodilo']]
In [16]:
%run prepare_data.py
data = Data('l', shuffle_all_inputs=False)
location_accented_words, accented_words = data.accentuate_word(test_input, letter_location_model, syllable_location_model, syllabled_letters_location_model,
                        letter_type_model, syllable_type_model, syllabled_letter_type_model,
                        dictionary, max_word, max_num_vowels, vowels, accented_vowels, feature_dictionary, syllable_dictionary)
In [19]:
print(location_accented_words)
print(accented_words)
['uradní', 'podatkí', 'policíje', 'kažéjo', 'ná', 'precéj', 'napeté', 'razmeré', 'v', 'piranském', 'zalivú', 'jé', 'danés', 'poročála', 'oddajá', 'dó', 'danés', 'sé', 'jé', 'zgodílo']
['uradnî', 'podatkî', 'policíje', 'kažëjo', 'ná', 'precëj', 'napetë', 'razmerë', 'v', 'piranskëm', 'zalivú', 'jë', 'danës', 'poročála', 'oddajá', 'dó', 'danës', 'së', 'jë', 'zgodílo']
In [67]:
def predict_word(word_acentuation_model, accent_type_model, word, msd, dictionary, max_word, max_num_vowels, vowels, accented_vowels, feature_dictionary):
    eye_input_accent = np.eye(10, dtype=int)
    
    english_msd = msd
    fake_content = [[word, '-', msd, '-']]
    x, x_other_features, fake_y = data._generate_x_and_y(dictionary, max_word, max_num_vowels, fake_content, vowels, accented_vowels, feature_dictionary, 'who cares')
#     print(x)
    accent_loc = word_acentuation_model.predict([x, x_other_features])
    
    j = 0
    word=list(word)[::-1]
#     print(word)
#     print(accent_loc)
    
    for i in range(len(word)):
        if data._is_vowel(word, i, vowels):
            if accent_loc[0][j] >= 0.5:
    #             print(x_other_features[0])
    #             print(eye_input_accent[i])
                new_x_other_features = np.array([np.concatenate((x_other_features[0], eye_input_accent[j]))])
    #             print(x_other_features)
    #             print(new_x_other_features)
                final_accent = accent_type_model.predict([x, new_x_other_features])
#                 print(accented_vowels[final_accent[0].argmax(axis=0)])
                word[i] = accented_vowels[final_accent[0].argmax(axis=0)]
#                 print(final_accent)
            j += 1

    
    
    return ''.join(word[::-1])

predict_word(word_acentuation_model, accent_type_model, 'nadnaravnih', 'Afpfdg', dictionary, max_word, max_num_vowels, vowels, accented_vowels, feature_dictionary)
Out [67]:
CREATING OTHER FEATURES...
OTHER FEATURES CREATED!
'nädnarávnih'
In [4]:
from lxml import etree

def xml_words_generator(xml_path):
    for event, element in etree.iterparse(xml_path, tag="LexicalEntry", encoding="UTF-8"):
        words = []
        for child in element:
            if child.tag == 'WordForm':
                msd = None
                word = None
                for wf in child:
                    if 'att' in wf.attrib and wf.attrib['att'] == 'msd':
                        msd = wf.attrib['val']
                    elif wf.tag == 'FormRepresentation':
                        for form_rep in wf:
                            if form_rep.attrib['att'] == 'zapis_oblike':
                                word = form_rep.attrib['val']
                        #if msd is not None and word is not None:
                        #    pass
                        #else:
                        #    print('NOOOOO')
                        words.append([word, '', msd, word])
        yield words
        
gen = xml_words_generator('data/Sloleks_v1.2.xml')
In [7]:
# SPLIT ALL TEXT!!!
NUM_OF_LINES=16660466
filename = 'data/Sloleks_v1.2.xml'
with open(filename) as fin:
    fout = open('data/Sloleks_v1.2_p2.xml',"a")
    for i,line in enumerate(fin):
        if NUM_OF_LINES < i:
            fout.write(line)
            fout.close()
            fout = open('data/Sloleks_v1.2_p2.xml',"a")

    fout.close()
In [5]:
words = []
while len(words) < 50000:
    words.extend(next(gen))
print(len(words))
50017
In [99]:
print(feature__en_dictionary)
print(feature__slo_dictionary)
[[21, 'A', ['g', 's'], ['p', 'c', 's'], ['m', 'f', 'n'], ['s', 'd', 'p'], ['n', 'g', 'd', 'a', 'l', 'i'], ['-', 'n', 'y']], [3, 'C', ['c', 's']], [1, 'I'], [21, 'M', ['l'], ['-', 'c', 'o', 's'], ['m', 'f', 'n'], ['s', 'd', 'p'], ['n', 'g', 'd', 'a', 'l', 'i'], ['-', 'n', 'y']], [17, 'N', ['c'], ['m', 'f', 'n'], ['s', 'd', 'p'], ['n', 'g', 'd', 'a', 'l', 'i'], ['-', 'n', 'y']], [40, 'P', ['p', 's', 'd', 'r', 'x', 'g', 'q', 'i', 'z'], ['-', '1', '2', '3'], ['-', 'm', 'f', 'n'], ['-', 's', 'd', 'p'], ['-', 'n', 'g', 'd', 'a', 'l', 'i'], ['-', 's', 'd', 'p'], ['-', 'm', 'f', 'n'], ['-', 'y', 'b']], [1, 'Q'], [5, 'R', ['g'], ['p', 'c', 's']], [7, 'S', ['-', 'g', 'd', 'a', 'l', 'i']], [24, 'V', ['m'], ['-'], ['n', 'u', 'p', 'r', 'f', 'c'], ['-', '1', '2', '3'], ['-', 's', 'p', 'd'], ['-', 'm', 'f', 'n'], ['-', 'n', 'y']]]
[[21, 'P', ['p', 's'], ['n', 'p', 's'], ['m', 'z', 's'], ['e', 'd', 'm'], ['i', 'r', 'd', 't', 'm', 'o'], ['-', 'n', 'd']], [3, 'V', ['p', 'd']], [1, 'M'], [21, 'K', ['b'], ['-', 'g', 'v', 'd'], ['m', 'z', 's'], ['e', 'd', 'm'], ['i', 'r', 'd', 't', 'm', 'o'], ['-', 'n', 'd']], [17, 'S', ['o'], ['m', 'z', 's'], ['e', 'd', 'm'], ['i', 'r', 'd', 't', 'm', 'o'], ['-', 'n', 'd']], [40, 'Z', ['o', 's', 'k', 'z', 'p', 'c', 'v', 'n', 'l'], ['-', 'p', 'd', 't'], ['-', 'm', 'z', 's'], ['-', 'e', 'd', 'm'], ['-', 'i', 'r', 'd', 't', 'm', 'o'], ['-', 'e', 'd', 'm'], ['-', 'm', 'z', 's'], ['-', 'k', 'z']], [1, 'L'], [5, 'R', ['s'], ['n', 'r', 's']], [7, 'D', ['-', 'r', 'd', 't', 'm', 'o']], [24, 'G', ['g'], ['-'], ['n', 'm', 'd', 's', 'p', 'g'], ['-', 'p', 'd', 't'], ['-', 'e', 'm', 'd'], ['-', 'm', 'z', 's'], ['-', 'n', 'd']]]
In [5]:
accented_vowels = ['ŕ', 'á', 'ä', 'é', 'ë', 'ě', 'í', 'î', 'ó', 'ô', 'ö', 'ú', 'ü']
words = [["Gorejevemu", "", "Psnsed", "Gorejevemu"]]
In [29]:
%run prepare_data.py
data = Data('l', shuffle_all_inputs=False)
location_accented_words, accented_words = data.accentuate_word(words, letter_location_model, syllable_location_model, syllabled_letters_location_model,
                        letter_type_model, syllable_type_model, syllabled_letter_type_model,
                        dictionary, max_word, max_num_vowels, vowels, accented_vowels, feature_dictionary, syllable_dictionary)
In [159]:
pos = 4282
print(location_accented_words)
print(accented_words)
print(words)
['Gorejévemu']
['Gorejěvemu']
[['Gorejevemu', '', 'Psnsed', 'Gorejevemu']]
In [165]:
import time

hello
---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-165-0a2d5e185f36> in <module>()
      3 start = time.time()
      4 print("hello")
----> 5 wait(2)
      6 end = time.time()
      7 print(end - start)

NameError: name 'wait' is not defined
In [5]:
from lxml import etree
import time

gen = xml_words_generator('data/Sloleks_v1.2.xml')
word_glob_num = 0
word_limit = 0
iter_num = 50000
word_index = 0
start_timer = time.time()
iter_index = 0
myfile = open('data/new_sloleks/test' + str(iter_index) + '.xml', 'ab')
#with open("data/new_sloleks/new_sloleks.xml", "ab") as myfile:

enable_print = False

for event, element in etree.iterparse('data/Sloleks_v1.2.xml', tag="LexicalEntry", encoding="UTF-8", remove_blank_text=True):
    # LOAD NEW WORDS AND ACCENTUATE THEM
    if word_glob_num >= word_limit:
        myfile.close()
        myfile = open('data/new_sloleks/test' + str(iter_index) + '.xml', 'ab')
        #if iter_index == 5:
        #    break
        
        iter_index += 1
        print("Words proccesed: " + str(word_glob_num))
        
        print("Word indeks: " + str(word_index))
        print("Word number: " + str(len(words)))
        
        end_timer = time.time()
        print("Elapsed time: " + "{0:.2f}".format((end_timer - start_timer)/60.0) + " minutes")
        
        
        word_index = 0
        words = []
        #print("HERE!!!")
        while len(words) < iter_num:
            try:
                words.extend(next(gen))
            except:
                break
        #print("HERE!!!")
        #if word_glob_num > 1:
        #    break

        word_limit += len(words)
    #print("HERE!!!")
    
    
    
    for child in element:
        if child.tag == 'WordForm':
            #msd = None
            #word = None
            for wf in child:
                if wf.tag == 'FormRepresentation':
                    sloleks_word = None
                    for form_rep in wf:
                        if form_rep.attrib['att'] == 'zapis_oblike':
                            sloleks_word = form_rep.attrib['val']
                    
                    if sloleks_word != words[word_index][0]:
                        print(sloleks_word)
                        print(words[word_index][0])
                        print(word_index)
                            
                    
                    #if sloleks_word == 
                    new_element = etree.Element('feat')
                    new_element.attrib['att']='naglasna_mesta_oblike'
                    new_element.attrib['val']=words[word_index][0]
                    wf.append(new_element)

                    new_element = etree.Element('feat')
                    new_element.attrib['att']='naglašena_oblika'
                    new_element.attrib['val']=words[word_index][0]
                    wf.append(new_element)
                    word_glob_num += 1
                    word_index += 1

    #myfile.write(etree.tostring(element, encoding="UTF-8", pretty_print=True))
    element.clear()
        
    
Words proccesed: 0
Word indeks: 0
---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-5-0e24b34aba55> in <module>()
     26 
     27         print("Word indeks: " + str(word_index))
---> 28         print("Word number: " + str(len(words)))
     29 
     30         end_timer = time.time()

NameError: name 'words' is not defined
In [6]:
problem_words = []
In [6]:
#Words proccesed: 650250
#Word indeks: 50023
#Word number: 50023

from lxml import etree
import time

gen = xml_words_generator('data/Sloleks_v1.2_p2.xml')
word_glob_num = 0
word_limit = 0
iter_num = 50000
word_index = 0
start_timer = time.time()
iter_index = 0
words = []

lexical_entries_load_number = 0
lexical_entries_save_number = 0


# INSIDE
#word_glob_num = 1500686
word_glob_num = 1550705

#word_limit = 1500686
word_limit = 1550705


iter_index = 31

#done_lexical_entries = 33522

with open("data/new_sloleks/new_sloleks.xml", "ab") as myfile:
    myfile2 = open('data/new_sloleks/p' + str(iter_index) + '.xml', 'ab')
    for event, element in etree.iterparse('data/Sloleks_v1.2_p2.xml', tag="LexicalEntry", encoding="UTF-8", remove_blank_text=True):
        # LOAD NEW WORDS AND ACCENTUATE THEM
        #print("HERE")
        
#        if lexical_entries_save_number < done_lexical_entries:
#            next(gen)
#            #print(lexical_entries_save_number)
#            lexical_entries_save_number += 1
#            lexical_entries_load_number += 1
#            continue
        
        if word_glob_num >= word_limit:
            myfile2.close()
            myfile2 = open('data/new_sloleks/p' + str(iter_index) + '.xml', 'ab')
            iter_index += 1
            print("Words proccesed: " + str(word_glob_num))

            print("Word indeks: " + str(word_index))
            print("Word number: " + str(len(words)))
            
            #print("lexical_entries_load_number: " + str(lexical_entries_load_number))
            #print("lexical_entries_save_number: " + str(lexical_entries_save_number))

            end_timer = time.time()
            print("Elapsed time: " + "{0:.2f}".format((end_timer - start_timer)/60.0) + " minutes")


            word_index = 0
            words = []

            while len(words) < iter_num:
                try:
                    words.extend(next(gen))
                    lexical_entries_load_number += 1
                except:
                    break
            #if word_glob_num > 1:
            #    break

            #problem_words = words
            #break
            data = Data('l', shuffle_all_inputs=False)
            location_accented_words, accented_words = data.accentuate_word(words, letter_location_model, syllable_location_model, syllabled_letters_location_model,
                                    letter_type_model, syllable_type_model, syllabled_letter_type_model,
                                    dictionary, max_word, max_num_vowels, vowels, accented_vowels, feature_dictionary, syllable_dictionary)

            word_limit += len(words)
            
        
        # READ DATA
        for child in element:
            if child.tag == 'WordForm':
                msd = None
                word = None
                for wf in child:
                    if wf.tag == 'FormRepresentation':
                        new_element = etree.Element('feat')
                        new_element.attrib['att']='naglasna_mesta_oblike'
                        new_element.attrib['val']=location_accented_words[word_index]
                        wf.append(new_element)

                        new_element = etree.Element('feat')
                        new_element.attrib['att']='naglašena_oblika'
                        new_element.attrib['val']=accented_words[word_index]
                        wf.append(new_element)
                        word_glob_num += 1
                        word_index += 1

        # print(etree.tostring(element, encoding="UTF-8"))
        myfile2.write(etree.tostring(element, encoding="UTF-8", pretty_print=True))
        myfile.write(etree.tostring(element, encoding="UTF-8", pretty_print=True))
        element.clear()
        lexical_entries_save_number += 1
    
Words proccesed: 1550705
Word indeks: 0
Word number: 0
Elapsed time: 0.00 minutes
Words proccesed: 1600757
Word indeks: 50052
Word number: 50052
Elapsed time: 9.39 minutes
Words proccesed: 1650762
Word indeks: 50005
Word number: 50005
Elapsed time: 18.22 minutes
Words proccesed: 1700781
Word indeks: 50019
Word number: 50019
Elapsed time: 27.47 minutes
Words proccesed: 1750833
Word indeks: 50052
Word number: 50052
Elapsed time: 36.58 minutes
Words proccesed: 1800864
Word indeks: 50031
Word number: 50031
Elapsed time: 45.39 minutes
Words proccesed: 1850886
Word indeks: 50022
Word number: 50022
Elapsed time: 54.31 minutes
Words proccesed: 1900898
Word indeks: 50012
Word number: 50012
Elapsed time: 62.81 minutes
Words proccesed: 1950911
Word indeks: 50013
Word number: 50013
Elapsed time: 70.84 minutes
Words proccesed: 2000920
Word indeks: 50009
Word number: 50009
Elapsed time: 79.08 minutes
Words proccesed: 2050927
Word indeks: 50007
Word number: 50007
Elapsed time: 87.50 minutes
Words proccesed: 2100944
Word indeks: 50017
Word number: 50017
Elapsed time: 95.62 minutes
Words proccesed: 2150949
Word indeks: 50005
Word number: 50005
Elapsed time: 104.08 minutes
Words proccesed: 2200958
Word indeks: 50009
Word number: 50009
Elapsed time: 112.44 minutes
Words proccesed: 2250969
Word indeks: 50011
Word number: 50011
Elapsed time: 120.68 minutes
Words proccesed: 2300978
Word indeks: 50009
Word number: 50009
Elapsed time: 129.58 minutes
Words proccesed: 2350986
Word indeks: 50008
Word number: 50008
Elapsed time: 139.40 minutes
Words proccesed: 2400993
Word indeks: 50007
Word number: 50007
Elapsed time: 148.05 minutes
Words proccesed: 2451000
Word indeks: 50007
Word number: 50007
Elapsed time: 156.79 minutes
Words proccesed: 2501005
Word indeks: 50005
Word number: 50005
Elapsed time: 165.57 minutes
Words proccesed: 2551016
Word indeks: 50011
Word number: 50011
Elapsed time: 174.29 minutes
Words proccesed: 2601024
Word indeks: 50008
Word number: 50008
Elapsed time: 183.20 minutes
Words proccesed: 2651037
Word indeks: 50013
Word number: 50013
Elapsed time: 191.94 minutes
Words proccesed: 2701046
Word indeks: 50009
Word number: 50009
Elapsed time: 200.67 minutes
Words proccesed: 2751050
Word indeks: 50004
Word number: 50004
Elapsed time: 209.40 minutes
In [10]:
len(problem_words)
Out [10]:
50052
In [ ]:
%run prepare_data.py
data = Data('l', shuffle_all_inputs=False)
location_accented_words, accented_words = data.accentuate_word(problem_words[:], letter_location_model, syllable_location_model, syllabled_letters_location_model,
                                    letter_type_model, syllable_type_model, syllabled_letter_type_model,
                                    dictionary, max_word, max_num_vowels, vowels, accented_vowels, feature_dictionary, syllable_dictionary)
In [21]:
# CALCULATE INDEX NUMBER:
previous_file_len = [622061, 618306, 618266, 618483, 619342]
word_nums = [50017, 50007, 50017, 50012, 50024]
def calculate_index(previous_files_len, word_nums):
    return sum(previous_files_len) - 2 * sum(word_nums) + 11
calculate_index(previous_file_len[:3], word_nums[:3])
Out [21]:
1558562
In [12]:
print(word_glob_num)
print(word_limit)
print(iter_num)
print(word_index)
250002
250000
50000
0
In [22]:
max_word
Out [22]:
23
In [7]:
words[0]
Out [7]:
['zapirati', '', 'Ggnn', 'zapirati']
In [ ]:
with open("new_sloleks.xml", "ab") as myfile:
    for event, element in etree.iterparse('data/Sloleks_v1.2.xml', tag="LexicalEntry", encoding="UTF-8", remove_blank_text=True):
        # READ DATA
        for child in element:
            if child.tag == 'WordForm':
                msd = None
                word = None
                for wf in child:
#                     print(wf.attrib['att'])
                    if 'att' in wf.attrib and wf.attrib['att'] == 'msd':
                        msd = wf.attrib['val']
                    elif wf.tag == 'FormRepresentation':
                        for form_rep in wf:
                            if form_rep.attrib['att'] == 'zapis_oblike':
                                word = form_rep.attrib['val']

                        new_element = etree.Element('feat')
                        new_element.attrib['att']='naglasna_mesta_oblike'
                        new_element.attrib['val']='test'
                        wf.append(new_element)

                        new_element = etree.Element('feat')
                        new_element.attrib['att']='naglašena_oblika'
                        new_element.attrib['val']='test'
                        wf.append(new_element)
                if msd is not None and word is not None:
                    print(msd)
                    print(word)
                else:
                    print('NOOOOO')
        print(etree.tostring(element, encoding="UTF-8"))
        myfile.write(etree.tostring(element, encoding="UTF-8", pretty_print=True))
        element.clear()
        break