You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
198 lines
7.5 KiB
198 lines
7.5 KiB
6 years ago
|
# Words proccesed: 650250
|
||
|
|
# Word indeks: 50023
|
||
|
|
# Word number: 50023
|
||
|
|
import re
|
||
|
|
|
||
|
|
from lxml import etree
|
||
|
|
import time
|
||
|
6 years ago
|
# from prepare_data import *
|
||
|
6 years ago
|
|
||
|
|
accented_vowels = ['ŕ', 'á', 'à', 'é', 'è', 'ê', 'í', 'ì', 'ó', 'ô', 'ò', 'ú', 'ù']
|
||
|
|
|
||
|
|
def stressed2unstressed(w):
|
||
|
|
w = w.replace('ŕ', 'r')
|
||
|
|
w = w.replace('á', 'a')
|
||
|
|
w = w.replace('à', 'a')
|
||
|
|
w = w.replace('é', 'e')
|
||
|
|
w = w.replace('è', 'e')
|
||
|
|
w = w.replace('ê', 'e')
|
||
|
|
w = w.replace('í', 'i')
|
||
|
|
w = w.replace('ì', 'i')
|
||
|
|
w = w.replace('ó', 'o')
|
||
|
|
w = w.replace('ô', 'o')
|
||
|
|
w = w.replace('ò', 'o')
|
||
|
|
w = w.replace('ú', 'u')
|
||
|
|
w = w.replace('ù', 'u')
|
||
|
|
|
||
|
|
return w
|
||
|
|
|
||
|
|
|
||
|
|
"""Works on finalized XML
|
||
|
|
"""
|
||
|
|
|
||
|
|
|
||
|
|
from text2SAMPA import *
|
||
|
|
|
||
|
|
# def xml_words_generator(xml_path):
|
||
|
|
# for event, element in etree.iterparse(xml_path, tag="LexicalEntry", encoding="UTF-8"):
|
||
|
|
# words = []
|
||
|
|
# for child in element:
|
||
|
|
# if child.tag == 'WordForm':
|
||
|
|
# msd = None
|
||
|
|
# word = None
|
||
|
|
# for wf in child:
|
||
|
|
# if 'att' in wf.attrib and wf.attrib['att'] == 'msd':
|
||
|
|
# msd = wf.attrib['val']
|
||
|
|
# elif wf.tag == 'FormRepresentation':
|
||
|
|
# for form_rep in wf:
|
||
|
|
# if form_rep.attrib['att'] == 'zapis_oblike':
|
||
|
|
# word = form_rep.attrib['val']
|
||
|
|
# #if msd is not None and word is not None:
|
||
|
|
# # pass
|
||
|
|
# #else:
|
||
|
|
# # print('NOOOOO')
|
||
|
|
# words.append([word, '', msd, word])
|
||
|
|
# yield words
|
||
|
|
#
|
||
|
|
#
|
||
|
|
# gen = xml_words_generator('data/Sloleks_v1.2_p2.xml')
|
||
|
|
word_glob_num = 0
|
||
|
|
word_limit = 50000
|
||
|
|
iter_num = 50000
|
||
|
|
word_index = 0
|
||
|
|
|
||
|
|
# iter_index = 0
|
||
|
|
# words = []
|
||
|
|
#
|
||
|
|
# lexical_entries_load_number = 0
|
||
|
|
# lexical_entries_save_number = 0
|
||
|
|
#
|
||
|
|
# # INSIDE
|
||
|
|
# # word_glob_num = 1500686
|
||
|
|
# word_glob_num = 1550705
|
||
|
|
#
|
||
|
|
# # word_limit = 1500686
|
||
|
|
# word_limit = 1550705
|
||
|
|
#
|
||
|
|
# iter_index = 31
|
||
|
|
|
||
|
|
# done_lexical_entries = 33522
|
||
|
6 years ago
|
# data = Data('s', shuffle_all_inputs=False)
|
||
|
6 years ago
|
# accentuated_content = data._read_content('data/new_sloleks/final_sloleks2.tab')
|
||
|
|
|
||
|
|
start_timer = time.time()
|
||
|
|
lemmas = 0
|
||
|
|
print('Copy initialization complete')
|
||
|
6 years ago
|
with open("data/contextual_changes/stressed_lemmas_sloleks2.xml", "ab") as myfile:
|
||
|
6 years ago
|
# myfile2 = open('data/new_sloleks/p' + str(iter_index) + '.xml', 'ab')
|
||
|
6 years ago
|
for event, element in etree.iterparse('data/contextual_changes/final_sloleks2_inhouse2S.xml', tag="LexicalEntry", encoding="UTF-8", remove_blank_text=True):
|
||
|
6 years ago
|
# for event, element in etree.iterparse('data/Sloleks_v1.2.xml', tag="LexicalEntry", encoding="UTF-8", remove_blank_text=True):
|
||
|
|
# if word_glob_num >= word_limit:
|
||
|
|
# myfile2.close()
|
||
|
|
# myfile2 = open('data/new_sloleks/p' + str(iter_index) + '.xml', 'ab')
|
||
|
|
# iter_index += 1
|
||
|
|
# print("Words proccesed: " + str(word_glob_num))
|
||
|
|
#
|
||
|
|
# print("Word indeks: " + str(word_index))
|
||
|
|
# print("Word number: " + str(len(words)))
|
||
|
|
#
|
||
|
|
# # print("lexical_entries_load_number: " + str(lexical_entries_load_number))
|
||
|
|
# # print("lexical_entries_save_number: " + str(lexical_entries_save_number))
|
||
|
|
#
|
||
|
|
# end_timer = time.time()
|
||
|
|
# print("Elapsed time: " + "{0:.2f}".format((end_timer - start_timer) / 60.0) + " minutes")
|
||
|
|
lemma = ''
|
||
|
|
stressed_lemma = ''
|
||
|
|
msd = ''
|
||
|
|
word_form_found = False
|
||
|
|
for child in element:
|
||
|
|
if child.tag == 'Lemma':
|
||
|
|
for wf in child:
|
||
|
|
if 'att' in wf.attrib and wf.attrib['att'] == 'zapis_oblike':
|
||
|
|
lemma = wf.attrib['val']
|
||
|
|
if child.tag == 'WordForm':
|
||
|
|
msd = None
|
||
|
|
word = None
|
||
|
|
for wf in child:
|
||
|
|
if 'att' in wf.attrib and wf.attrib['att'] == 'msd':
|
||
|
|
msd = wf.attrib['val']
|
||
|
|
elif wf.tag == 'FormRepresentation':
|
||
|
|
for form_rep in wf:
|
||
|
|
if form_rep.attrib['att'] == 'naglašena_beseda':
|
||
|
|
stressed_lemma = form_rep.attrib['val']
|
||
|
|
word_form_found = True
|
||
|
|
break
|
||
|
|
|
||
|
|
break
|
||
|
|
|
||
|
|
# new_element = etree.Element('feat')
|
||
|
|
# new_element.attrib['att'] = 'SAMPA'
|
||
|
|
#
|
||
|
|
# wf.append(new_element)
|
||
|
|
#
|
||
|
|
# word_glob_num += 1
|
||
|
|
# word_index += 1
|
||
|
|
break
|
||
|
|
|
||
|
|
if re.match(r'S..ei', msd) or re.match(r'S..mi', msd) or re.match(r'Sometn', msd) or re.match(r'P..mei.*', msd) \
|
||
|
|
or re.match(r'P..zei.*', msd) or re.match(r'P..sei.*', msd) or re.match(r'G..n.*', msd) \
|
||
|
|
or re.match(r'R.n', msd) or re.match(r'Rss', msd) or re.match(r'Rd', msd) \
|
||
|
|
or re.match(r'K.*', msd) or re.match(r'D.', msd) or re.match(r'L', msd) or re.match(r'M', msd) \
|
||
|
|
or re.match(r'O', msd) or re.match(r'Z.*', msd) or re.match(r'V.', msd) or re.match(r'Rsr.', msd)\
|
||
|
|
or msd == "":
|
||
|
6 years ago
|
|
||
|
|
# when lemma does not equal unstressed version of what is supposed to be lemma, try to find parts of the
|
||
|
|
# word that are equal and transfer stress to lemma (if possible)
|
||
|
6 years ago
|
if lemma != stressed2unstressed(stressed_lemma):
|
||
|
6 years ago
|
identical_length = 0
|
||
|
|
# if lemma == 'Latkov':
|
||
|
|
# print('HERE')
|
||
|
|
for i in range(min(len(lemma), len(stressed2unstressed(stressed_lemma)))):
|
||
|
|
# a = list(lemma)
|
||
|
|
# b = list(stressed2unstressed(stressed_lemma))
|
||
|
|
identical_length += 1
|
||
|
|
if list(lemma)[i] != list(stressed2unstressed(stressed_lemma))[i]:
|
||
|
|
break
|
||
|
|
|
||
|
|
|
||
|
|
for l in list(stressed_lemma[identical_length:]):
|
||
|
|
if l in accented_vowels:
|
||
|
|
# print(lemma)
|
||
|
|
# print(stressed2unstressed(stressed_lemma))
|
||
|
|
# print(stressed_lemma[identical_length:])
|
||
|
|
print(lemma + " : " + stressed_lemma + " - " + msd)
|
||
|
|
stressed_lemma = stressed_lemma[:identical_length] + lemma[identical_length:]
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
# pass
|
||
|
|
# if lemma != stressed2unstressed(stressed_lemma):
|
||
|
|
# print(lemma + " : " + stressed_lemma + " - " + msd)
|
||
|
6 years ago
|
else:
|
||
|
|
# print("Error2 - " + msd + " " + lemma + " - " + stressed_lemma)
|
||
|
|
# print(lemma + " - " + msd)
|
||
|
|
pass
|
||
|
|
|
||
|
|
for child in element:
|
||
|
|
if child.tag == 'Lemma':
|
||
|
|
for wf in child:
|
||
|
|
if 'att' in wf.attrib and wf.attrib['att'] == 'zapis_oblike':
|
||
|
|
wf.attrib['val'] = stressed_lemma
|
||
|
|
break
|
||
|
|
else:
|
||
|
|
print('Error1')
|
||
|
|
break
|
||
|
|
|
||
|
|
|
||
|
|
lemmas += 1
|
||
|
|
# print(etree.tostring(element, encoding="UTF-8"))
|
||
|
|
# myfile2.write(etree.tostring(element, encoding="UTF-8", pretty_print=True))
|
||
|
|
if word_glob_num > word_limit:
|
||
|
|
# print('Proccessed ' + str(word_glob_num) + ' words')
|
||
|
|
end_timer = time.time()
|
||
|
|
# print("Elapsed time: " + "{0:.2f}".format((end_timer - start_timer) / 60.0) + " minutes")
|
||
|
|
word_limit += iter_num
|
||
|
|
myfile.write(etree.tostring(element, encoding="UTF-8", pretty_print=True))
|
||
|
|
element.clear()
|
||
|
|
|
||
|
|
print(lemmas)
|