Accentuation on sloleks
This commit is contained in:
+135
-19
@@ -87,18 +87,6 @@
|
||||
" i += 1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 98,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"feature__en_dictionary = data._create_feature_dictionary()\n",
|
||||
"feature__slo_dictionary = data._create_slovene_feature_dictionary()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
@@ -120,7 +108,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": 4,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
@@ -131,14 +119,14 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"execution_count": 5,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%run prepare_data.py\n",
|
||||
"data = Data('l', shuffle_all_inputs=False)\n",
|
||||
"data = Data('s', shuffle_all_inputs=False)\n",
|
||||
"location_accented_words, accented_words = data.accentuate_word(test_input, letter_location_model, syllable_location_model, syllabled_letters_location_model,\n",
|
||||
" letter_type_model, syllable_type_model, syllabled_letter_type_model,\n",
|
||||
" dictionary, max_word, max_num_vowels, vowels, accented_vowels, feature_dictionary, syllable_dictionary)"
|
||||
@@ -146,15 +134,15 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 19,
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"['uradní', 'podatkí', 'policíje', 'kažéjo', 'ná', 'precéj', 'napeté', 'razmeré', 'v', 'piranském', 'zalivú', 'jé', 'danés', 'poročála', 'oddajá', 'dó', 'danés', 'sé', 'jé', 'zgodílo']\n",
|
||||
"['uradnî', 'podatkî', 'policíje', 'kažëjo', 'ná', 'precëj', 'napetë', 'razmerë', 'v', 'piranskëm', 'zalivú', 'jë', 'danës', 'poročála', 'oddajá', 'dó', 'danës', 'së', 'jë', 'zgodílo']\n"
|
||||
"['uradni', 'podatkí', 'policíje', 'kažéjo', 'ná', 'precéj', 'napeté', 'razmeré', 'v', 'piranském', 'zalivú', 'jé', 'danés', 'poročála', 'oddajá', 'dó', 'danés', 'sé', 'jé', 'zgodílo']\n",
|
||||
"['uradni', 'pödatki', 'polícije', 'kažëjo', 'ná', 'prëcej', 'nápete', 'räzmere', 'v', 'pîranskem', 'zálivu', 'jë', 'dánes', 'poróčala', 'öddaja', 'dó', 'dánes', 'së', 'jë', 'zgodílo']\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -225,7 +213,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"execution_count": 9,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
@@ -508,6 +496,134 @@
|
||||
" "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "NameError",
|
||||
"evalue": "name 'xml_words_generator' is not defined",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
|
||||
"\u001b[0;32m<ipython-input-8-44b0367c6cbf>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mgen\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mxml_words_generator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'data/Sloleks_v1.2_p2.xml'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 9\u001b[0m \u001b[0mword_glob_num\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0mword_limit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;31mNameError\u001b[0m: name 'xml_words_generator' is not defined"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"#Words proccesed: 650250\n",
|
||||
"#Word indeks: 50023\n",
|
||||
"#Word number: 50023\n",
|
||||
"\n",
|
||||
"from lxml import etree\n",
|
||||
"import time\n",
|
||||
"\n",
|
||||
"gen = xml_words_generator('data/Sloleks_v1.2_p2.xml')\n",
|
||||
"word_glob_num = 0\n",
|
||||
"word_limit = 0\n",
|
||||
"iter_num = 50000\n",
|
||||
"word_index = 0\n",
|
||||
"start_timer = time.time()\n",
|
||||
"iter_index = 0\n",
|
||||
"words = []\n",
|
||||
"\n",
|
||||
"lexical_entries_load_number = 0\n",
|
||||
"lexical_entries_save_number = 0\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# INSIDE\n",
|
||||
"#word_glob_num = 1500686\n",
|
||||
"word_glob_num = 1550705\n",
|
||||
"\n",
|
||||
"#word_limit = 1500686\n",
|
||||
"word_limit = 1550705\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"iter_index = 31\n",
|
||||
"\n",
|
||||
"#done_lexical_entries = 33522\n",
|
||||
"\n",
|
||||
"with open(\"data/new_sloleks/new_sloleks.xml\", \"ab\") as myfile:\n",
|
||||
" myfile2 = open('data/new_sloleks/pa' + str(iter_index) + '.xml', 'ab')\n",
|
||||
" for event, element in etree.iterparse('data/Sloleks_v1.2_p2.xml', tag=\"LexicalEntry\", encoding=\"UTF-8\", remove_blank_text=True):\n",
|
||||
" # LOAD NEW WORDS AND ACCENTUATE THEM\n",
|
||||
" #print(\"HERE\")\n",
|
||||
" \n",
|
||||
"# if lexical_entries_save_number < done_lexical_entries:\n",
|
||||
"# next(gen)\n",
|
||||
"# #print(lexical_entries_save_number)\n",
|
||||
"# lexical_entries_save_number += 1\n",
|
||||
"# lexical_entries_load_number += 1\n",
|
||||
"# continue\n",
|
||||
" \n",
|
||||
" if word_glob_num >= word_limit:\n",
|
||||
" myfile2.close()\n",
|
||||
" myfile2 = open('data/new_sloleks/p' + str(iter_index) + '.xml', 'ab')\n",
|
||||
" iter_index += 1\n",
|
||||
" print(\"Words proccesed: \" + str(word_glob_num))\n",
|
||||
"\n",
|
||||
" print(\"Word indeks: \" + str(word_index))\n",
|
||||
" print(\"Word number: \" + str(len(words)))\n",
|
||||
" \n",
|
||||
" #print(\"lexical_entries_load_number: \" + str(lexical_entries_load_number))\n",
|
||||
" #print(\"lexical_entries_save_number: \" + str(lexical_entries_save_number))\n",
|
||||
"\n",
|
||||
" end_timer = time.time()\n",
|
||||
" print(\"Elapsed time: \" + \"{0:.2f}\".format((end_timer - start_timer)/60.0) + \" minutes\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" word_index = 0\n",
|
||||
" words = []\n",
|
||||
"\n",
|
||||
" while len(words) < iter_num:\n",
|
||||
" try:\n",
|
||||
" words.extend(next(gen))\n",
|
||||
" lexical_entries_load_number += 1\n",
|
||||
" except:\n",
|
||||
" break\n",
|
||||
" #if word_glob_num > 1:\n",
|
||||
" # break\n",
|
||||
"\n",
|
||||
" #problem_words = words\n",
|
||||
" #break\n",
|
||||
" data = Data('l', shuffle_all_inputs=False)\n",
|
||||
" location_accented_words, accented_words = data.accentuate_word(words, letter_location_model, syllable_location_model, syllabled_letters_location_model,\n",
|
||||
" letter_type_model, syllable_type_model, syllabled_letter_type_model,\n",
|
||||
" dictionary, max_word, max_num_vowels, vowels, accented_vowels, feature_dictionary, syllable_dictionary)\n",
|
||||
"\n",
|
||||
" word_limit += len(words)\n",
|
||||
" \n",
|
||||
" \n",
|
||||
" # READ DATA\n",
|
||||
" for child in element:\n",
|
||||
" if child.tag == 'WordForm':\n",
|
||||
" msd = None\n",
|
||||
" word = None\n",
|
||||
" for wf in child:\n",
|
||||
" if wf.tag == 'FormRepresentation':\n",
|
||||
" new_element = etree.Element('feat')\n",
|
||||
" new_element.attrib['att']='naglasna_mesta_oblike'\n",
|
||||
" new_element.attrib['val']=location_accented_words[word_index]\n",
|
||||
" wf.append(new_element)\n",
|
||||
"\n",
|
||||
" new_element = etree.Element('feat')\n",
|
||||
" new_element.attrib['att']='naglašena_oblika'\n",
|
||||
" new_element.attrib['val']=accented_words[word_index]\n",
|
||||
" wf.append(new_element)\n",
|
||||
" word_glob_num += 1\n",
|
||||
" word_index += 1\n",
|
||||
"\n",
|
||||
" # print(etree.tostring(element, encoding=\"UTF-8\"))\n",
|
||||
" myfile2.write(etree.tostring(element, encoding=\"UTF-8\", pretty_print=True))\n",
|
||||
" myfile.write(etree.tostring(element, encoding=\"UTF-8\", pretty_print=True))\n",
|
||||
" element.clear()\n",
|
||||
" lexical_entries_save_number += 1\n",
|
||||
" "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
|
||||
Reference in New Issue
Block a user