|
|
|
@ -3,6 +3,7 @@ import os
|
|
|
|
|
import shutil
|
|
|
|
|
import codecs
|
|
|
|
|
import re
|
|
|
|
|
import lxml.etree as lxml
|
|
|
|
|
|
|
|
|
|
import classla
|
|
|
|
|
from classla import Document
|
|
|
|
@ -24,6 +25,8 @@ OBELIKS_JAR_FILE_NAME = '../resources/obeliks.jar'
|
|
|
|
|
TRANSLATION_FILE_NAME = '../resources/dict.xml'
|
|
|
|
|
CLASSLA_MODELS_DIRECTORY = '../resources/classla'
|
|
|
|
|
STRUCTURE_CURRENT_FILE_NAME = '../resources/structures.xml'
|
|
|
|
|
STRUCTURE_SCHEMA_FILE_NAME = '../resources/structures.xsd'
|
|
|
|
|
DICTIONARY_SCHEMA_FILE_NAME = '../resources/monolingual_dictionaries.xsd'
|
|
|
|
|
|
|
|
|
|
STRING_LIST_FILE_NAME = TMP_DIRECTORY + '/strings.txt'
|
|
|
|
|
OBELIKS_RAW_FILE_NAME = TMP_DIRECTORY + '/obeliks_raw.conllu'
|
|
|
|
@ -57,9 +60,11 @@ def run_pipeline(input_file_name, output_file_name):
|
|
|
|
|
run_mwe_extraction(STRUCTURE_CURRENT_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_1_FILE_NAME)
|
|
|
|
|
run_structure_assignment(STRING_LIST_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_1_FILE_NAME, TEI_STRUCTURE_1_FILE_NAME)
|
|
|
|
|
run_structure_creation(STRUCTURE_CURRENT_FILE_NAME, TEI_STRUCTURE_1_FILE_NAME, STRUCTURE_NEW_FILE_NAME)
|
|
|
|
|
validate_structures(STRUCTURE_NEW_FILE_NAME)
|
|
|
|
|
run_mwe_extraction(STRUCTURE_NEW_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_2_FILE_NAME)
|
|
|
|
|
run_structure_assignment(STRING_LIST_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_2_FILE_NAME, TEI_STRUCTURE_2_FILE_NAME)
|
|
|
|
|
run_dictionary_conversion(TEI_STRUCTURE_2_FILE_NAME, DICTIONARY_XML_FILE_NAME)
|
|
|
|
|
validate_dictionary(DICTIONARY_XML_FILE_NAME)
|
|
|
|
|
shutil.copyfile(DICTIONARY_XML_FILE_NAME, output_file_name)
|
|
|
|
|
|
|
|
|
|
def run_obeliks4J(obeliks_file_name, classla_file_name):
|
|
|
|
@ -100,6 +105,12 @@ def run_mwe_extraction(structure_file_name, tei_file_name, mwe_csv_file_name):
|
|
|
|
|
print(extraction_command)
|
|
|
|
|
os.system(extraction_command)
|
|
|
|
|
|
|
|
|
|
def validate_structures(structure_file_name):
|
|
|
|
|
print('Validating updated structure specifications ...')
|
|
|
|
|
xml_schema = lxml.XMLSchema(lxml.parse(STRUCTURE_SCHEMA_FILE_NAME))
|
|
|
|
|
xml_tree = lxml.parse(structure_file_name)
|
|
|
|
|
xml_schema.assertValid(xml_tree)
|
|
|
|
|
|
|
|
|
|
def run_structure_assignment(input_file_name, tei_file_name, mwe_csv_file_name, output_file_name):
|
|
|
|
|
print('Assigning structure ids ...')
|
|
|
|
|
assignment_command = ' '.join(['python', STRUCTURE_ASSIGNMENT_SCRIPT_NAME, input_file_name, tei_file_name, mwe_csv_file_name, output_file_name])
|
|
|
|
@ -115,4 +126,10 @@ def run_dictionary_conversion(tei_file_name, xml_file_name):
|
|
|
|
|
convert_command = ' '.join(['python', TEI_DICTIONARY_SCRIPT_NAME, '-infile', tei_file_name, '-outfile', xml_file_name])
|
|
|
|
|
os.system(convert_command)
|
|
|
|
|
|
|
|
|
|
def validate_dictionary(dictionary_file_name):
|
|
|
|
|
print('Validating output dictionary file ...')
|
|
|
|
|
xml_schema = lxml.XMLSchema(lxml.parse(DICTIONARY_SCHEMA_FILE_NAME))
|
|
|
|
|
xml_tree = lxml.parse(dictionary_file_name)
|
|
|
|
|
xml_schema.assertValid(xml_tree)
|
|
|
|
|
|
|
|
|
|
run_pipeline(input_file_name, output_file_name)
|
|
|
|
|