From d6ded65f67d67194e4cc65cba1e331bc38df30be Mon Sep 17 00:00:00 2001 From: Cyprian Laskowski Date: Wed, 30 Sep 2020 16:38:42 +0200 Subject: [PATCH] IssueID #1487: expanded/configured pipeline --- scripts/pipeline.py | 79 +++++++++++++++++++++++++++------------------ 1 file changed, 48 insertions(+), 31 deletions(-) diff --git a/scripts/pipeline.py b/scripts/pipeline.py index 8d2cec5..aa91419 100644 --- a/scripts/pipeline.py +++ b/scripts/pipeline.py @@ -12,20 +12,27 @@ input_file_name = sys.argv[1] output_file_name = sys.argv[2] TMP_DIRECTORY = '../tmp/structure_assignment' -STRING_LIST_FILE_NAME = TMP_DIRECTORY + '/strings.txt' + +CONLLU_TEI_SCRIPT_NAME = 'conllu_to_xml.py' +MWE_EXTRACTION_SCRIPT_NAME = 'wani.py' +STRUCTURE_ASSIGNMENT_SCRIPT_NAME = 'assign_structures.py' +STRUCTURE_CREATION_SCRIPT_NAME = 'create_structures.py' + OBELIKS_JAR_FILE_NAME = '../resources/obeliks.jar' +TRANSLATION_FILE_NAME = '../resources/dict.xml' +CLASSLA_MODELS_DIRECTORY = '../resources/classla' +STRUCTURE_CURRENT_FILE_NAME = '../resources/structures.xml' + +STRING_LIST_FILE_NAME = TMP_DIRECTORY + '/strings.txt' OBELIKS_RAW_FILE_NAME = TMP_DIRECTORY + '/obeliks_raw.conllu' OBELIKS_TWEAKED_FILE_NAME = TMP_DIRECTORY + '/obeliks_tweaked.conllu' -CLASSLA_MODELS_DIRECTORY = '../resources/classla' CLASSLA_FILE_NAME = TMP_DIRECTORY + '/classla.conllu' -CONLLU_TEI_SCRIPT_NAME = 'conllu_to_xml.py' -TEI_FILE_NAME = TMP_DIRECTORY + '/tei.xml' -TRANSLATION_FILE_NAME = '../resources/dict.xml' -MWE_EXTRACTION_SCRIPT_NAME = 'wani.py' -STRUCTURE_FILE_NAME = '../resources/structures.xml' -MWE_CSV_FILE_NAME = TMP_DIRECTORY + '/mwes.csv' -STRUCTURE_ASSIGNMENT_SCRIPT_NAME = 'assign_structures.py' -STRING_STRUCTURE_FILE_NAME = TMP_DIRECTORY + '/strings_with_structures.csv' +TEI_INIT_FILE_NAME = TMP_DIRECTORY + '/tei_initial.xml' +TEI_STRUCTURE_FILE_NAME = TMP_DIRECTORY + '/tei_with_structure_ids.xml' +MWE_CSV_1_FILE_NAME = TMP_DIRECTORY + '/mwes1.csv' +MWE_CSV_2_FILE_NAME = TMP_DIRECTORY + '/mwes2.csv' +STRUCTURE_NEW_FILE_NAME = TMP_DIRECTORY + '/structures_new.xml' +TEI_STRUCTURE_FILE_NAME = TMP_DIRECTORY + '/tei_with_structure_ids.xml' NLP_CONFIG_MAP = { 'treebank': 'sl_ssj_jos', @@ -40,23 +47,27 @@ def run_pipeline(input_file_name, output_file_name): shutil.rmtree(TMP_DIRECTORY, True) os.makedirs(TMP_DIRECTORY, exist_ok=True) shutil.copyfile(input_file_name, STRING_LIST_FILE_NAME) - run_obeliks4J() - fix_xml_ids() - run_classla() - convert_to_tei() - run_mwe_extraction() - run_structure_assignment() - shutil.copyfile(STRING_STRUCTURE_FILE_NAME, output_file_name) + run_obeliks4J(STRING_LIST_FILE_NAME, OBELIKS_RAW_FILE_NAME) + fix_xml_ids(OBELIKS_RAW_FILE_NAME, OBELIKS_TWEAKED_FILE_NAME) + run_classla(OBELIKS_TWEAKED_FILE_NAME, CLASSLA_FILE_NAME) + convert_to_tei(CLASSLA_FILE_NAME, TEI_INIT_FILE_NAME) + run_mwe_extraction(STRUCTURE_CURRENT_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_1_FILE_NAME) + run_structure_assignment(STRING_LIST_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_1_FILE_NAME, TEI_STRUCTURE_FILE_NAME) + run_structure_creation(STRUCTURE_CURRENT_FILE_NAME, TEI_STRUCTURE_FILE_NAME, STRUCTURE_NEW_FILE_NAME) + run_mwe_extraction(STRUCTURE_NEW_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_2_FILE_NAME) + run_structure_assignment(STRING_LIST_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_2_FILE_NAME, TEI_STRUCTURE_FILE_NAME) + # run_format_lexicon() # TODO: implement + shutil.copyfile(TEI_STRUCTURE_FILE_NAME, output_file_name) -def run_obeliks4J(): +def run_obeliks4J(obeliks_file_name, classla_file_name): print('Running obeliks ...') obeliks_command = 'java -jar ' + OBELIKS_JAR_FILE_NAME + ' -d -if ' + STRING_LIST_FILE_NAME + ' -o ' + OBELIKS_RAW_FILE_NAME os.system(obeliks_command) -def fix_xml_ids(): +def fix_xml_ids(input_file_name, output_file_name): print('Fixing xml ids ...') - output_file = codecs.open(OBELIKS_TWEAKED_FILE_NAME, 'w') - input_file = codecs.open(OBELIKS_RAW_FILE_NAME, 'r') + output_file = codecs.open(output_file_name, 'w') + input_file = codecs.open(input_file_name, 'r') regexp = r'^(# sent_id = )(\d+\.\d+)$' for line in input_file: match = re.search(regexp, line) @@ -66,28 +77,34 @@ def fix_xml_ids(): input_file.close() output_file.close() -def run_classla(): +def run_classla(obeliks_file_name, classla_file_name): print('Running classla ...') doc = Document(text=None) - conll_file = CoNLLFile(filename=OBELIKS_TWEAKED_FILE_NAME) + conll_file = CoNLLFile(filename=obeliks_file_name) doc.conll_file = conll_file nlp = classla.Pipeline('sl', **NLP_CONFIG_MAP) result = nlp(doc) - result.conll_file.write_conll(CLASSLA_FILE_NAME) + result.conll_file.write_conll(classla_file_name) -def convert_to_tei(): +def convert_to_tei(classla_file_name, tei_file_name): print('Converting to tei ...') - convert_command = ' '.join(['python', CONLLU_TEI_SCRIPT_NAME, CLASSLA_FILE_NAME, TEI_FILE_NAME, '--translate', TRANSLATION_FILE_NAME]) + convert_command = ' '.join(['python', CONLLU_TEI_SCRIPT_NAME, classla_file_name, tei_file_name, '--translate', TRANSLATION_FILE_NAME]) os.system(convert_command) -def run_mwe_extraction(): +def run_mwe_extraction(structure_file_name, tei_file_name, mwe_csv_file_name): print('Extracting MWEs from tei ...') - extraction_command = ' '.join(['python', MWE_EXTRACTION_SCRIPT_NAME, STRUCTURE_FILE_NAME, TEI_FILE_NAME, '--all', MWE_CSV_FILE_NAME, '--skip-id-check']) + extraction_command = ' '.join(['python', MWE_EXTRACTION_SCRIPT_NAME, structure_file_name, tei_file_name, '--all', mwe_csv_file_name, '--skip-id-check']) + print(extraction_command) os.system(extraction_command) -def run_structure_assignment(): - print('Assign structure ids ...') - assignment_command = ' '.join(['python', STRUCTURE_ASSIGNMENT_SCRIPT_NAME, STRING_LIST_FILE_NAME, TEI_FILE_NAME, MWE_CSV_FILE_NAME, STRING_STRUCTURE_FILE_NAME]) +def run_structure_assignment(input_file_name, tei_file_name, mwe_csv_file_name, output_file_name): + print('Assigning structure ids ...') + assignment_command = ' '.join(['python', STRUCTURE_ASSIGNMENT_SCRIPT_NAME, input_file_name, tei_file_name, mwe_csv_file_name, output_file_name]) + os.system(assignment_command) + +def run_structure_creation(input_file_name, tei_file_name, output_file_name): + print('Creating missing structures ...') + assignment_command = ' '.join(['python', STRUCTURE_CREATION_SCRIPT_NAME, '-infile', input_file_name, '-tei', tei_file_name, '-outfile', output_file_name]) os.system(assignment_command) run_pipeline(input_file_name, output_file_name)