From 51ed244a5c5a938565b0d50ed00a7773262e2bb1 Mon Sep 17 00:00:00 2001 From: Cyprian Laskowski Date: Wed, 4 Nov 2020 19:02:15 +0100 Subject: [PATCH] IssueID #1487: split structure pipeline into two --- scripts/.gitignore | 1 + scripts/constants.py | 31 ++++++++++ scripts/pipeline.py | 135 ------------------------------------------- scripts/pipeline1.py | 71 +++++++++++++++++++++++ scripts/pipeline2.py | 68 ++++++++++++++++++++++ 5 files changed, 171 insertions(+), 135 deletions(-) create mode 100644 scripts/constants.py delete mode 100644 scripts/pipeline.py create mode 100644 scripts/pipeline1.py create mode 100644 scripts/pipeline2.py diff --git a/scripts/.gitignore b/scripts/.gitignore index ecd1440..52e9bf3 100644 --- a/scripts/.gitignore +++ b/scripts/.gitignore @@ -1,3 +1,4 @@ +*.pyc /assign_structures.py /conllu_to_xml.py /wani.py diff --git a/scripts/constants.py b/scripts/constants.py new file mode 100644 index 0000000..62c4b11 --- /dev/null +++ b/scripts/constants.py @@ -0,0 +1,31 @@ +# temporary directory +TMP_DIRECTORY = '../tmp/structure_assignment' + +# scripts +CONLLU_TEI_SCRIPT_NAME = 'conllu_to_xml.py' +MWE_EXTRACTION_SCRIPT_NAME = 'wani.py' +STRUCTURE_ASSIGNMENT_SCRIPT_NAME = 'assign_structures.py' +STRUCTURE_CREATION_SCRIPT_NAME = 'create_structures.py' +STRUCTURE_CREATION_SCRIPT_NAME = 'create_structures.py' +TEI_DICTIONARY_SCRIPT_NAME = 'tei_to_dictionary.py' + +# resources +OBELIKS_JAR_FILE_NAME = '../resources/obeliks.jar' +TRANSLATION_FILE_NAME = '../resources/dict.xml' +CLASSLA_MODELS_DIRECTORY = '../resources/classla' +STRUCTURE_CURRENT_FILE_NAME = '../resources/structures.xml' +STRUCTURE_SCHEMA_FILE_NAME = '../resources/structures.xsd' +DICTIONARY_SCHEMA_FILE_NAME = '../resources/monolingual_dictionaries.xsd' + +# temporary outputs +STRING_LIST_FILE_NAME = TMP_DIRECTORY + '/strings.txt' +OBELIKS_RAW_FILE_NAME = TMP_DIRECTORY + '/obeliks_raw.conllu' +OBELIKS_TWEAKED_FILE_NAME = TMP_DIRECTORY + '/obeliks_tweaked.conllu' +CLASSLA_FILE_NAME = TMP_DIRECTORY + '/classla.conllu' +TEI_INIT_FILE_NAME = TMP_DIRECTORY + '/tei_initial.xml' +TEI_STRUCTURE_1_FILE_NAME = TMP_DIRECTORY + '/tei_with_structure_ids1.xml' +TEI_STRUCTURE_2_FILE_NAME = TMP_DIRECTORY + '/tei_with_structure_ids2.xml' +MWE_CSV_1_FILE_NAME = TMP_DIRECTORY + '/mwes1.csv' +MWE_CSV_2_FILE_NAME = TMP_DIRECTORY + '/mwes2.csv' +STRUCTURE_NEW_FILE_NAME = TMP_DIRECTORY + '/structures_new.xml' +DICTIONARY_XML_FILE_NAME = TMP_DIRECTORY + '/dictionary.xml' diff --git a/scripts/pipeline.py b/scripts/pipeline.py deleted file mode 100644 index 528668d..0000000 --- a/scripts/pipeline.py +++ /dev/null @@ -1,135 +0,0 @@ -import sys -import os -import shutil -import codecs -import re -import lxml.etree as lxml - -import classla -from classla import Document -from classla.models.common.conll import CoNLLFile - -input_file_name = sys.argv[1] -output_file_name = sys.argv[2] - -TMP_DIRECTORY = '../tmp/structure_assignment' - -CONLLU_TEI_SCRIPT_NAME = 'conllu_to_xml.py' -MWE_EXTRACTION_SCRIPT_NAME = 'wani.py' -STRUCTURE_ASSIGNMENT_SCRIPT_NAME = 'assign_structures.py' -STRUCTURE_CREATION_SCRIPT_NAME = 'create_structures.py' -STRUCTURE_CREATION_SCRIPT_NAME = 'create_structures.py' -TEI_DICTIONARY_SCRIPT_NAME = 'tei_to_dictionary.py' - -OBELIKS_JAR_FILE_NAME = '../resources/obeliks.jar' -TRANSLATION_FILE_NAME = '../resources/dict.xml' -CLASSLA_MODELS_DIRECTORY = '../resources/classla' -STRUCTURE_CURRENT_FILE_NAME = '../resources/structures.xml' -STRUCTURE_SCHEMA_FILE_NAME = '../resources/structures.xsd' -DICTIONARY_SCHEMA_FILE_NAME = '../resources/monolingual_dictionaries.xsd' - -STRING_LIST_FILE_NAME = TMP_DIRECTORY + '/strings.txt' -OBELIKS_RAW_FILE_NAME = TMP_DIRECTORY + '/obeliks_raw.conllu' -OBELIKS_TWEAKED_FILE_NAME = TMP_DIRECTORY + '/obeliks_tweaked.conllu' -CLASSLA_FILE_NAME = TMP_DIRECTORY + '/classla.conllu' -TEI_INIT_FILE_NAME = TMP_DIRECTORY + '/tei_initial.xml' -TEI_STRUCTURE_1_FILE_NAME = TMP_DIRECTORY + '/tei_with_structure_ids1.xml' -TEI_STRUCTURE_2_FILE_NAME = TMP_DIRECTORY + '/tei_with_structure_ids2.xml' -MWE_CSV_1_FILE_NAME = TMP_DIRECTORY + '/mwes1.csv' -MWE_CSV_2_FILE_NAME = TMP_DIRECTORY + '/mwes2.csv' -STRUCTURE_NEW_FILE_NAME = TMP_DIRECTORY + '/structures_new.xml' -DICTIONARY_XML_FILE_NAME = TMP_DIRECTORY + '/dictionary.xml' - -NLP_CONFIG_MAP = { - 'treebank': 'sl_ssj_jos', - 'processors': 'tokenize,pos,lemma,depparse', - 'tokenize_pretokenized': True, - 'models_dir': CLASSLA_MODELS_DIRECTORY -} - -XML_ID_PREFIX = 's' - -def run_pipeline(input_file_name, output_file_name): - shutil.rmtree(TMP_DIRECTORY, True) - os.makedirs(TMP_DIRECTORY, exist_ok=True) - shutil.copyfile(input_file_name, STRING_LIST_FILE_NAME) - run_obeliks4J(STRING_LIST_FILE_NAME, OBELIKS_RAW_FILE_NAME) - fix_xml_ids(OBELIKS_RAW_FILE_NAME, OBELIKS_TWEAKED_FILE_NAME) - run_classla(OBELIKS_TWEAKED_FILE_NAME, CLASSLA_FILE_NAME) - run_tei_conversion(CLASSLA_FILE_NAME, TEI_INIT_FILE_NAME) - run_mwe_extraction(STRUCTURE_CURRENT_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_1_FILE_NAME) - run_structure_assignment(STRING_LIST_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_1_FILE_NAME, TEI_STRUCTURE_1_FILE_NAME) - run_structure_creation(STRUCTURE_CURRENT_FILE_NAME, TEI_STRUCTURE_1_FILE_NAME, STRUCTURE_NEW_FILE_NAME) - validate_structures(STRUCTURE_NEW_FILE_NAME) - run_mwe_extraction(STRUCTURE_NEW_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_2_FILE_NAME) - run_structure_assignment(STRING_LIST_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_2_FILE_NAME, TEI_STRUCTURE_2_FILE_NAME) - run_dictionary_conversion(TEI_STRUCTURE_2_FILE_NAME, DICTIONARY_XML_FILE_NAME) - validate_dictionary(DICTIONARY_XML_FILE_NAME) - shutil.copyfile(DICTIONARY_XML_FILE_NAME, output_file_name) - -def run_obeliks4J(obeliks_file_name, classla_file_name): - print('Running obeliks ...') - obeliks_command = 'java -jar ' + OBELIKS_JAR_FILE_NAME + ' -d -if ' + STRING_LIST_FILE_NAME + ' -o ' + OBELIKS_RAW_FILE_NAME - os.system(obeliks_command) - -def fix_xml_ids(input_file_name, output_file_name): - print('Fixing xml ids ...') - output_file = codecs.open(output_file_name, 'w') - input_file = codecs.open(input_file_name, 'r') - regexp = r'^(# sent_id = )(\d+\.\d+)$' - for line in input_file: - match = re.search(regexp, line) - if (match): - line = match.group(1) + XML_ID_PREFIX + match.group(2) + '\n' - output_file.write(line) - input_file.close() - output_file.close() - -def run_classla(obeliks_file_name, classla_file_name): - print('Running classla ...') - doc = Document(text=None) - conll_file = CoNLLFile(filename=obeliks_file_name) - doc.conll_file = conll_file - nlp = classla.Pipeline('sl', **NLP_CONFIG_MAP) - result = nlp(doc) - result.conll_file.write_conll(classla_file_name) - -def run_tei_conversion(classla_file_name, tei_file_name): - print('Converting to tei ...') - convert_command = ' '.join(['python', CONLLU_TEI_SCRIPT_NAME, classla_file_name, tei_file_name, '--translate', TRANSLATION_FILE_NAME]) - os.system(convert_command) - -def run_mwe_extraction(structure_file_name, tei_file_name, mwe_csv_file_name): - print('Extracting MWEs from tei ...') - extraction_command = ' '.join(['python', MWE_EXTRACTION_SCRIPT_NAME, structure_file_name, tei_file_name, '--all', mwe_csv_file_name, '--skip-id-check', '--fixed-restriction-order']) - print(extraction_command) - os.system(extraction_command) - -def validate_structures(structure_file_name): - print('Validating updated structure specifications ...') - xml_schema = lxml.XMLSchema(lxml.parse(STRUCTURE_SCHEMA_FILE_NAME)) - xml_tree = lxml.parse(structure_file_name) - xml_schema.assertValid(xml_tree) - -def run_structure_assignment(input_file_name, tei_file_name, mwe_csv_file_name, output_file_name): - print('Assigning structure ids ...') - assignment_command = ' '.join(['python', STRUCTURE_ASSIGNMENT_SCRIPT_NAME, input_file_name, tei_file_name, mwe_csv_file_name, output_file_name]) - os.system(assignment_command) - -def run_structure_creation(input_file_name, tei_file_name, output_file_name): - print('Creating missing structures ...') - creation_command = ' '.join(['python', STRUCTURE_CREATION_SCRIPT_NAME, '-infile', input_file_name, '-tei', tei_file_name, '-outfile', output_file_name]) - os.system(creation_command) - -def run_dictionary_conversion(tei_file_name, xml_file_name): - print('Converting to dictionary xml format ...') - convert_command = ' '.join(['python', TEI_DICTIONARY_SCRIPT_NAME, '-infile', tei_file_name, '-outfile', xml_file_name]) - os.system(convert_command) - -def validate_dictionary(dictionary_file_name): - print('Validating output dictionary file ...') - xml_schema = lxml.XMLSchema(lxml.parse(DICTIONARY_SCHEMA_FILE_NAME)) - xml_tree = lxml.parse(dictionary_file_name) - xml_schema.assertValid(xml_tree) - -run_pipeline(input_file_name, output_file_name) diff --git a/scripts/pipeline1.py b/scripts/pipeline1.py new file mode 100644 index 0000000..0d63f26 --- /dev/null +++ b/scripts/pipeline1.py @@ -0,0 +1,71 @@ +import argparse +import os +import shutil +import codecs +import re + +import classla +from classla import Document +from classla.models.common.conll import CoNLLFile + +from constants import * + +arg_parser = argparse.ArgumentParser(description='Parse Slovene strings and convert to TEI.') +arg_parser.add_argument('-inlist', type=str, help='Input list file') +arg_parser.add_argument('-outtei', type=str, help='Output TEI file') +arguments = arg_parser.parse_args() +input_file_name = arguments.infile +output_file_name = arguments.outfile + +NLP_CONFIG_MAP = { + 'treebank': 'sl_ssj_jos', + 'processors': 'tokenize,pos,lemma,depparse', + 'tokenize_pretokenized': True, + 'models_dir': CLASSLA_MODELS_DIRECTORY +} + +XML_ID_PREFIX = 's' + +def run_pipeline(input_file_name, output_file_name): + shutil.rmtree(TMP_DIRECTORY, True) + os.makedirs(TMP_DIRECTORY, exist_ok=True) + shutil.copyfile(input_file_name, STRING_LIST_FILE_NAME) + run_obeliks4J(STRING_LIST_FILE_NAME, OBELIKS_RAW_FILE_NAME) + fix_xml_ids(OBELIKS_RAW_FILE_NAME, OBELIKS_TWEAKED_FILE_NAME) + run_classla(OBELIKS_TWEAKED_FILE_NAME, CLASSLA_FILE_NAME) + run_tei_conversion(CLASSLA_FILE_NAME, TEI_INIT_FILE_NAME) + shutil.copyfile(TEI_INIT_FILE_NAME, output_file_name) + +def run_obeliks4J(obeliks_file_name, classla_file_name): + print('Running obeliks ...') + obeliks_command = 'java -jar ' + OBELIKS_JAR_FILE_NAME + ' -d -if ' + STRING_LIST_FILE_NAME + ' -o ' + OBELIKS_RAW_FILE_NAME + os.system(obeliks_command) + +def fix_xml_ids(input_file_name, output_file_name): + print('Fixing xml ids ...') + output_file = codecs.open(output_file_name, 'w') + input_file = codecs.open(input_file_name, 'r') + regexp = r'^(# sent_id = )(\d+\.\d+)$' + for line in input_file: + match = re.search(regexp, line) + if (match): + line = match.group(1) + XML_ID_PREFIX + match.group(2) + '\n' + output_file.write(line) + input_file.close() + output_file.close() + +def run_classla(obeliks_file_name, classla_file_name): + print('Running classla ...') + doc = Document(text=None) + conll_file = CoNLLFile(filename=obeliks_file_name) + doc.conll_file = conll_file + nlp = classla.Pipeline('sl', **NLP_CONFIG_MAP) + result = nlp(doc) + result.conll_file.write_conll(classla_file_name) + +def run_tei_conversion(classla_file_name, tei_file_name): + print('Converting to tei ...') + convert_command = ' '.join(['python', CONLLU_TEI_SCRIPT_NAME, classla_file_name, tei_file_name, '--translate', TRANSLATION_FILE_NAME]) + os.system(convert_command) + +run_pipeline(input_file_name, output_file_name) diff --git a/scripts/pipeline2.py b/scripts/pipeline2.py new file mode 100644 index 0000000..e8187dc --- /dev/null +++ b/scripts/pipeline2.py @@ -0,0 +1,68 @@ +import argparse +import os +import shutil +import lxml.etree as lxml + +from constants import * + +arg_parser = argparse.ArgumentParser(description='Assign parsed Slovene strings to structures and generate lexicon.') +arg_parser.add_argument('-inlist', type=str, help='Original input list file') +arg_parser.add_argument('-intei', type=str, help='Parsed and manually edited TEI file') +arg_parser.add_argument('-outstructures', type=str, help='Output structure file') +arg_parser.add_argument('-outlexicon', type=str, help='Output lexicon file') +arguments = arg_parser.parse_args() +input_list_file_name = arguments.inlist +input_tei_file_name = arguments.intei +output_lexicon_file_name = arguments.outlexicon +output_structure_file_name = arguments.outstructures + +def run_pipeline(input_list_file_name, input_tei_file_name, output_lexicon_file_name, output_structure_file_name): + shutil.rmtree(TMP_DIRECTORY, True) + os.makedirs(TMP_DIRECTORY, exist_ok=True) + shutil.copyfile(input_list_file_name, STRING_LIST_FILE_NAME) + shutil.copyfile(input_tei_file_name, TEI_INIT_FILE_NAME) + run_mwe_extraction(STRUCTURE_CURRENT_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_1_FILE_NAME) + run_structure_assignment(STRING_LIST_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_1_FILE_NAME, TEI_STRUCTURE_1_FILE_NAME) + run_structure_creation(STRUCTURE_CURRENT_FILE_NAME, TEI_STRUCTURE_1_FILE_NAME, STRUCTURE_NEW_FILE_NAME) + validate_structures(STRUCTURE_NEW_FILE_NAME) + run_mwe_extraction(STRUCTURE_NEW_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_2_FILE_NAME) + run_structure_assignment(STRING_LIST_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_2_FILE_NAME, TEI_STRUCTURE_2_FILE_NAME) + run_dictionary_conversion(TEI_STRUCTURE_2_FILE_NAME, DICTIONARY_XML_FILE_NAME) + validate_dictionary(DICTIONARY_XML_FILE_NAME) + shutil.copyfile(DICTIONARY_XML_FILE_NAME, output_lexicon_file_name) + shutil.copyfile(STRUCTURE_NEW_FILE_NAME, output_structure_file_name) + +def run_mwe_extraction(structure_file_name, tei_file_name, mwe_csv_file_name): + print('Extracting MWEs from tei ...') + extraction_command = ' '.join(['python', MWE_EXTRACTION_SCRIPT_NAME, structure_file_name, tei_file_name, '--all', mwe_csv_file_name, '--skip-id-check', '--fixed-restriction-order']) + print(extraction_command) + os.system(extraction_command) + +def validate_structures(structure_file_name): + print('Validating updated structure specifications ...') + xml_schema = lxml.XMLSchema(lxml.parse(STRUCTURE_SCHEMA_FILE_NAME)) + xml_tree = lxml.parse(structure_file_name) + xml_schema.assertValid(xml_tree) + +def run_structure_assignment(input_file_name, tei_file_name, mwe_csv_file_name, output_file_name): + print('Assigning structure ids ...') + assignment_command = ' '.join(['python', STRUCTURE_ASSIGNMENT_SCRIPT_NAME, input_file_name, tei_file_name, mwe_csv_file_name, output_file_name]) + os.system(assignment_command) + +def run_structure_creation(input_file_name, tei_file_name, output_file_name): + print('Creating missing structures ...') + creation_command = ' '.join(['python', STRUCTURE_CREATION_SCRIPT_NAME, '-infile', input_file_name, '-tei', tei_file_name, '-outfile', output_file_name]) + os.system(creation_command) + +def run_dictionary_conversion(tei_file_name, xml_file_name): + print('Converting to dictionary xml format ...') + convert_command = ' '.join(['python', TEI_DICTIONARY_SCRIPT_NAME, '-infile', tei_file_name, '-outfile', xml_file_name]) + os.system(convert_command) + +def validate_dictionary(dictionary_file_name): + print('Validating output dictionary file ...') + xml_schema = lxml.XMLSchema(lxml.parse(DICTIONARY_SCHEMA_FILE_NAME)) + xml_tree = lxml.parse(dictionary_file_name) + xml_schema.assertValid(xml_tree) + +run_pipeline(input_list_file_name, input_tei_file_name, output_lexicon_file_name, output_structure_file_name)