IssueID #1487: expanded/configured pipeline

scripts/pipeline.py

@@ -12,20 +12,27 @@ input_file_name = sys.argv[1]
 output_file_name = sys.argv[2]
 
 TMP_DIRECTORY = '../tmp/structure_assignment'
-STRING_LIST_FILE_NAME = TMP_DIRECTORY + '/strings.txt'
+
+CONLLU_TEI_SCRIPT_NAME = 'conllu_to_xml.py'
+MWE_EXTRACTION_SCRIPT_NAME = 'wani.py'
+STRUCTURE_ASSIGNMENT_SCRIPT_NAME = 'assign_structures.py'
+STRUCTURE_CREATION_SCRIPT_NAME = 'create_structures.py'
+
 OBELIKS_JAR_FILE_NAME = '../resources/obeliks.jar'
+TRANSLATION_FILE_NAME = '../resources/dict.xml'
+CLASSLA_MODELS_DIRECTORY = '../resources/classla'
+STRUCTURE_CURRENT_FILE_NAME = '../resources/structures.xml'
+
+STRING_LIST_FILE_NAME = TMP_DIRECTORY + '/strings.txt'
 OBELIKS_RAW_FILE_NAME = TMP_DIRECTORY + '/obeliks_raw.conllu'
 OBELIKS_TWEAKED_FILE_NAME = TMP_DIRECTORY + '/obeliks_tweaked.conllu'
-CLASSLA_MODELS_DIRECTORY = '../resources/classla'
 CLASSLA_FILE_NAME = TMP_DIRECTORY + '/classla.conllu'
-CONLLU_TEI_SCRIPT_NAME = 'conllu_to_xml.py'
-TEI_FILE_NAME = TMP_DIRECTORY + '/tei.xml'
-TRANSLATION_FILE_NAME = '../resources/dict.xml'
-MWE_EXTRACTION_SCRIPT_NAME = 'wani.py'
-STRUCTURE_FILE_NAME = '../resources/structures.xml'
-MWE_CSV_FILE_NAME = TMP_DIRECTORY + '/mwes.csv'
-STRUCTURE_ASSIGNMENT_SCRIPT_NAME = 'assign_structures.py'
-STRING_STRUCTURE_FILE_NAME = TMP_DIRECTORY + '/strings_with_structures.csv'
+TEI_INIT_FILE_NAME = TMP_DIRECTORY + '/tei_initial.xml'
+TEI_STRUCTURE_FILE_NAME = TMP_DIRECTORY + '/tei_with_structure_ids.xml'
+MWE_CSV_1_FILE_NAME = TMP_DIRECTORY + '/mwes1.csv'
+MWE_CSV_2_FILE_NAME = TMP_DIRECTORY + '/mwes2.csv'
+STRUCTURE_NEW_FILE_NAME = TMP_DIRECTORY + '/structures_new.xml'
+TEI_STRUCTURE_FILE_NAME = TMP_DIRECTORY + '/tei_with_structure_ids.xml'
 
 NLP_CONFIG_MAP = {
     'treebank': 'sl_ssj_jos',
@@ -40,23 +47,27 @@ def run_pipeline(input_file_name, output_file_name):
     shutil.rmtree(TMP_DIRECTORY, True)
     os.makedirs(TMP_DIRECTORY, exist_ok=True)
     shutil.copyfile(input_file_name, STRING_LIST_FILE_NAME)
-    run_obeliks4J()
-    fix_xml_ids()
-    run_classla()
-    convert_to_tei()
-    run_mwe_extraction()
-    run_structure_assignment()
-    shutil.copyfile(STRING_STRUCTURE_FILE_NAME, output_file_name)
-
-def run_obeliks4J():
+    run_obeliks4J(STRING_LIST_FILE_NAME, OBELIKS_RAW_FILE_NAME)
+    fix_xml_ids(OBELIKS_RAW_FILE_NAME, OBELIKS_TWEAKED_FILE_NAME)
+    run_classla(OBELIKS_TWEAKED_FILE_NAME, CLASSLA_FILE_NAME)
+    convert_to_tei(CLASSLA_FILE_NAME, TEI_INIT_FILE_NAME)
+    run_mwe_extraction(STRUCTURE_CURRENT_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_1_FILE_NAME)
+    run_structure_assignment(STRING_LIST_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_1_FILE_NAME, TEI_STRUCTURE_FILE_NAME)
+    run_structure_creation(STRUCTURE_CURRENT_FILE_NAME, TEI_STRUCTURE_FILE_NAME, STRUCTURE_NEW_FILE_NAME)
+    run_mwe_extraction(STRUCTURE_NEW_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_2_FILE_NAME)
+    run_structure_assignment(STRING_LIST_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_2_FILE_NAME, TEI_STRUCTURE_FILE_NAME)
+    # run_format_lexicon() # TODO: implement
+    shutil.copyfile(TEI_STRUCTURE_FILE_NAME, output_file_name)
+
+def run_obeliks4J(obeliks_file_name, classla_file_name):
     print('Running obeliks ...')
     obeliks_command = 'java -jar ' + OBELIKS_JAR_FILE_NAME + ' -d -if ' + STRING_LIST_FILE_NAME + ' -o ' + OBELIKS_RAW_FILE_NAME
     os.system(obeliks_command)
 
-def fix_xml_ids():
+def fix_xml_ids(input_file_name, output_file_name):
     print('Fixing xml ids ...')
-    output_file = codecs.open(OBELIKS_TWEAKED_FILE_NAME, 'w')
-    input_file = codecs.open(OBELIKS_RAW_FILE_NAME, 'r')
+    output_file = codecs.open(output_file_name, 'w')
+    input_file = codecs.open(input_file_name, 'r')
     regexp = r'^(# sent_id = )(\d+\.\d+)$'
     for line in input_file:
         match = re.search(regexp, line)
@@ -66,28 +77,34 @@ def fix_xml_ids():
     input_file.close()
     output_file.close()
 
-def run_classla():
+def run_classla(obeliks_file_name, classla_file_name):
     print('Running classla ...')
     doc = Document(text=None)
-    conll_file = CoNLLFile(filename=OBELIKS_TWEAKED_FILE_NAME)
+    conll_file = CoNLLFile(filename=obeliks_file_name)
     doc.conll_file = conll_file
     nlp = classla.Pipeline('sl', **NLP_CONFIG_MAP)
     result = nlp(doc)
-    result.conll_file.write_conll(CLASSLA_FILE_NAME)
+    result.conll_file.write_conll(classla_file_name)
 
-def convert_to_tei():
+def convert_to_tei(classla_file_name, tei_file_name):
     print('Converting to tei ...')
-    convert_command = ' '.join(['python', CONLLU_TEI_SCRIPT_NAME, CLASSLA_FILE_NAME, TEI_FILE_NAME, '--translate', TRANSLATION_FILE_NAME])
+    convert_command = ' '.join(['python', CONLLU_TEI_SCRIPT_NAME, classla_file_name, tei_file_name, '--translate', TRANSLATION_FILE_NAME])
     os.system(convert_command)
 
-def run_mwe_extraction():
+def run_mwe_extraction(structure_file_name, tei_file_name, mwe_csv_file_name):
     print('Extracting MWEs from tei ...')
-    extraction_command = ' '.join(['python', MWE_EXTRACTION_SCRIPT_NAME, STRUCTURE_FILE_NAME, TEI_FILE_NAME, '--all', MWE_CSV_FILE_NAME, '--skip-id-check'])
+    extraction_command = ' '.join(['python', MWE_EXTRACTION_SCRIPT_NAME, structure_file_name, tei_file_name, '--all', mwe_csv_file_name, '--skip-id-check'])
+    print(extraction_command)
     os.system(extraction_command)
 
-def run_structure_assignment():
-    print('Assign structure ids ...')
-    assignment_command = ' '.join(['python', STRUCTURE_ASSIGNMENT_SCRIPT_NAME, STRING_LIST_FILE_NAME, TEI_FILE_NAME, MWE_CSV_FILE_NAME, STRING_STRUCTURE_FILE_NAME])
+def run_structure_assignment(input_file_name, tei_file_name, mwe_csv_file_name, output_file_name):
+    print('Assigning structure ids ...')
+    assignment_command = ' '.join(['python', STRUCTURE_ASSIGNMENT_SCRIPT_NAME, input_file_name, tei_file_name, mwe_csv_file_name, output_file_name])
+    os.system(assignment_command)
+
+def run_structure_creation(input_file_name, tei_file_name, output_file_name):
+    print('Creating missing structures ...')
+    assignment_command = ' '.join(['python', STRUCTURE_CREATION_SCRIPT_NAME, '-infile', input_file_name, '-tei', tei_file_name, '-outfile', output_file_name])
     os.system(assignment_command)
     
 run_pipeline(input_file_name, output_file_name)