structure_assignment/scripts/pipeline2.py

import argparse
import os
import shutil
import lxml.etree as lxml

from constants import *

arg_parser = argparse.ArgumentParser(description='Assign parsed Slovene strings to structures and generate lexicon.')
arg_parser.add_argument('-intei', type=str, required=True, help='Parsed and manually edited TEI file')
arg_parser.add_argument('-instructures', type=str, required=True, help='Input structure file')
arg_parser.add_argument('-outlexicon', type=str, required=True, help='Output lexicon file')
arg_parser.add_argument('-outstructures', type=str, required=True, help='Output structure file')
arguments = arg_parser.parse_args()

input_tei_file_name = arguments.intei
input_structure_file_name = arguments.instructures
output_lexicon_file_name = arguments.outlexicon
output_structure_file_name = arguments.outstructures

def run_pipeline(input_tei_file_name, input_structure_file_name, output_lexicon_file_name, output_structure_file_name):

    # setup and split
    shutil.rmtree(TMP_DIRECTORY, True)
    os.makedirs(TMP_DIRECTORY, exist_ok=True)
    shutil.copyfile(input_tei_file_name, TEI_INIT_FILE_NAME)
    shutil.copyfile(input_structure_file_name, STRUCTURE_OLD_FILE_NAME)
    split_tei_input(TEI_INIT_FILE_NAME, TEI_SINGLE_FILE_NAME, TEI_MULTIPLE_FILE_NAME)

    # single-token units
    run_structure_single_assignment(TEI_SINGLE_FILE_NAME, STRUCTURE_OLD_FILE_NAME, TEI_SINGLE_STRUCTURE_FILE_NAME)
    run_dictionary_conversion(TEI_SINGLE_STRUCTURE_FILE_NAME, DICTIONARY_SINGLE_FILE_NAME)

    # multiple-token units
    run_mwe_extraction(STRUCTURE_OLD_FILE_NAME, TEI_MULTIPLE_FILE_NAME, MWE_CSV_1_FILE_NAME)
    run_structure_multiple_assignment(TEI_MULTIPLE_FILE_NAME, MWE_CSV_1_FILE_NAME, TEI_MULTIPLE_STRUCTURE_1_FILE_NAME)
    run_structure_creation(STRUCTURE_OLD_FILE_NAME, TEI_MULTIPLE_STRUCTURE_1_FILE_NAME, STRUCTURE_NEW_FILE_NAME)
    validate_structures(STRUCTURE_NEW_FILE_NAME)
    run_mwe_extraction(STRUCTURE_NEW_FILE_NAME, TEI_MULTIPLE_FILE_NAME, MWE_CSV_2_FILE_NAME)
    run_structure_multiple_assignment(TEI_MULTIPLE_FILE_NAME, MWE_CSV_2_FILE_NAME, TEI_MULTIPLE_STRUCTURE_2_FILE_NAME)
    run_dictionary_conversion(TEI_MULTIPLE_STRUCTURE_2_FILE_NAME, DICTIONARY_MULTIPLE_FILE_NAME)

    # merge and finish
    merge_dictionaries(DICTIONARY_SINGLE_FILE_NAME, DICTIONARY_MULTIPLE_FILE_NAME, DICTIONARY_FILE_NAME)
    validate_dictionary(DICTIONARY_FILE_NAME)
    shutil.copyfile(DICTIONARY_FILE_NAME, output_lexicon_file_name)
    shutil.copyfile(STRUCTURE_NEW_FILE_NAME, output_structure_file_name)

def split_tei_input(input_file_name, single_file_name, multiple_file_name):
    print('Splitting TEI input file ...')
    split_command = ' '.join(['python', TEI_SPLIT_SCRIPT_NAME, '-infile', input_file_name, '-single', single_file_name, '-multiple', multiple_file_name])
    os.system(split_command)

def run_mwe_extraction(structure_file_name, tei_file_name, mwe_csv_file_name):
    print('Extracting MWEs from tei ...')
    extraction_command = ' '.join(['python', MWE_EXTRACTION_SCRIPT_NAME, structure_file_name, tei_file_name, '--all', mwe_csv_file_name, '--skip-id-check', '--fixed-restriction-order', '--new-tei'])
    print(extraction_command)
    os.system(extraction_command)

def validate_structures(structure_file_name):
    print('Validating updated structure specifications ...')
    xml_schema = lxml.XMLSchema(lxml.parse(STRUCTURE_SCHEMA_FILE_NAME))
    xml_tree = lxml.parse(structure_file_name)
    xml_schema.assertValid(xml_tree)

def run_structure_single_assignment(input_file_name, structure_file_name, output_file_name):
    print('Assigning structure ids ...')
    assignment_command = ' '.join(['python', STRUCTURE_SINGLE_ASSIGNMENT_SCRIPT_NAME, '-infile', input_file_name, '-structures', structure_file_name, '-outfile', output_file_name])
    os.system(assignment_command)

def run_structure_multiple_assignment(tei_file_name, mwe_csv_file_name, output_file_name):
    print('Assigning structure ids ...')
    assignment_command = ' '.join(['python', STRUCTURE_ASSIGNMENT_SCRIPT_NAME, tei_file_name, mwe_csv_file_name, output_file_name])
    os.system(assignment_command)

def run_structure_creation(input_file_name, tei_file_name, output_file_name):
    print('Creating missing structures ...')
    creation_command = ' '.join(['python', STRUCTURE_CREATION_SCRIPT_NAME, '-infile', input_file_name, '-tei', tei_file_name, '-outfile', output_file_name])
    os.system(creation_command)
    
def run_dictionary_conversion(tei_file_name, xml_file_name):
    print('Converting to dictionary xml format ...')
    convert_command = ' '.join(['python', TEI_DICTIONARY_SCRIPT_NAME, '-infile', tei_file_name, '-outfile', xml_file_name])
    os.system(convert_command)

def merge_dictionaries(single_file_name, multiple_file_name, output_file_name):
    print('Merging dictionary files ...')
    merge_command = ' '.join(['python', DICTIONARY_MERGE_SCRIPT_NAME, '-single', single_file_name, '-multiple', multiple_file_name, '-outfile', output_file_name])
    os.system(merge_command)

def validate_dictionary(dictionary_file_name):
    print('Validating output dictionary file ...')
    xml_schema = lxml.XMLSchema(lxml.parse(DICTIONARY_SCHEMA_FILE_NAME))
    xml_tree = lxml.parse(dictionary_file_name)
    xml_schema.assertValid(xml_tree)

run_pipeline(input_tei_file_name, input_structure_file_name, output_lexicon_file_name, output_structure_file_name)