import argparse import os import shutil import codecs import classla from classla import Document from classla.models.common.conll import CoNLLFile from constants import * arg_parser = argparse.ArgumentParser(description='Parse Slovene strings and convert to TEI.') arg_parser.add_argument('-inlist', type=str, help='Input list file') arg_parser.add_argument('-outtei', type=str, help='Output TEI file') arguments = arg_parser.parse_args() input_file_name = arguments.inlist output_file_name = arguments.outtei NLP_CONFIG_MAP = { 'treebank': 'sl_ssj_jos', 'processors': 'tokenize,pos,lemma,depparse', 'tokenize_pretokenized': True, 'models_dir': CLASSLA_MODELS_DIRECTORY } XML_ID_PREFIX = 's' def run_pipeline(input_file_name, output_file_name): shutil.rmtree(TMP_DIRECTORY, True) os.makedirs(TMP_DIRECTORY, exist_ok=True) shutil.copyfile(input_file_name, STRING_LIST_FILE_NAME) run_obeliks(STRING_LIST_FILE_NAME, OBELIKS_RAW_FILE_NAME) tweak_conllu(OBELIKS_RAW_FILE_NAME, OBELIKS_TWEAKED_FILE_NAME) run_classla(OBELIKS_TWEAKED_FILE_NAME, CLASSLA_OUTPUT_FILE_NAME) run_jos_translation(CLASSLA_OUTPUT_FILE_NAME, CLASSLA_TRANSLATED_FILE_NAME) run_tei_conversion(CLASSLA_TRANSLATED_FILE_NAME, TEI_INIT_FILE_NAME) shutil.copyfile(TEI_INIT_FILE_NAME, output_file_name) def run_obeliks(list_file_name, conllu_file_name): print('Running obeliks ...') obeliks_command = ' '.join(['obeliks', '-c', '-if', list_file_name, '-o', conllu_file_name]) os.system(obeliks_command) def tweak_conllu(input_file_name, output_file_name): print('Tweaking conllu results ...') tweak_command = ' '.join(['python', CONLLU_TWEAK_SCRIPT_NAME, '-infile', input_file_name, '-outfile', output_file_name]) os.system(tweak_command) def run_classla(obeliks_file_name, classla_file_name): print('Running classla ...') doc = Document(text=None) conll_file = CoNLLFile(filename=obeliks_file_name) doc.conll_file = conll_file nlp = classla.Pipeline('sl', **NLP_CONFIG_MAP) result = nlp(doc) result.conll_file.write_conll(classla_file_name) def run_jos_translation(input_file_name, output_file_name): print('Translating JOS ...') translate_command = ' '.join(['python', TRANSLATION_SCRIPT_NAME, '-infile', input_file_name, '-dict', TRANSLATION_FILE_NAME, '-outfile', output_file_name]) print(translate_command) os.system(translate_command) def run_tei_conversion(classla_file_name, tei_file_name): print('Converting to tei ...') convert_command = ' '.join(['python', CONLLU_TEI_SCRIPT_NAME, '-o', tei_file_name, classla_file_name]) print(convert_command) os.system(convert_command) run_pipeline(input_file_name, output_file_name)