71 lines
2.7 KiB
Python
71 lines
2.7 KiB
Python
import argparse
|
|
import os
|
|
import shutil
|
|
import codecs
|
|
|
|
import classla
|
|
from classla import Document
|
|
from classla.models.common.conll import CoNLLFile
|
|
|
|
from constants import *
|
|
|
|
arg_parser = argparse.ArgumentParser(description='Parse Slovene strings and convert to TEI.')
|
|
arg_parser.add_argument('-inlist', type=str, help='Input list file')
|
|
arg_parser.add_argument('-outtei', type=str, help='Output TEI file')
|
|
arguments = arg_parser.parse_args()
|
|
input_file_name = arguments.inlist
|
|
output_file_name = arguments.outtei
|
|
|
|
NLP_CONFIG_MAP = {
|
|
'treebank': 'sl_ssj_jos',
|
|
'processors': 'tokenize,pos,lemma,depparse',
|
|
'tokenize_pretokenized': True,
|
|
'models_dir': CLASSLA_MODELS_DIRECTORY
|
|
}
|
|
|
|
XML_ID_PREFIX = 's'
|
|
|
|
def run_pipeline(input_file_name, output_file_name):
|
|
shutil.rmtree(TMP_DIRECTORY, True)
|
|
os.makedirs(TMP_DIRECTORY, exist_ok=True)
|
|
shutil.copyfile(input_file_name, STRING_LIST_FILE_NAME)
|
|
run_obeliks(STRING_LIST_FILE_NAME, OBELIKS_RAW_FILE_NAME)
|
|
tweak_conllu(OBELIKS_RAW_FILE_NAME, OBELIKS_TWEAKED_FILE_NAME)
|
|
run_classla(OBELIKS_TWEAKED_FILE_NAME, CLASSLA_OUTPUT_FILE_NAME)
|
|
run_jos_translation(CLASSLA_OUTPUT_FILE_NAME, CLASSLA_TRANSLATED_FILE_NAME)
|
|
run_tei_conversion(CLASSLA_TRANSLATED_FILE_NAME, TEI_INIT_FILE_NAME)
|
|
shutil.copyfile(TEI_INIT_FILE_NAME, output_file_name)
|
|
|
|
def run_obeliks(list_file_name, conllu_file_name):
|
|
print('Running obeliks ...')
|
|
obeliks_command = ' '.join(['obeliks', '-c', '-if', list_file_name, '-o', conllu_file_name])
|
|
os.system(obeliks_command)
|
|
|
|
def tweak_conllu(input_file_name, output_file_name):
|
|
print('Tweaking conllu results ...')
|
|
tweak_command = ' '.join(['python', CONLLU_TWEAK_SCRIPT_NAME, '-infile', input_file_name, '-outfile', output_file_name])
|
|
os.system(tweak_command)
|
|
|
|
def run_classla(obeliks_file_name, classla_file_name):
|
|
print('Running classla ...')
|
|
doc = Document(text=None)
|
|
conll_file = CoNLLFile(filename=obeliks_file_name)
|
|
doc.conll_file = conll_file
|
|
nlp = classla.Pipeline('sl', **NLP_CONFIG_MAP)
|
|
result = nlp(doc)
|
|
result.conll_file.write_conll(classla_file_name)
|
|
|
|
def run_jos_translation(input_file_name, output_file_name):
|
|
print('Translating JOS ...')
|
|
translate_command = ' '.join(['python', TRANSLATION_SCRIPT_NAME, '-infile', input_file_name, '-dict', TRANSLATION_FILE_NAME, '-outfile', output_file_name])
|
|
print(translate_command)
|
|
os.system(translate_command)
|
|
|
|
def run_tei_conversion(classla_file_name, tei_file_name):
|
|
print('Converting to tei ...')
|
|
convert_command = ' '.join(['python', CONLLU_TEI_SCRIPT_NAME, '-o', tei_file_name, classla_file_name])
|
|
print(convert_command)
|
|
os.system(convert_command)
|
|
|
|
run_pipeline(input_file_name, output_file_name)
|