|
|
|
import codecs
|
|
|
|
import shutil
|
|
|
|
import os
|
|
|
|
|
|
|
|
import obeliks
|
|
|
|
|
|
|
|
import classla
|
|
|
|
from classla import Document
|
|
|
|
from classla.models.common.conll import CoNLLFile
|
|
|
|
|
|
|
|
from structure_assignment.constants import *
|
|
|
|
from structure_assignment.tweak_conllu import tweak as tweak_conllu
|
|
|
|
from nova_slovnica.translate_jos import translate as translate_jos
|
|
|
|
from nova_slovnica.conllu_to_xml import convert_file as conllu_to_tei
|
|
|
|
|
|
|
|
NLP_CONFIG_MAP = {
|
|
|
|
'treebank': 'sl_ssj_jos',
|
|
|
|
'processors': 'tokenize,pos,lemma,depparse',
|
|
|
|
'tokenize_pretokenized': True,
|
|
|
|
'models_dir': None
|
|
|
|
}
|
|
|
|
|
|
|
|
XML_ID_PREFIX = 's'
|
|
|
|
|
|
|
|
resource_directory = None
|
|
|
|
tmp_directory = None
|
|
|
|
nlp = None
|
|
|
|
|
|
|
|
def __get_tmp_file_name(file_name):
|
|
|
|
return tmp_directory + '/' + file_name
|
|
|
|
|
|
|
|
def initialise(**argument_map):
|
|
|
|
global tmp_directory, resource_directory, nlp
|
|
|
|
tmp_directory = argument_map['temp_dir']
|
|
|
|
resource_directory = argument_map['resource_dir']
|
|
|
|
shutil.rmtree(tmp_directory, True)
|
|
|
|
os.makedirs(tmp_directory, exist_ok=True)
|
|
|
|
NLP_CONFIG_MAP['models_dir'] = resource_directory + '/classla'
|
|
|
|
nlp = classla.Pipeline('sl', **NLP_CONFIG_MAP)
|
|
|
|
|
|
|
|
def import_string_file(file_name):
|
|
|
|
shutil.copyfile(file_name, __get_tmp_file_name(STRING_LIST_FILE_NAME))
|
|
|
|
|
|
|
|
def do_tokenise():
|
|
|
|
input_file_name = __get_tmp_file_name(STRING_LIST_FILE_NAME)
|
|
|
|
output_file_name = __get_tmp_file_name(OBELIKS_RAW_FILE_NAME)
|
|
|
|
obeliks.run(in_file=input_file_name, out_file=output_file_name, conllu=True)
|
|
|
|
|
|
|
|
def do_tweak_conllu():
|
|
|
|
input_file_name = __get_tmp_file_name(OBELIKS_RAW_FILE_NAME)
|
|
|
|
output_file_name = __get_tmp_file_name(OBELIKS_TWEAKED_FILE_NAME)
|
|
|
|
tweak_conllu(input_file_name, output_file_name)
|
|
|
|
|
|
|
|
def do_parse():
|
|
|
|
input_file_name = __get_tmp_file_name(OBELIKS_TWEAKED_FILE_NAME)
|
|
|
|
output_file_name = __get_tmp_file_name(CLASSLA_OUTPUT_FILE_NAME)
|
|
|
|
doc = Document(text=None)
|
|
|
|
conll_file = CoNLLFile(filename=input_file_name)
|
|
|
|
doc.conll_file = conll_file
|
|
|
|
result = nlp(doc)
|
|
|
|
result.conll_file.write_conll(output_file_name)
|
|
|
|
|
|
|
|
def do_translate_jos():
|
|
|
|
input_file_name = __get_tmp_file_name(CLASSLA_OUTPUT_FILE_NAME)
|
|
|
|
dictionary_file_name = resource_directory + '/dict.xml'
|
|
|
|
output_file_name = __get_tmp_file_name(CLASSLA_TRANSLATED_FILE_NAME)
|
|
|
|
translate_jos(input_file_name, dictionary_file_name, output_file_name)
|
|
|
|
|
|
|
|
def do_conllu_to_tei():
|
|
|
|
input_file_name = __get_tmp_file_name(CLASSLA_TRANSLATED_FILE_NAME)
|
|
|
|
output_file_name = __get_tmp_file_name(TEI_INIT_FILE_NAME)
|
|
|
|
conllu_to_tei(input_file_name, output_file_name)
|
|
|
|
|
|
|
|
def export_parsed_file(file_name):
|
|
|
|
shutil.copyfile(__get_tmp_file_name(TEI_INIT_FILE_NAME), file_name)
|