IssueID #1835: moved pipeline1 components into package
parent
d88809be8d
commit
5f25682036
@ -1,15 +1,40 @@
|
||||
#!/usr/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import os
|
||||
|
||||
import lxml.etree as lxml
|
||||
|
||||
from flask import Flask, jsonify, make_response, request
|
||||
from flask_httpauth import HTTPBasicAuth
|
||||
|
||||
import structure_assignment.pipeline as pipeline
|
||||
|
||||
app = Flask(__name__)
|
||||
api_prefix = os.environ['API_PREFIX']
|
||||
resource_directory = os.environ['API_RESOURCE_DIR']
|
||||
tmp_directory = os.environ['API_TMP']
|
||||
|
||||
@app.route(api_prefix + '/test/<string:string>', methods=['GET'])
|
||||
def test(string):
|
||||
results = {'input':string}
|
||||
|
||||
string_file_name = '/tmp/string.txt'
|
||||
parse_file_name = '/tmp/parse.xml'
|
||||
with open(string_file_name, 'w') as string_file:
|
||||
string_file.write(string + '\n')
|
||||
|
||||
try:
|
||||
pipeline.initialise(temp_dir=tmp_directory, resource_dir=resource_directory)
|
||||
pipeline.import_string_file(string_file_name)
|
||||
pipeline.do_tokenise()
|
||||
pipeline.do_tweak_conllu()
|
||||
pipeline.do_parse()
|
||||
pipeline.do_translate_jos()
|
||||
pipeline.do_conllu_to_tei()
|
||||
pipeline.export_parsed_file(parse_file_name)
|
||||
tei = lxml.parse(parse_file_name).getroot()
|
||||
message = lxml.tostring(tei, encoding='UTF-8', pretty_print=True).decode()
|
||||
ok = True
|
||||
except Exception as e:
|
||||
message = str(e)
|
||||
ok = False
|
||||
|
||||
results = {'ok':ok, 'message':message}
|
||||
return jsonify(results)
|
||||
|
||||
@ -0,0 +1,37 @@
|
||||
# scripts
|
||||
TEI_SPLIT_SCRIPT_NAME = 'split_tei.py'
|
||||
CONLLU_TWEAK_SCRIPT_NAME = 'tweak_conllu.py'
|
||||
TRANSLATION_SCRIPT_NAME = 'translate_jos.py'
|
||||
CONLLU_TEI_SCRIPT_NAME = 'conllu_to_xml.py'
|
||||
MWE_EXTRACTION_SCRIPT_NAME = 'wani.py'
|
||||
STRUCTURE_SINGLE_ASSIGNMENT_SCRIPT_NAME = 'assign_single_structures.py'
|
||||
STRUCTURE_ASSIGNMENT_SCRIPT_NAME = 'assign_structures.py'
|
||||
STRUCTURE_CREATION_SCRIPT_NAME = 'create_structures.py'
|
||||
STRUCTURE_CREATION_SCRIPT_NAME = 'create_structures.py'
|
||||
TEI_DICTIONARY_SCRIPT_NAME = 'tei_to_dictionary.py'
|
||||
DICTIONARY_MERGE_SCRIPT_NAME = 'merge_dictionaries.py'
|
||||
|
||||
# resources
|
||||
TRANSLATION_FILE_NAME = '../resources/dict.xml'
|
||||
STRUCTURE_SCHEMA_FILE_NAME = '../resources/structures.xsd'
|
||||
DICTIONARY_SCHEMA_FILE_NAME = '../resources/monolingual_dictionaries.xsd'
|
||||
|
||||
# temporary outputs
|
||||
STRING_LIST_FILE_NAME = 'strings.txt'
|
||||
OBELIKS_RAW_FILE_NAME = 'obeliks_raw.conllu'
|
||||
OBELIKS_TWEAKED_FILE_NAME = 'obeliks_tweaked.conllu'
|
||||
CLASSLA_OUTPUT_FILE_NAME = 'classla_raw.conllu'
|
||||
CLASSLA_TRANSLATED_FILE_NAME = 'classla_translated.conllu'
|
||||
TEI_INIT_FILE_NAME = 'tei_initial.xml'
|
||||
TEI_SINGLE_FILE_NAME = 'tei_single.xml'
|
||||
TEI_SINGLE_STRUCTURE_FILE_NAME = 'tei_single_with_ids.xml'
|
||||
TEI_MULTIPLE_FILE_NAME = 'tei_multiple.xml'
|
||||
TEI_MULTIPLE_STRUCTURE_1_FILE_NAME = 'tei_multiple_with_ids1.xml'
|
||||
TEI_MULTIPLE_STRUCTURE_2_FILE_NAME = 'tei_multiple_with_ids2.xml'
|
||||
MWE_CSV_1_FILE_NAME = 'mwes1.csv'
|
||||
MWE_CSV_2_FILE_NAME = 'mwes2.csv'
|
||||
STRUCTURE_OLD_FILE_NAME = 'structures_old.xml'
|
||||
STRUCTURE_NEW_FILE_NAME = 'structures_new.xml'
|
||||
DICTIONARY_SINGLE_FILE_NAME = 'dictionary_single.xml'
|
||||
DICTIONARY_MULTIPLE_FILE_NAME = 'dictionary_multiple.xml'
|
||||
DICTIONARY_FILE_NAME = 'dictionary.xml'
|
||||
@ -0,0 +1,74 @@
|
||||
import codecs
|
||||
import shutil
|
||||
import os
|
||||
|
||||
import obeliks
|
||||
|
||||
import classla
|
||||
from classla import Document
|
||||
from classla.models.common.conll import CoNLLFile
|
||||
|
||||
from structure_assignment.constants import *
|
||||
from structure_assignment.tweak_conllu import tweak as tweak_conllu
|
||||
from nova_slovnica.translate_jos import translate as translate_jos
|
||||
from nova_slovnica.conllu_to_xml import convert_file as conllu_to_tei
|
||||
|
||||
NLP_CONFIG_MAP = {
|
||||
'treebank': 'sl_ssj_jos',
|
||||
'processors': 'tokenize,pos,lemma,depparse',
|
||||
'tokenize_pretokenized': True,
|
||||
'models_dir': None
|
||||
}
|
||||
|
||||
XML_ID_PREFIX = 's'
|
||||
|
||||
resource_directory = None
|
||||
tmp_directory = None
|
||||
|
||||
def __get_tmp_file_name(file_name):
|
||||
return tmp_directory + '/' + file_name
|
||||
|
||||
def initialise(**argument_map):
|
||||
global tmp_directory, resource_directory
|
||||
tmp_directory = argument_map['temp_dir']
|
||||
resource_directory = argument_map['resource_dir']
|
||||
shutil.rmtree(tmp_directory, True)
|
||||
os.makedirs(tmp_directory, exist_ok=True)
|
||||
|
||||
def import_string_file(file_name):
|
||||
shutil.copyfile(file_name, __get_tmp_file_name(STRING_LIST_FILE_NAME))
|
||||
|
||||
def do_tokenise():
|
||||
input_file_name = __get_tmp_file_name(STRING_LIST_FILE_NAME)
|
||||
output_file_name = __get_tmp_file_name(OBELIKS_RAW_FILE_NAME)
|
||||
obeliks.run(in_file=input_file_name, out_file=output_file_name, conllu=True)
|
||||
|
||||
def do_tweak_conllu():
|
||||
input_file_name = __get_tmp_file_name(OBELIKS_RAW_FILE_NAME)
|
||||
output_file_name = __get_tmp_file_name(OBELIKS_TWEAKED_FILE_NAME)
|
||||
tweak_conllu(input_file_name, output_file_name)
|
||||
|
||||
def do_parse():
|
||||
input_file_name = __get_tmp_file_name(OBELIKS_TWEAKED_FILE_NAME)
|
||||
output_file_name = __get_tmp_file_name(CLASSLA_OUTPUT_FILE_NAME)
|
||||
doc = Document(text=None)
|
||||
conll_file = CoNLLFile(filename=input_file_name)
|
||||
doc.conll_file = conll_file
|
||||
NLP_CONFIG_MAP['models_dir'] = resource_directory + '/classla'
|
||||
nlp = classla.Pipeline('sl', **NLP_CONFIG_MAP)
|
||||
result = nlp(doc)
|
||||
result.conll_file.write_conll(output_file_name)
|
||||
|
||||
def do_translate_jos():
|
||||
input_file_name = __get_tmp_file_name(CLASSLA_OUTPUT_FILE_NAME)
|
||||
dictionary_file_name = resource_directory + '/dict.xml'
|
||||
output_file_name = __get_tmp_file_name(CLASSLA_TRANSLATED_FILE_NAME)
|
||||
translate_jos(input_file_name, dictionary_file_name, output_file_name)
|
||||
|
||||
def do_conllu_to_tei():
|
||||
input_file_name = __get_tmp_file_name(CLASSLA_TRANSLATED_FILE_NAME)
|
||||
output_file_name = __get_tmp_file_name(TEI_INIT_FILE_NAME)
|
||||
conllu_to_tei(input_file_name, output_file_name)
|
||||
|
||||
def export_parsed_file(file_name):
|
||||
shutil.copyfile(__get_tmp_file_name(TEI_INIT_FILE_NAME), file_name)
|
||||
@ -0,0 +1,54 @@
|
||||
import argparse
|
||||
import codecs
|
||||
import re
|
||||
|
||||
def write(output_file, line):
|
||||
output_file.write(line + '\n')
|
||||
|
||||
def write_paragraph(output_file, output_map):
|
||||
if (output_map is not None):
|
||||
write(output_file, output_map['paragraph'])
|
||||
write(output_file, output_map['sentence'])
|
||||
write(output_file, '# text = ' + ' '.join(output_map['texts']))
|
||||
for (index, token_line) in enumerate(output_map['tokens'], start=1):
|
||||
write(output_file, '\t'.join([str(index)] + token_line.split('\t')[1:]))
|
||||
write(output_file, '')
|
||||
|
||||
def tweak(input_file_name, output_file_name):
|
||||
|
||||
output_file = codecs.open(output_file_name, 'w')
|
||||
input_file = codecs.open(input_file_name, 'r')
|
||||
|
||||
output_map = None
|
||||
for line in input_file:
|
||||
if (line[0].isdigit()):
|
||||
output_map['tokens'].append(line.strip())
|
||||
else:
|
||||
match = re.search('^# (.+?) = (.+)$', line)
|
||||
if (match):
|
||||
(name, value) = match.groups()
|
||||
if (name == 'newpar id'):
|
||||
write_paragraph(output_file, output_map)
|
||||
paragraph_line = re.sub('^(# newpar id = )(\d+)$', r'\1p\2', line.strip())
|
||||
output_map = {'paragraph': paragraph_line, 'sentence':None, 'texts':[], 'tokens':[]}
|
||||
elif (name == 'sent_id'):
|
||||
if (value.endswith('.1')):
|
||||
output_map['sentence'] = re.sub('^(# sent_id = )(\d+\.1)$', r'\1s\2', line.strip())
|
||||
elif (name == 'text'):
|
||||
output_map['texts'].append(value)
|
||||
write_paragraph(output_file, output_map)
|
||||
|
||||
input_file.close()
|
||||
output_file.close()
|
||||
|
||||
|
||||
if (__name__ == '__main__'):
|
||||
|
||||
arg_parser = argparse.ArgumentParser(description='Fix invalid XML ids.')
|
||||
arg_parser.add_argument('-infile', type=str, help='Input file')
|
||||
arg_parser.add_argument('-outfile', type=str, help='Output file')
|
||||
arguments = arg_parser.parse_args()
|
||||
input_file_name = arguments.infile
|
||||
output_file_name = arguments.outfile
|
||||
|
||||
tweak(input_file_name, output_file_name)
|
||||
@ -1,41 +0,0 @@
|
||||
# temporary directory
|
||||
TMP_DIRECTORY = '../tmp/structure_assignment'
|
||||
|
||||
# scripts
|
||||
TEI_SPLIT_SCRIPT_NAME = 'split_tei.py'
|
||||
CONLLU_TWEAK_SCRIPT_NAME = 'tweak_conllu.py'
|
||||
TRANSLATION_SCRIPT_NAME = 'translate_jos.py'
|
||||
CONLLU_TEI_SCRIPT_NAME = 'conllu_to_xml.py'
|
||||
MWE_EXTRACTION_SCRIPT_NAME = 'wani.py'
|
||||
STRUCTURE_SINGLE_ASSIGNMENT_SCRIPT_NAME = 'assign_single_structures.py'
|
||||
STRUCTURE_ASSIGNMENT_SCRIPT_NAME = 'assign_structures.py'
|
||||
STRUCTURE_CREATION_SCRIPT_NAME = 'create_structures.py'
|
||||
STRUCTURE_CREATION_SCRIPT_NAME = 'create_structures.py'
|
||||
TEI_DICTIONARY_SCRIPT_NAME = 'tei_to_dictionary.py'
|
||||
DICTIONARY_MERGE_SCRIPT_NAME = 'merge_dictionaries.py'
|
||||
|
||||
# resources
|
||||
TRANSLATION_FILE_NAME = '../resources/dict.xml'
|
||||
CLASSLA_MODELS_DIRECTORY = '../resources/classla'
|
||||
STRUCTURE_SCHEMA_FILE_NAME = '../resources/structures.xsd'
|
||||
DICTIONARY_SCHEMA_FILE_NAME = '../resources/monolingual_dictionaries.xsd'
|
||||
|
||||
# temporary outputs
|
||||
STRING_LIST_FILE_NAME = TMP_DIRECTORY + '/strings.txt'
|
||||
OBELIKS_RAW_FILE_NAME = TMP_DIRECTORY + '/obeliks_raw.conllu'
|
||||
OBELIKS_TWEAKED_FILE_NAME = TMP_DIRECTORY + '/obeliks_tweaked.conllu'
|
||||
CLASSLA_OUTPUT_FILE_NAME = TMP_DIRECTORY + '/classla_raw.conllu'
|
||||
CLASSLA_TRANSLATED_FILE_NAME = TMP_DIRECTORY + '/classla_translated.conllu'
|
||||
TEI_INIT_FILE_NAME = TMP_DIRECTORY + '/tei_initial.xml'
|
||||
TEI_SINGLE_FILE_NAME = TMP_DIRECTORY + '/tei_single.xml'
|
||||
TEI_SINGLE_STRUCTURE_FILE_NAME = TMP_DIRECTORY + '/tei_single_with_ids.xml'
|
||||
TEI_MULTIPLE_FILE_NAME = TMP_DIRECTORY + '/tei_multiple.xml'
|
||||
TEI_MULTIPLE_STRUCTURE_1_FILE_NAME = TMP_DIRECTORY + '/tei_multiple_with_ids1.xml'
|
||||
TEI_MULTIPLE_STRUCTURE_2_FILE_NAME = TMP_DIRECTORY + '/tei_multiple_with_ids2.xml'
|
||||
MWE_CSV_1_FILE_NAME = TMP_DIRECTORY + '/mwes1.csv'
|
||||
MWE_CSV_2_FILE_NAME = TMP_DIRECTORY + '/mwes2.csv'
|
||||
STRUCTURE_OLD_FILE_NAME = TMP_DIRECTORY + '/structures_old.xml'
|
||||
STRUCTURE_NEW_FILE_NAME = TMP_DIRECTORY + '/structures_new.xml'
|
||||
DICTIONARY_SINGLE_FILE_NAME = TMP_DIRECTORY + '/dictionary_single.xml'
|
||||
DICTIONARY_MULTIPLE_FILE_NAME = TMP_DIRECTORY + '/dictionary_multiple.xml'
|
||||
DICTIONARY_FILE_NAME = TMP_DIRECTORY + '/dictionary.xml'
|
||||
@ -1,47 +0,0 @@
|
||||
import argparse
|
||||
import codecs
|
||||
import re
|
||||
|
||||
arg_parser = argparse.ArgumentParser(description='Fix invalid XML ids.')
|
||||
arg_parser.add_argument('-infile', type=str, help='Input file')
|
||||
arg_parser.add_argument('-outfile', type=str, help='Output file')
|
||||
arguments = arg_parser.parse_args()
|
||||
input_file_name = arguments.infile
|
||||
output_file_name = arguments.outfile
|
||||
|
||||
output_file = codecs.open(output_file_name, 'w')
|
||||
input_file = codecs.open(input_file_name, 'r')
|
||||
|
||||
def write(output_file, line):
|
||||
output_file.write(line + '\n')
|
||||
|
||||
def write_paragraph(output_file, output_map):
|
||||
if (output_map is not None):
|
||||
write(output_file, output_map['paragraph'])
|
||||
write(output_file, output_map['sentence'])
|
||||
write(output_file, '# text = ' + ' '.join(output_map['texts']))
|
||||
for (index, token_line) in enumerate(output_map['tokens'], start=1):
|
||||
write(output_file, '\t'.join([str(index)] + token_line.split('\t')[1:]))
|
||||
write(output_file, '')
|
||||
|
||||
output_map = None
|
||||
for line in input_file:
|
||||
if (line[0].isdigit()):
|
||||
output_map['tokens'].append(line.strip())
|
||||
else:
|
||||
match = re.search('^# (.+?) = (.+)$', line)
|
||||
if (match):
|
||||
(name, value) = match.groups()
|
||||
if (name == 'newpar id'):
|
||||
write_paragraph(output_file, output_map)
|
||||
paragraph_line = re.sub('^(# newpar id = )(\d+)$', r'\1p\2', line.strip())
|
||||
output_map = {'paragraph': paragraph_line, 'sentence':None, 'texts':[], 'tokens':[]}
|
||||
elif (name == 'sent_id'):
|
||||
if (value.endswith('.1')):
|
||||
output_map['sentence'] = re.sub('^(# sent_id = )(\d+\.1)$', r'\1s\2', line.strip())
|
||||
elif (name == 'text'):
|
||||
output_map['texts'].append(value)
|
||||
write_paragraph(output_file, output_map)
|
||||
|
||||
input_file.close()
|
||||
output_file.close()
|
||||
Loading…
Reference in new issue