generic/structure_assignment
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

IssueID #1487: expanded/configured pipeline

Browse Source
This commit is contained in:
Cyprian Laskowski 2020-09-30 16:38:42 +02:00
parent 298dfecbd6
commit d6ded65f67
1 changed files with 48 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
scripts/pipeline.py
View File

@ -12,20 +12,27 @@ input_file_name = sys.argv[1]
output_file_name = sys.argv[2] output_file_name = sys.argv[2]
TMP_DIRECTORY = '../tmp/structure_assignment' TMP_DIRECTORY = '../tmp/structure_assignment'
STRING_LIST_FILE_NAME = TMP_DIRECTORY + '/strings.txt'
CONLLU_TEI_SCRIPT_NAME = 'conllu_to_xml.py'
MWE_EXTRACTION_SCRIPT_NAME = 'wani.py'
STRUCTURE_ASSIGNMENT_SCRIPT_NAME = 'assign_structures.py'
STRUCTURE_CREATION_SCRIPT_NAME = 'create_structures.py'
OBELIKS_JAR_FILE_NAME = '../resources/obeliks.jar' OBELIKS_JAR_FILE_NAME = '../resources/obeliks.jar'
TRANSLATION_FILE_NAME = '../resources/dict.xml'
CLASSLA_MODELS_DIRECTORY = '../resources/classla'
STRUCTURE_CURRENT_FILE_NAME = '../resources/structures.xml'
STRING_LIST_FILE_NAME = TMP_DIRECTORY + '/strings.txt'
OBELIKS_RAW_FILE_NAME = TMP_DIRECTORY + '/obeliks_raw.conllu' OBELIKS_RAW_FILE_NAME = TMP_DIRECTORY + '/obeliks_raw.conllu'
OBELIKS_TWEAKED_FILE_NAME = TMP_DIRECTORY + '/obeliks_tweaked.conllu' OBELIKS_TWEAKED_FILE_NAME = TMP_DIRECTORY + '/obeliks_tweaked.conllu'
CLASSLA_MODELS_DIRECTORY = '../resources/classla'
CLASSLA_FILE_NAME = TMP_DIRECTORY + '/classla.conllu' CLASSLA_FILE_NAME = TMP_DIRECTORY + '/classla.conllu'
CONLLU_TEI_SCRIPT_NAME = 'conllu_to_xml.py' TEI_INIT_FILE_NAME = TMP_DIRECTORY + '/tei_initial.xml'
TEI_FILE_NAME = TMP_DIRECTORY + '/tei.xml' TEI_STRUCTURE_FILE_NAME = TMP_DIRECTORY + '/tei_with_structure_ids.xml'
TRANSLATION_FILE_NAME = '../resources/dict.xml' MWE_CSV_1_FILE_NAME = TMP_DIRECTORY + '/mwes1.csv'
MWE_EXTRACTION_SCRIPT_NAME = 'wani.py' MWE_CSV_2_FILE_NAME = TMP_DIRECTORY + '/mwes2.csv'
STRUCTURE_FILE_NAME = '../resources/structures.xml' STRUCTURE_NEW_FILE_NAME = TMP_DIRECTORY + '/structures_new.xml'
MWE_CSV_FILE_NAME = TMP_DIRECTORY + '/mwes.csv' TEI_STRUCTURE_FILE_NAME = TMP_DIRECTORY + '/tei_with_structure_ids.xml'
STRUCTURE_ASSIGNMENT_SCRIPT_NAME = 'assign_structures.py'
STRING_STRUCTURE_FILE_NAME = TMP_DIRECTORY + '/strings_with_structures.csv'
NLP_CONFIG_MAP = { NLP_CONFIG_MAP = {
'treebank': 'sl_ssj_jos', 'treebank': 'sl_ssj_jos',
@ -40,23 +47,27 @@ def run_pipeline(input_file_name, output_file_name):
shutil.rmtree(TMP_DIRECTORY, True) shutil.rmtree(TMP_DIRECTORY, True)
os.makedirs(TMP_DIRECTORY, exist_ok=True) os.makedirs(TMP_DIRECTORY, exist_ok=True)
shutil.copyfile(input_file_name, STRING_LIST_FILE_NAME) shutil.copyfile(input_file_name, STRING_LIST_FILE_NAME)
run_obeliks4J() run_obeliks4J(STRING_LIST_FILE_NAME, OBELIKS_RAW_FILE_NAME)
fix_xml_ids() fix_xml_ids(OBELIKS_RAW_FILE_NAME, OBELIKS_TWEAKED_FILE_NAME)
run_classla() run_classla(OBELIKS_TWEAKED_FILE_NAME, CLASSLA_FILE_NAME)
convert_to_tei() convert_to_tei(CLASSLA_FILE_NAME, TEI_INIT_FILE_NAME)
run_mwe_extraction() run_mwe_extraction(STRUCTURE_CURRENT_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_1_FILE_NAME)
run_structure_assignment() run_structure_assignment(STRING_LIST_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_1_FILE_NAME, TEI_STRUCTURE_FILE_NAME)
shutil.copyfile(STRING_STRUCTURE_FILE_NAME, output_file_name) run_structure_creation(STRUCTURE_CURRENT_FILE_NAME, TEI_STRUCTURE_FILE_NAME, STRUCTURE_NEW_FILE_NAME)
run_mwe_extraction(STRUCTURE_NEW_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_2_FILE_NAME)
run_structure_assignment(STRING_LIST_FILE_NAME, TEI_INIT_FILE_NAME, MWE_CSV_2_FILE_NAME, TEI_STRUCTURE_FILE_NAME)
# run_format_lexicon() # TODO: implement
shutil.copyfile(TEI_STRUCTURE_FILE_NAME, output_file_name)
def run_obeliks4J(): def run_obeliks4J(obeliks_file_name, classla_file_name):
print('Running obeliks ...') print('Running obeliks ...')
obeliks_command = 'java -jar ' + OBELIKS_JAR_FILE_NAME + ' -d -if ' + STRING_LIST_FILE_NAME + ' -o ' + OBELIKS_RAW_FILE_NAME obeliks_command = 'java -jar ' + OBELIKS_JAR_FILE_NAME + ' -d -if ' + STRING_LIST_FILE_NAME + ' -o ' + OBELIKS_RAW_FILE_NAME
os.system(obeliks_command) os.system(obeliks_command)
def fix_xml_ids(): def fix_xml_ids(input_file_name, output_file_name):
print('Fixing xml ids ...') print('Fixing xml ids ...')
output_file = codecs.open(OBELIKS_TWEAKED_FILE_NAME, 'w') output_file = codecs.open(output_file_name, 'w')
input_file = codecs.open(OBELIKS_RAW_FILE_NAME, 'r') input_file = codecs.open(input_file_name, 'r')
regexp = r'^(# sent_id = )(\d+\.\d+)$' regexp = r'^(# sent_id = )(\d+\.\d+)$'
for line in input_file: for line in input_file:
match = re.search(regexp, line) match = re.search(regexp, line)
@ -66,28 +77,34 @@ def fix_xml_ids():
input_file.close() input_file.close()
output_file.close() output_file.close()
def run_classla(): def run_classla(obeliks_file_name, classla_file_name):
print('Running classla ...') print('Running classla ...')
doc = Document(text=None) doc = Document(text=None)
conll_file = CoNLLFile(filename=OBELIKS_TWEAKED_FILE_NAME) conll_file = CoNLLFile(filename=obeliks_file_name)
doc.conll_file = conll_file doc.conll_file = conll_file
nlp = classla.Pipeline('sl', **NLP_CONFIG_MAP) nlp = classla.Pipeline('sl', **NLP_CONFIG_MAP)
result = nlp(doc) result = nlp(doc)
result.conll_file.write_conll(CLASSLA_FILE_NAME) result.conll_file.write_conll(classla_file_name)
def convert_to_tei(): def convert_to_tei(classla_file_name, tei_file_name):
print('Converting to tei ...') print('Converting to tei ...')
convert_command = ' '.join(['python', CONLLU_TEI_SCRIPT_NAME, CLASSLA_FILE_NAME, TEI_FILE_NAME, '--translate', TRANSLATION_FILE_NAME]) convert_command = ' '.join(['python', CONLLU_TEI_SCRIPT_NAME, classla_file_name, tei_file_name, '--translate', TRANSLATION_FILE_NAME])
os.system(convert_command) os.system(convert_command)
def run_mwe_extraction(): def run_mwe_extraction(structure_file_name, tei_file_name, mwe_csv_file_name):
print('Extracting MWEs from tei ...') print('Extracting MWEs from tei ...')
extraction_command = ' '.join(['python', MWE_EXTRACTION_SCRIPT_NAME, STRUCTURE_FILE_NAME, TEI_FILE_NAME, '--all', MWE_CSV_FILE_NAME, '--skip-id-check']) extraction_command = ' '.join(['python', MWE_EXTRACTION_SCRIPT_NAME, structure_file_name, tei_file_name, '--all', mwe_csv_file_name, '--skip-id-check'])
print(extraction_command)
os.system(extraction_command) os.system(extraction_command)
def run_structure_assignment(): def run_structure_assignment(input_file_name, tei_file_name, mwe_csv_file_name, output_file_name):
print('Assign structure ids ...') print('Assigning structure ids ...')
assignment_command = ' '.join(['python', STRUCTURE_ASSIGNMENT_SCRIPT_NAME, STRING_LIST_FILE_NAME, TEI_FILE_NAME, MWE_CSV_FILE_NAME, STRING_STRUCTURE_FILE_NAME]) assignment_command = ' '.join(['python', STRUCTURE_ASSIGNMENT_SCRIPT_NAME, input_file_name, tei_file_name, mwe_csv_file_name, output_file_name])
os.system(assignment_command)
def run_structure_creation(input_file_name, tei_file_name, output_file_name):
print('Creating missing structures ...')
assignment_command = ' '.join(['python', STRUCTURE_CREATION_SCRIPT_NAME, '-infile', input_file_name, '-tei', tei_file_name, '-outfile', output_file_name])
os.system(assignment_command) os.system(assignment_command)
run_pipeline(input_file_name, output_file_name) run_pipeline(input_file_name, output_file_name)