IssueID #1487: expanded conllu postprocessing and cleaned a bit

This commit is contained in:
Cyprian Laskowski 2020-12-02 23:51:28 +01:00
parent 9d8bd3f43e
commit 822ce25add
4 changed files with 54 additions and 37 deletions

View File

@ -2,6 +2,7 @@
TMP_DIRECTORY = '../tmp/structure_assignment' TMP_DIRECTORY = '../tmp/structure_assignment'
# scripts # scripts
CONLLU_TWEAK_SCRIPT_NAME = 'tweak_conllu.py'
CONLLU_TEI_SCRIPT_NAME = 'conllu_to_xml.py' CONLLU_TEI_SCRIPT_NAME = 'conllu_to_xml.py'
MWE_EXTRACTION_SCRIPT_NAME = 'wani.py' MWE_EXTRACTION_SCRIPT_NAME = 'wani.py'
STRUCTURE_ASSIGNMENT_SCRIPT_NAME = 'assign_structures.py' STRUCTURE_ASSIGNMENT_SCRIPT_NAME = 'assign_structures.py'

View File

@ -1,21 +0,0 @@
import argparse
import codecs
import re
arg_parser = argparse.ArgumentParser(description='Fix invalid XML ids.')
arg_parser.add_argument('-infile', type=str, help='Input file')
arg_parser.add_argument('-outfile', type=str, help='Output file')
arguments = arg_parser.parse_args()
input_file_name = arguments.infile
output_file_name = arguments.outfile
output_file = codecs.open(output_file_name, 'w')
input_file = codecs.open(input_file_name, 'r')
for line in input_file:
line = re.sub('xml:id="(?=\d)','xml:id="s', line)
line = line.replace('#', '#s')
output_file.write(line)
input_file.close()
output_file.close()

View File

@ -2,7 +2,6 @@ import argparse
import os import os
import shutil import shutil
import codecs import codecs
import re
import classla import classla
from classla import Document from classla import Document
@ -31,28 +30,20 @@ def run_pipeline(input_file_name, output_file_name):
os.makedirs(TMP_DIRECTORY, exist_ok=True) os.makedirs(TMP_DIRECTORY, exist_ok=True)
shutil.copyfile(input_file_name, STRING_LIST_FILE_NAME) shutil.copyfile(input_file_name, STRING_LIST_FILE_NAME)
run_obeliks4J(STRING_LIST_FILE_NAME, OBELIKS_RAW_FILE_NAME) run_obeliks4J(STRING_LIST_FILE_NAME, OBELIKS_RAW_FILE_NAME)
fix_xml_ids(OBELIKS_RAW_FILE_NAME, OBELIKS_TWEAKED_FILE_NAME) tweak_conllu(OBELIKS_RAW_FILE_NAME, OBELIKS_TWEAKED_FILE_NAME)
run_classla(OBELIKS_TWEAKED_FILE_NAME, CLASSLA_FILE_NAME) run_classla(OBELIKS_TWEAKED_FILE_NAME, CLASSLA_FILE_NAME)
run_tei_conversion(CLASSLA_FILE_NAME, TEI_INIT_FILE_NAME) run_tei_conversion(CLASSLA_FILE_NAME, TEI_INIT_FILE_NAME)
shutil.copyfile(TEI_INIT_FILE_NAME, output_file_name) shutil.copyfile(TEI_INIT_FILE_NAME, output_file_name)
def run_obeliks4J(obeliks_file_name, classla_file_name): def run_obeliks4J(list_file_name, conllu_file_name):
print('Running obeliks ...') print('Running obeliks ...')
obeliks_command = 'java -jar ' + OBELIKS_JAR_FILE_NAME + ' -d -if ' + STRING_LIST_FILE_NAME + ' -o ' + OBELIKS_RAW_FILE_NAME obeliks_command = 'java -jar ' + OBELIKS_JAR_FILE_NAME + ' -d -if ' + list_file_name + ' -o ' + conllu_file_name
os.system(obeliks_command) os.system(obeliks_command)
def fix_xml_ids(input_file_name, output_file_name): def tweak_conllu(input_file_name, output_file_name):
print('Fixing xml ids ...') print('Tweaking conllu results ...')
output_file = codecs.open(output_file_name, 'w') tweak_command = ' '.join(['python', CONLLU_TWEAK_SCRIPT_NAME, '-infile', input_file_name, '-outfile', output_file_name])
input_file = codecs.open(input_file_name, 'r') os.system(tweak_command)
regexp = r'^(# sent_id = )(\d+\.\d+)$'
for line in input_file:
match = re.search(regexp, line)
if (match):
line = match.group(1) + XML_ID_PREFIX + match.group(2) + '\n'
output_file.write(line)
input_file.close()
output_file.close()
def run_classla(obeliks_file_name, classla_file_name): def run_classla(obeliks_file_name, classla_file_name):
print('Running classla ...') print('Running classla ...')

46
scripts/tweak_conllu.py Normal file
View File

@ -0,0 +1,46 @@
import argparse
import codecs
import re
arg_parser = argparse.ArgumentParser(description='Fix invalid XML ids.')
arg_parser.add_argument('-infile', type=str, help='Input file')
arg_parser.add_argument('-outfile', type=str, help='Output file')
arguments = arg_parser.parse_args()
input_file_name = arguments.infile
output_file_name = arguments.outfile
output_file = codecs.open(output_file_name, 'w')
input_file = codecs.open(input_file_name, 'r')
def write(output_file, line):
output_file.write(line + '\n')
def write_paragraph(output_file, output_map):
if (output_map is not None):
write(output_file, output_map['paragraph'])
write(output_file, output_map['sentence'])
write(output_file, '# text = ' + ' '.join(output_map['texts']))
for (index, token_line) in enumerate(output_map['tokens'], start=1):
write(output_file, '\t'.join([str(index)] + token_line.split('\t')[1:]))
write(output_file, '')
output_map = None
for line in input_file:
if (line[0].isdigit()):
output_map['tokens'].append(line.strip())
else:
match = re.search('^# (.+) = (.+)$', line)
if (match):
(name, value) = match.groups()
if (name == 'newpar id'):
write_paragraph(output_file, output_map)
output_map = {'paragraph': line.strip(), 'sentence':None, 'texts':[], 'tokens':[]}
elif (name == 'sent_id'):
if (value.endswith('.1')):
output_map['sentence'] = re.sub('^(# sent_id = )(\d+\.1)$', r'\1s\2', line.strip())
elif (name == 'text'):
output_map['texts'].append(value)
write_paragraph(output_file, output_map)
input_file.close()
output_file.close()