IssueID #1487: expanded conllu postprocessing and cleaned a bit
This commit is contained in:
parent
9d8bd3f43e
commit
822ce25add
|
@ -2,6 +2,7 @@
|
|||
TMP_DIRECTORY = '../tmp/structure_assignment'
|
||||
|
||||
# scripts
|
||||
CONLLU_TWEAK_SCRIPT_NAME = 'tweak_conllu.py'
|
||||
CONLLU_TEI_SCRIPT_NAME = 'conllu_to_xml.py'
|
||||
MWE_EXTRACTION_SCRIPT_NAME = 'wani.py'
|
||||
STRUCTURE_ASSIGNMENT_SCRIPT_NAME = 'assign_structures.py'
|
||||
|
|
|
@ -1,21 +0,0 @@
|
|||
import argparse
|
||||
import codecs
|
||||
import re
|
||||
|
||||
arg_parser = argparse.ArgumentParser(description='Fix invalid XML ids.')
|
||||
arg_parser.add_argument('-infile', type=str, help='Input file')
|
||||
arg_parser.add_argument('-outfile', type=str, help='Output file')
|
||||
arguments = arg_parser.parse_args()
|
||||
input_file_name = arguments.infile
|
||||
output_file_name = arguments.outfile
|
||||
|
||||
output_file = codecs.open(output_file_name, 'w')
|
||||
input_file = codecs.open(input_file_name, 'r')
|
||||
|
||||
for line in input_file:
|
||||
line = re.sub('xml:id="(?=\d)','xml:id="s', line)
|
||||
line = line.replace('#', '#s')
|
||||
output_file.write(line)
|
||||
|
||||
input_file.close()
|
||||
output_file.close()
|
|
@ -2,7 +2,6 @@ import argparse
|
|||
import os
|
||||
import shutil
|
||||
import codecs
|
||||
import re
|
||||
|
||||
import classla
|
||||
from classla import Document
|
||||
|
@ -31,28 +30,20 @@ def run_pipeline(input_file_name, output_file_name):
|
|||
os.makedirs(TMP_DIRECTORY, exist_ok=True)
|
||||
shutil.copyfile(input_file_name, STRING_LIST_FILE_NAME)
|
||||
run_obeliks4J(STRING_LIST_FILE_NAME, OBELIKS_RAW_FILE_NAME)
|
||||
fix_xml_ids(OBELIKS_RAW_FILE_NAME, OBELIKS_TWEAKED_FILE_NAME)
|
||||
tweak_conllu(OBELIKS_RAW_FILE_NAME, OBELIKS_TWEAKED_FILE_NAME)
|
||||
run_classla(OBELIKS_TWEAKED_FILE_NAME, CLASSLA_FILE_NAME)
|
||||
run_tei_conversion(CLASSLA_FILE_NAME, TEI_INIT_FILE_NAME)
|
||||
shutil.copyfile(TEI_INIT_FILE_NAME, output_file_name)
|
||||
|
||||
def run_obeliks4J(obeliks_file_name, classla_file_name):
|
||||
def run_obeliks4J(list_file_name, conllu_file_name):
|
||||
print('Running obeliks ...')
|
||||
obeliks_command = 'java -jar ' + OBELIKS_JAR_FILE_NAME + ' -d -if ' + STRING_LIST_FILE_NAME + ' -o ' + OBELIKS_RAW_FILE_NAME
|
||||
obeliks_command = 'java -jar ' + OBELIKS_JAR_FILE_NAME + ' -d -if ' + list_file_name + ' -o ' + conllu_file_name
|
||||
os.system(obeliks_command)
|
||||
|
||||
def fix_xml_ids(input_file_name, output_file_name):
|
||||
print('Fixing xml ids ...')
|
||||
output_file = codecs.open(output_file_name, 'w')
|
||||
input_file = codecs.open(input_file_name, 'r')
|
||||
regexp = r'^(# sent_id = )(\d+\.\d+)$'
|
||||
for line in input_file:
|
||||
match = re.search(regexp, line)
|
||||
if (match):
|
||||
line = match.group(1) + XML_ID_PREFIX + match.group(2) + '\n'
|
||||
output_file.write(line)
|
||||
input_file.close()
|
||||
output_file.close()
|
||||
def tweak_conllu(input_file_name, output_file_name):
|
||||
print('Tweaking conllu results ...')
|
||||
tweak_command = ' '.join(['python', CONLLU_TWEAK_SCRIPT_NAME, '-infile', input_file_name, '-outfile', output_file_name])
|
||||
os.system(tweak_command)
|
||||
|
||||
def run_classla(obeliks_file_name, classla_file_name):
|
||||
print('Running classla ...')
|
||||
|
|
46
scripts/tweak_conllu.py
Normal file
46
scripts/tweak_conllu.py
Normal file
|
@ -0,0 +1,46 @@
|
|||
import argparse
|
||||
import codecs
|
||||
import re
|
||||
|
||||
arg_parser = argparse.ArgumentParser(description='Fix invalid XML ids.')
|
||||
arg_parser.add_argument('-infile', type=str, help='Input file')
|
||||
arg_parser.add_argument('-outfile', type=str, help='Output file')
|
||||
arguments = arg_parser.parse_args()
|
||||
input_file_name = arguments.infile
|
||||
output_file_name = arguments.outfile
|
||||
|
||||
output_file = codecs.open(output_file_name, 'w')
|
||||
input_file = codecs.open(input_file_name, 'r')
|
||||
|
||||
def write(output_file, line):
|
||||
output_file.write(line + '\n')
|
||||
|
||||
def write_paragraph(output_file, output_map):
|
||||
if (output_map is not None):
|
||||
write(output_file, output_map['paragraph'])
|
||||
write(output_file, output_map['sentence'])
|
||||
write(output_file, '# text = ' + ' '.join(output_map['texts']))
|
||||
for (index, token_line) in enumerate(output_map['tokens'], start=1):
|
||||
write(output_file, '\t'.join([str(index)] + token_line.split('\t')[1:]))
|
||||
write(output_file, '')
|
||||
|
||||
output_map = None
|
||||
for line in input_file:
|
||||
if (line[0].isdigit()):
|
||||
output_map['tokens'].append(line.strip())
|
||||
else:
|
||||
match = re.search('^# (.+) = (.+)$', line)
|
||||
if (match):
|
||||
(name, value) = match.groups()
|
||||
if (name == 'newpar id'):
|
||||
write_paragraph(output_file, output_map)
|
||||
output_map = {'paragraph': line.strip(), 'sentence':None, 'texts':[], 'tokens':[]}
|
||||
elif (name == 'sent_id'):
|
||||
if (value.endswith('.1')):
|
||||
output_map['sentence'] = re.sub('^(# sent_id = )(\d+\.1)$', r'\1s\2', line.strip())
|
||||
elif (name == 'text'):
|
||||
output_map['texts'].append(value)
|
||||
write_paragraph(output_file, output_map)
|
||||
|
||||
input_file.close()
|
||||
output_file.close()
|
Loading…
Reference in New Issue
Block a user