2019-02-18 07:49:04 +00:00
|
|
|
from parser.parser import Parser
|
|
|
|
|
import os
|
|
|
|
|
from os.path import join, dirname
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
import re
|
|
|
|
|
import sys
|
|
|
|
|
import cProfile
|
2019-02-27 08:15:40 +00:00
|
|
|
import configparser
|
|
|
|
|
|
|
|
|
|
# some defaults
|
|
|
|
|
INDIR = Path("../data/kres_example")
|
|
|
|
|
OUTDIR = Path("../data/kres_example_tsv")
|
|
|
|
|
|
|
|
|
|
SSJ500K_2_1 = 27829 # number of sentences
|
|
|
|
|
par = Parser()
|
|
|
|
|
|
|
|
|
|
# path to data
|
|
|
|
|
config = configparser.ConfigParser()
|
|
|
|
|
config.read("tools.cfg")
|
|
|
|
|
INDIR = Path(config["tools"]["kres_orig"])
|
|
|
|
|
OUTDIR = Path(config["tools"]["kres_tsv"])
|
|
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
print("parsing ssj")
|
|
|
|
|
ssj_file = "../data/ssj500k-sl.sample.xml"
|
|
|
|
|
ssj_dict = par.parse_tei(ssj_file)
|
|
|
|
|
# assert (len(ssj_dict) == 27829), "Parsed wrong number of sentences."
|
|
|
|
|
print("end parsing ssj")
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
print("parsing kres")
|
|
|
|
|
# kres_file = "../data/kres_example/F0019343.xml.parsed.xml"
|
|
|
|
|
OUTDIR.mkdir(exist_ok=True)
|
|
|
|
|
|
|
|
|
|
for kres_file in [x for x in INDIR.iterdir() if x.is_file()]:
|
|
|
|
|
|
|
|
|
|
print("Processing file: " + str(kres_file))
|
|
|
|
|
res_dict = par.parse_tei(kres_file)
|
|
|
|
|
longest_sent = max([len(e["tokens"]) for k, e in res_dict.items()])
|
|
|
|
|
print("Longest sentence: ", longest_sent)
|
|
|
|
|
kres_out_str = ""
|
|
|
|
|
|
|
|
|
|
for _, sentence in res_dict.items():
|
|
|
|
|
kres_out_str += par.to_conll_2009_SRL(sentence, longest_sent)
|
|
|
|
|
|
|
|
|
|
with (OUTDIR / kres_file.name).with_suffix(".tsv").open("wb+") as fp:
|
|
|
|
|
fp.write(kres_out_str.encode("utf-8"))
|
|
|
|
|
fp.close()
|
|
|
|
|
print("end parsing kres")
|
