added tools.cfg for configurable paths

per-file
voje 5 years ago
parent bcaf226b9e
commit de7035dfe5

@ -7,7 +7,7 @@ json_files: #TODO srl_tagged_files
srl_tagged_files: tsv_files srl_tagged_files: tsv_files
# cd tools/srl-20131216; ./scripts/parse_srl_only_mod.sh; cd - # cd tools/srl-20131216; ./scripts/parse_srl_only_mod.sh; cd -
cd tools/srl-20131216; ./tag_all.sh ../../data/kres_example_tsv ../../data/kres_example_srl cd tools/srl-20131216; ./tag_all.sh
tsv_files: fillpred_model/model.pickle tsv_files: fillpred_model/model.pickle
cd tools; python3 parse_all.py cd tools; python3 parse_all.py

@ -1,10 +1,19 @@
from pathlib import Path from pathlib import Path
from parser.parser import Parser from parser.parser import Parser
import configparser
# defaults
ORIGPATH = Path("../data/kres_example") # we need the IDs ORIGPATH = Path("../data/kres_example") # we need the IDs
INPATH = Path("../data/kres_example_srl") INPATH = Path("../data/kres_example_srl")
OUTPATH = Path("../data/kres_example_json") OUTPATH = Path("../data/kres_example_json")
# parse config
config = configparser.ConfigParser()
config.read("tools.cfg")
ORIGPATH = Path(config["tools"]["kres_orig"])
INPATH = Path(config["tools"]["kres_srl"])
OUTPATH = Path(config["tools"]["kres_json"])
def get_origfile(filename): def get_origfile(filename):
for origfile in ORIGPATH.iterdir(): for origfile in ORIGPATH.iterdir():
if filename.name.split('.')[0] == origfile.name.split('.')[0]: if filename.name.split('.')[0] == origfile.name.split('.')[0]:
@ -25,7 +34,7 @@ def match_sentence_id(string, rd):
str1 = " ".join([token[1] for token in sentence_arr]) str1 = " ".join([token[1] for token in sentence_arr])
for k, e in rd.items(): for k, e in rd.items():
str2 = " ".join(token[2] for token in dict_entry["tokens"]) str2 = " ".join(token[2] for token in dict_entry["tokens"])
if str1 == str2 if str1 == str2:
return k return k
raise KeyError raise KeyError

@ -5,42 +5,45 @@ from pathlib import Path
import re import re
import sys import sys
import cProfile import cProfile
import configparser
if __name__ == "__main__": # some defaults
# make sure you sanitize every input into unicode INDIR = Path("../data/kres_example")
OUTDIR = Path("../data/kres_example_tsv")
SSJ500K_2_1 = 27829 # number of sentences
par = Parser() SSJ500K_2_1 = 27829 # number of sentences
par = Parser()
"""
print("parsing ssj") # path to data
ssj_file = "../data/ssj500k-sl.sample.xml" config = configparser.ConfigParser()
ssj_dict = par.parse_tei(ssj_file) config.read("tools.cfg")
# assert (len(ssj_dict) == 27829), "Parsed wrong number of sentences." INDIR = Path(config["tools"]["kres_orig"])
print("end parsing ssj") OUTDIR = Path(config["tools"]["kres_tsv"])
"""
"""
print("parsing kres") print("parsing ssj")
# kres_file = "../data/kres_example/F0019343.xml.parsed.xml" ssj_file = "../data/ssj500k-sl.sample.xml"
kres_dir = Path("../data/kres_example/").resolve() ssj_dict = par.parse_tei(ssj_file)
# assert (len(ssj_dict) == 27829), "Parsed wrong number of sentences."
kres_out_dir = kres_dir.parent / (kres_dir.name + "_tsv") print("end parsing ssj")
kres_out_dir.mkdir(exist_ok=True) """
for kres_file in [x for x in kres_dir.iterdir() if x.is_file()]: print("parsing kres")
# kres_file = "../data/kres_example/F0019343.xml.parsed.xml"
print("Processing file: " + str(kres_file)) OUTDIR.mkdir(exist_ok=True)
res_dict = par.parse_tei(kres_file)
longest_sent = max([len(e["tokens"]) for k, e in res_dict.items()]) for kres_file in [x for x in INDIR.iterdir() if x.is_file()]:
print("Longest sentence: ", longest_sent)
kres_out_str = "" print("Processing file: " + str(kres_file))
res_dict = par.parse_tei(kres_file)
for _, sentence in res_dict.items(): longest_sent = max([len(e["tokens"]) for k, e in res_dict.items()])
kres_out_str += par.to_conll_2009_SRL(sentence, longest_sent) print("Longest sentence: ", longest_sent)
kres_out_str = ""
with (kres_out_dir / kres_file.name).with_suffix(".tsv").open("wb+") as fp:
fp.write(kres_out_str.encode("utf-8")) for _, sentence in res_dict.items():
fp.close() kres_out_str += par.to_conll_2009_SRL(sentence, longest_sent)
print("end parsing kres") with (OUTDIR / kres_file.name).with_suffix(".tsv").open("wb+") as fp:
fp.write(kres_out_str.encode("utf-8"))
fp.close()
print("end parsing kres")

@ -4,6 +4,7 @@ from parser.msd.msdmap import Msdmap
import pickle import pickle
from pathlib import Path from pathlib import Path
from fillpred_model.step1 import build_model_row from fillpred_model.step1 import build_model_row
import sys
class Parser: class Parser:
# reads a TEI xml file and returns a dictionary: # reads a TEI xml file and returns a dictionary:
@ -18,8 +19,12 @@ class Parser:
self.W_TAGS = ['w'] self.W_TAGS = ['w']
self.C_TAGS = ['c'] self.C_TAGS = ['c']
self.S_TAGS = ['S', 'pc'] self.S_TAGS = ['S', 'pc']
with Path("./fillpred_model/model.pickle").open("rb") as fp: try:
fp = Path("./fillpred_model/model.pickle").open("rb")
self.fillpred_model = pickle.load(fp) self.fillpred_model = pickle.load(fp)
except IOError:
print("Generate the model first: $ make tools/fillpred_mode/model.pickle")
sys.exit(1)
def parse_tei(self, filepath): def parse_tei(self, filepath):

@ -1,7 +1,11 @@
#!/bin/bash #!/bin/bash
IN_FOLDER="$1" # parsing tools.cfg values
OUT_FOLDER="$2" IN_FOLDER="../$(sed -n -e 's/^\s*kres_tsv\s*=\s*//p' ../tools.cfg)"
echo "input folder: $IN_FOLDER"
OUT_FOLDER="../$(sed -n -e 's/^\s*kres_srl\s*=\s*//p' ../tools.cfg)"
echo "output folder: $OUT_FOLDER"
SUFFIX="srl.tsv" SUFFIX="srl.tsv"
mkdir -p $OUT_FOLDER mkdir -p $OUT_FOLDER

@ -0,0 +1,5 @@
[tools]
kres_orig = ../data/kres_example
kres_tsv = ../data/kres_example_tsv
kres_srl = ../data/kres_example_srl
kres_json = ../data/kres/example_json
Loading…
Cancel
Save