lkrsnik/cjvt-valency
1
0
Fork 0
forked from kristjan/cjvt-valency
Code Issues Pull Requests Releases Wiki Activity

ssj dumps to json; todo: dump kres to multiple files; dump sentence_dict objects to DB

Browse Source
This commit is contained in:
voje 2019-03-14 09:01:41 +01:00
parent dc20480e20
commit 83c24609bf
2 changed files with 47 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/pkg/corpusparser/corpusparser/Parser.py
View File

@ -10,7 +10,7 @@ logging.basicConfig(level=logging.INFO)
# Create an iterator that outputs resulting sentences (python dict format). # Create an iterator that outputs resulting sentences (python dict format).
class Parser(): class Parser():
def __init__(self, corpus, infiles, output=None, outdir=None, dbaddr=None, logger=None): def __init__(self, corpus, infiles, logger=None):
if corpus == "kres": if corpus == "kres":
self.kres_folder = Path(infiles[0]) self.kres_folder = Path(infiles[0])
@ -20,13 +20,6 @@ class Parser():
else: else:
raise ValueError("Argument corpus should be 'ssj' or 'kres'.") raise ValueError("Argument corpus should be 'ssj' or 'kres'.")
self.output = output # None | file | db
if self.output == "file":
self.outdir = Path(outdir)
self.outdir.mkdir(parents=True, exist_ok=True)
elif self.output == "db":
self.dbaddr = "TODO"
self.corpus = corpus self.corpus = corpus
self.W_TAGS = ['w'] self.W_TAGS = ['w']
self.C_TAGS = ['c'] self.C_TAGS = ['c']

83
src/pkg/corpusparser/corpusparser/main.py
View File

@ -1,3 +1,4 @@
from pathlib import Path
from corpusparser import Parser from corpusparser import Parser
import argparse import argparse
import logging import logging
@ -8,37 +9,54 @@ logger = logging.getLogger(__name__)
## Main handles command line arguments and writing to files / DB. ## Main handles command line arguments and writing to files / DB.
def ssj_to_json_file(sentence_generator, outfile):
# this funciton is based on the fact that files are parsed sequentially
data_buffer = []
for s in sentence_generator:
sdata = s[1]
data_buffer += [sdata]
# outfile = Path(outfile)
with open(outfile, "w") as fp:
logger.info("Writing to {}".format(outfile))
json.dump(data_buffer, fp)
def kres_to_json_files()
return "TODO"
def to_db():
return "TODO"
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Parsing corpora kres and ssj500k.") parser = argparse.ArgumentParser(description="Parsing corpora kres and ssj500k.")
parser.add_argument('--kres-folder', required=True) parser.add_argument('--kres-folder', required=True)
parser.add_argument('--kres-srl-folder', required=True) parser.add_argument('--kres-srl-folder', required=True)
parser.add_argument('--ssj-file', required=True) parser.add_argument('--ssj-file', required=True)
parser.add_argument('--output', required=False, default=None) parser.add_argument('--output', required=False, default=None)
parser.add_argument('--outdir', required=False, default=None) parser.add_argument('--outdir', required=False, default=None)
parser.add_argument('--dbaddr', required=False, default=None) parser.add_argument('--dbaddr', required=False, default=None)
args = parser.parse_args() args = parser.parse_args()
# parse ssj # parse ssj
logger.info("Parsing ssj500k: {}".format(args.ssj_file)) logger.info("Parsing ssj500k: {}".format(args.ssj_file))
ssj_parser = Parser( ssj_parser = Parser(
corpus="ssj", corpus="ssj",
infiles=[args.ssj_file], infiles=[args.ssj_file],
output=args.output, )
outdir=args.outdir, # res = [x[1]["sid"] for x in ssj_parser.sentence_generator()]
) # logger.info("Parsed {} sentences (ssj500k)".format(len(res)))
res = [x[1]["sid"] for x in ssj_parser.sentence_generator()]
logger.info("Parsed {} sentences (ssj500k)".format(len(res)))
# parse kres # ssj to json
logger.info("Parsing Kres: {}".format(args.ssj_file)) ssj_to_json_file(ssj_parser.sentence_generator(), "/home/voje/workdir/ssj.json")
kres_parser = Parser(
corpus="kres", # parse kres
infiles=[args.kres_folder, args.kres_srl_folder], logger.info("Parsing Kres: {}".format(args.ssj_file))
output=args.output, kres_parser = Parser(
outdir=args.outdir, corpus="kres",
) infiles=[args.kres_folder, args.kres_srl_folder],
res = [x[1]["sid"] for x in kres_parser.sentence_generator()] )
logger.info("Parsed {} sentences (kres)".format(len(res))) res = [x[1]["sid"] for x in kres_parser.sentence_generator()]
logger.info("Parsed {} sentences (kres)".format(len(res)))
## Handling output is situational --- implement it outside of Parser. ## Handling output is situational --- implement it outside of Parser.
@ -47,12 +65,3 @@ if __name__ == "__main__":
# 2. parse and save to DB # 2. parse and save to DB
# TODO # TODO
def handle_output(self, sent_ent, xml_file):
if self.output is None:
pass
if self.output == "file":
outfile = Path(self.outdir) / Path(xml_file.name.split(".")[0]).with_suffix(".json")
with outfile.open("a") as fp:
print(sent_ent)
json.dumps(sent_ent, fp)