forked from kristjan/cjvt-valency
ssj dumps to json; todo: dump kres to multiple files; dump sentence_dict objects to DB
This commit is contained in:
parent
dc20480e20
commit
83c24609bf
|
@ -10,7 +10,7 @@ logging.basicConfig(level=logging.INFO)
|
||||||
# Create an iterator that outputs resulting sentences (python dict format).
|
# Create an iterator that outputs resulting sentences (python dict format).
|
||||||
class Parser():
|
class Parser():
|
||||||
|
|
||||||
def __init__(self, corpus, infiles, output=None, outdir=None, dbaddr=None, logger=None):
|
def __init__(self, corpus, infiles, logger=None):
|
||||||
|
|
||||||
if corpus == "kres":
|
if corpus == "kres":
|
||||||
self.kres_folder = Path(infiles[0])
|
self.kres_folder = Path(infiles[0])
|
||||||
|
@ -20,13 +20,6 @@ class Parser():
|
||||||
else:
|
else:
|
||||||
raise ValueError("Argument corpus should be 'ssj' or 'kres'.")
|
raise ValueError("Argument corpus should be 'ssj' or 'kres'.")
|
||||||
|
|
||||||
self.output = output # None | file | db
|
|
||||||
if self.output == "file":
|
|
||||||
self.outdir = Path(outdir)
|
|
||||||
self.outdir.mkdir(parents=True, exist_ok=True)
|
|
||||||
elif self.output == "db":
|
|
||||||
self.dbaddr = "TODO"
|
|
||||||
|
|
||||||
self.corpus = corpus
|
self.corpus = corpus
|
||||||
self.W_TAGS = ['w']
|
self.W_TAGS = ['w']
|
||||||
self.C_TAGS = ['c']
|
self.C_TAGS = ['c']
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
from pathlib import Path
|
||||||
from corpusparser import Parser
|
from corpusparser import Parser
|
||||||
import argparse
|
import argparse
|
||||||
import logging
|
import logging
|
||||||
|
@ -8,6 +9,24 @@ logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
## Main handles command line arguments and writing to files / DB.
|
## Main handles command line arguments and writing to files / DB.
|
||||||
|
|
||||||
|
def ssj_to_json_file(sentence_generator, outfile):
|
||||||
|
# this funciton is based on the fact that files are parsed sequentially
|
||||||
|
data_buffer = []
|
||||||
|
for s in sentence_generator:
|
||||||
|
sdata = s[1]
|
||||||
|
data_buffer += [sdata]
|
||||||
|
|
||||||
|
# outfile = Path(outfile)
|
||||||
|
with open(outfile, "w") as fp:
|
||||||
|
logger.info("Writing to {}".format(outfile))
|
||||||
|
json.dump(data_buffer, fp)
|
||||||
|
|
||||||
|
def kres_to_json_files()
|
||||||
|
return "TODO"
|
||||||
|
|
||||||
|
def to_db():
|
||||||
|
return "TODO"
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
parser = argparse.ArgumentParser(description="Parsing corpora kres and ssj500k.")
|
parser = argparse.ArgumentParser(description="Parsing corpora kres and ssj500k.")
|
||||||
parser.add_argument('--kres-folder', required=True)
|
parser.add_argument('--kres-folder', required=True)
|
||||||
|
@ -23,19 +42,18 @@ if __name__ == "__main__":
|
||||||
ssj_parser = Parser(
|
ssj_parser = Parser(
|
||||||
corpus="ssj",
|
corpus="ssj",
|
||||||
infiles=[args.ssj_file],
|
infiles=[args.ssj_file],
|
||||||
output=args.output,
|
|
||||||
outdir=args.outdir,
|
|
||||||
)
|
)
|
||||||
res = [x[1]["sid"] for x in ssj_parser.sentence_generator()]
|
# res = [x[1]["sid"] for x in ssj_parser.sentence_generator()]
|
||||||
logger.info("Parsed {} sentences (ssj500k)".format(len(res)))
|
# logger.info("Parsed {} sentences (ssj500k)".format(len(res)))
|
||||||
|
|
||||||
|
# ssj to json
|
||||||
|
ssj_to_json_file(ssj_parser.sentence_generator(), "/home/voje/workdir/ssj.json")
|
||||||
|
|
||||||
# parse kres
|
# parse kres
|
||||||
logger.info("Parsing Kres: {}".format(args.ssj_file))
|
logger.info("Parsing Kres: {}".format(args.ssj_file))
|
||||||
kres_parser = Parser(
|
kres_parser = Parser(
|
||||||
corpus="kres",
|
corpus="kres",
|
||||||
infiles=[args.kres_folder, args.kres_srl_folder],
|
infiles=[args.kres_folder, args.kres_srl_folder],
|
||||||
output=args.output,
|
|
||||||
outdir=args.outdir,
|
|
||||||
)
|
)
|
||||||
res = [x[1]["sid"] for x in kres_parser.sentence_generator()]
|
res = [x[1]["sid"] for x in kres_parser.sentence_generator()]
|
||||||
logger.info("Parsed {} sentences (kres)".format(len(res)))
|
logger.info("Parsed {} sentences (kres)".format(len(res)))
|
||||||
|
@ -47,12 +65,3 @@ if __name__ == "__main__":
|
||||||
# 2. parse and save to DB
|
# 2. parse and save to DB
|
||||||
|
|
||||||
# TODO
|
# TODO
|
||||||
def handle_output(self, sent_ent, xml_file):
|
|
||||||
if self.output is None:
|
|
||||||
pass
|
|
||||||
if self.output == "file":
|
|
||||||
outfile = Path(self.outdir) / Path(xml_file.name.split(".")[0]).with_suffix(".json")
|
|
||||||
with outfile.open("a") as fp:
|
|
||||||
print(sent_ent)
|
|
||||||
json.dumps(sent_ent, fp)
|
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue
Block a user