diff --git a/.gitignore b/.gitignore index 3feb78a..37e399d 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ *.egg-info/ +*/__pycache__/ diff --git a/corpusparser/main.py b/corpusparser/main.py index e0ba065..9eaaa42 100644 --- a/corpusparser/main.py +++ b/corpusparser/main.py @@ -3,6 +3,8 @@ from corpusparser import Parser import argparse import logging import json +from pymongo import MongoClient +import sys logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -54,11 +56,23 @@ def kres_to_json_files(sentence_generator, outfolder): outfile_buffer += [sdata] write_buffer_to_file(current_outfile, outfile_buffer) +def data_to_valdb(sentence_generator, dbaddr, username, password, collection_name): + logger.info("Connecting to: {}".format(dbaddr)) + client = MongoClient( + "mongodb://{}".format(dbaddr), + username=username, + password=password, + authSource="valdb", + authMechanism='SCRAM-SHA-256' + ) + valdb = client.valdb + logger.info("Writing data to {}.".format(collection_name)) + col = valdb[collection_name] + for s in sentence_generator: + sdata = s[1] + col.insert_one(sdata) -def to_db(): - return "TODO" - if __name__ == "__main__": parser = argparse.ArgumentParser(description="Parsing corpora kres and ssj500k.") parser.add_argument('--kres-folder', required=True) @@ -67,36 +81,44 @@ if __name__ == "__main__": parser.add_argument('--output', required=False, default=None) parser.add_argument('--outdir', required=False, default=None) parser.add_argument('--dbaddr', required=False, default=None) + parser.add_argument('--dbuser', required=False, default=None) + parser.add_argument('--dbpass', required=False, default=None) args = parser.parse_args() + # parse ssj logger.info("Parsing ssj500k: {}".format(args.ssj_file)) ssj_parser = Parser( corpus="ssj", infiles=[args.ssj_file], ) - # res = [x[1]["sid"] for x in ssj_parser.sentence_generator()] - # logger.info("Parsed {} sentences (ssj500k)".format(len(res))) - # ssj to json - ssj_to_json_file(ssj_parser.sentence_generator(), args.outdir) + if args.output == "file": + ssj_to_json_file(ssj_parser.sentence_generator(), args.outdir) + elif args.output == "db": + data_to_valdb( + ssj_parser.sentence_generator(), + args.dbaddr, + args.dbuser, + args.dbpass, + collection_name="ssj" + ) + sys.exit() # parse kres logger.info("Parsing Kres: {}".format(args.ssj_file)) kres_parser = Parser( corpus="kres", infiles=[args.kres_folder, args.kres_srl_folder], ) - # res = [x[1]["sid"] for x in kres_parser.sentence_generator()] - # logger.info("Parsed {} sentences (kres)".format(len(res))) - # kres to json - kres_to_json_files(kres_parser.sentence_generator(), args.outdir) - - -## Handling output is situational --- implement it outside of Parser. -## Parser returns tuples (orig_file, element) -# 1. parse per-file and output to file (JSON) -# 2. parse and save to DB - -# TODO + if args.output == "file": + kres_to_json_files(kres_parser.sentence_generator(), args.outdir) + elif args.output == "db": + data_to_valdb( + ssj_parser.sentence_generator(), + args.dbaddr, + args.dbuser, + args.dbpass, + collection_name="kres" + ) diff --git a/dockerfiles/cjvt-corpusparser-env/Dockerfile b/dockerfiles/cjvt-corpusparser-env/Dockerfile index e44a6e6..6951dbf 100644 --- a/dockerfiles/cjvt-corpusparser-env/Dockerfile +++ b/dockerfiles/cjvt-corpusparser-env/Dockerfile @@ -10,6 +10,7 @@ sshfs RUN pip3 install \ lxml \ argparse \ - pathlib + pathlib \ + pymongo ENV PYTHONIOENCODING UTF-8