kristjan/cjvt-corpusparser
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

separate parsing of ssj and kres (mem management)

Browse Source
This commit is contained in:
voje 2019-04-21 19:18:19 +02:00
parent 00d9192993
commit bcc64c767c
1 changed files with 46 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
corpusparser/main.py
View File

@ -9,11 +9,13 @@ import sys
from multiprocessing import Pool from multiprocessing import Pool
import time import time
CORPORA = ["kres", "ssj"]
# logging.basicConfig(filename=Path("/var/tmp/corpusparser.log"), filemode='a', level=logging.INFO) # logging.basicConfig(filename=Path("/var/tmp/corpusparser.log"), filemode='a', level=logging.INFO)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# lfh = logging.FileHandler("/var/tmp/fill-database.log") lfh = logging.FileHandler("/project/logs/fill-database.log")
lfh = logging.StreamHandler(sys.stdout) # lfh = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s") formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
lfh.setFormatter(formatter) lfh.setFormatter(formatter)
logger.addHandler(lfh) logger.addHandler(lfh)
@ -100,8 +102,37 @@ def _get_dbclient(args):
return dbclient return dbclient
# wrap it in a function for better garbage collection if __name__ == "__main__":
def parse_ssj(args): parser = argparse.ArgumentParser(description="Parsing corpora kres and ssj500k.")
parser.add_argument('--corpus', required=True)
parser.add_argument('--kres-folder', required=False)
parser.add_argument('--kres-srl-folder', required=False)
parser.add_argument('--ssj-file', required=False)
parser.add_argument('--output', required=False, default=None)
parser.add_argument('--outdir', required=False, default=None)
parser.add_argument('--dbaddr', required=False, default=None)
parser.add_argument('--dbuser', required=False, default=None)
parser.add_argument('--dbpass', required=False, default=None)
parser.add_argument('--cores', required=False, default=1)
args = parser.parse_args()
corpus = args.corpus
assert (corpus in CORPORA), "Wrong corpus name."
outdir = None
if args.output == "file":
outdir = Path(args.outdir)
outdir.mkdir(parents=True, exist_ok=True)
elif args.output == "db":
dbclient = _get_dbclient(args)
dbclient.valdb[corpus].ensure_index([("sid", pymongo.ASCENDING)])
dbclient.valdb[corpus].ensure_index([("headwords", pymongo.ASCENDING)])
dbclient.valdb[corpus].ensure_index([("functors", pymongo.ASCENDING)])
if corpus == "ssj":
logger.info("Parsing Ssj: {}".format(args.ssj_file)) logger.info("Parsing Ssj: {}".format(args.ssj_file))
ssj_parser = Parser(logger=logger, corpus="ssj") ssj_parser = Parser(logger=logger, corpus="ssj")
ssj_data = ssj_parser.parse_xml_file(Path(args.ssj_file)) ssj_data = ssj_parser.parse_xml_file(Path(args.ssj_file))
@ -116,42 +147,9 @@ def parse_ssj(args):
for sentence in ssj_data: for sentence in ssj_data:
sentence = _db_preprocess(sentence) sentence = _db_preprocess(sentence)
ssj_col.update({"sid": sentence["sid"]}, sentence, upsert=True) ssj_col.update({"sid": sentence["sid"]}, sentence, upsert=True)
del ssj_parser
del ssj_data
if __name__ == "__main__": if corpus == "kres":
parser = argparse.ArgumentParser(description="Parsing corpora kres and ssj500k.")
parser.add_argument('--kres-folder', required=True)
parser.add_argument('--kres-srl-folder', required=True)
parser.add_argument('--ssj-file', required=True)
parser.add_argument('--output', required=False, default=None)
parser.add_argument('--outdir', required=False, default=None)
parser.add_argument('--dbaddr', required=False, default=None)
parser.add_argument('--dbuser', required=False, default=None)
parser.add_argument('--dbpass', required=False, default=None)
parser.add_argument('--cores', required=False, default=1)
args = parser.parse_args()
outdir = None
if args.output == "file":
outdir = Path(args.outdir)
outdir.mkdir(parents=True, exist_ok=True)
elif args.output == "db":
# Force unique sid
dbclient = _get_dbclient(args)
for corpus in ["kres", "ssj"]:
dbclient.valdb[corpus].ensure_index([("sid", pymongo.ASCENDING)])
dbclient.valdb[corpus].ensure_index([("headwords", pymongo.ASCENDING)])
dbclient.valdb[corpus].ensure_index([("functors", pymongo.ASCENDING)])
# SSJ
p = Pool(1)
p.map(parse_ssj, [args])
time.sleep(30)
# Kres # Kres
logger.info("Parsing Kres: {}".format(args.kres_folder)) logger.info("Parsing Kres: {}".format(args.kres_folder))
kres_parser = Parser( kres_parser = Parser(
@ -168,4 +166,5 @@ if __name__ == "__main__":
p = Pool(int(args.cores)) p = Pool(int(args.cores))
p.map(_handle_kres_file_tpl, kres_files) p.map(_handle_kres_file_tpl, kres_files)
logger.info("Finished parsing.") logger.info("Finished parsing.")