|
|
@ -99,6 +99,27 @@ def _get_dbclient(args):
|
|
|
|
)
|
|
|
|
)
|
|
|
|
return dbclient
|
|
|
|
return dbclient
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# wrap it in a function for better garbage collection
|
|
|
|
|
|
|
|
def parse_ssj(args):
|
|
|
|
|
|
|
|
logger.info("Parsing Ssj: {}".format(args.ssj_file))
|
|
|
|
|
|
|
|
ssj_parser = Parser(logger=logger, corpus="ssj")
|
|
|
|
|
|
|
|
ssj_data = ssj_parser.parse_xml_file(Path(args.ssj_file))
|
|
|
|
|
|
|
|
if args.output == "file":
|
|
|
|
|
|
|
|
ssj_outfile = outdir / "ssj500k.json"
|
|
|
|
|
|
|
|
with ssj_outfile.open("w") as fp:
|
|
|
|
|
|
|
|
json.dump(ssj_data, fp)
|
|
|
|
|
|
|
|
elif args.output == "db":
|
|
|
|
|
|
|
|
dbclient = _get_dbclient(args)
|
|
|
|
|
|
|
|
valdb = dbclient.valdb
|
|
|
|
|
|
|
|
ssj_col = valdb["ssj"]
|
|
|
|
|
|
|
|
for sentence in ssj_data:
|
|
|
|
|
|
|
|
sentence = _db_preprocess(sentence)
|
|
|
|
|
|
|
|
ssj_col.update({"sid": sentence["sid"]}, sentence, upsert=True)
|
|
|
|
|
|
|
|
del ssj_parser
|
|
|
|
|
|
|
|
del ssj_data
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
if __name__ == "__main__":
|
|
|
|
parser = argparse.ArgumentParser(description="Parsing corpora kres and ssj500k.")
|
|
|
|
parser = argparse.ArgumentParser(description="Parsing corpora kres and ssj500k.")
|
|
|
|
parser.add_argument('--kres-folder', required=True)
|
|
|
|
parser.add_argument('--kres-folder', required=True)
|
|
|
@ -124,22 +145,12 @@ if __name__ == "__main__":
|
|
|
|
dbclient.valdb[corpus].ensure_index([("headwords", pymongo.ASCENDING)])
|
|
|
|
dbclient.valdb[corpus].ensure_index([("headwords", pymongo.ASCENDING)])
|
|
|
|
dbclient.valdb[corpus].ensure_index([("functors", pymongo.ASCENDING)])
|
|
|
|
dbclient.valdb[corpus].ensure_index([("functors", pymongo.ASCENDING)])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# SSJ
|
|
|
|
# SSJ
|
|
|
|
logger.info("Parsing Ssj: {}".format(args.ssj_file))
|
|
|
|
p = Pool(1)
|
|
|
|
ssj_parser = Parser(logger=logger, corpus="ssj")
|
|
|
|
p.map(parse_ssj, [args])
|
|
|
|
ssj_data = ssj_parser.parse_xml_file(Path(args.ssj_file))
|
|
|
|
|
|
|
|
if args.output == "file":
|
|
|
|
|
|
|
|
ssj_outfile = outdir / "ssj500k.json"
|
|
|
|
|
|
|
|
with ssj_outfile.open("w") as fp:
|
|
|
|
|
|
|
|
json.dump(ssj_data, fp)
|
|
|
|
|
|
|
|
elif args.output == "db":
|
|
|
|
|
|
|
|
dbclient = _get_dbclient(args)
|
|
|
|
|
|
|
|
valdb = dbclient.valdb
|
|
|
|
|
|
|
|
ssj_col = valdb["ssj"]
|
|
|
|
|
|
|
|
for sentence in ssj_data:
|
|
|
|
|
|
|
|
sentence = _db_preprocess(sentence)
|
|
|
|
|
|
|
|
ssj_col.update({"sid": sentence["sid"]}, sentence, upsert=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
time.sleep(30)
|
|
|
|
|
|
|
|
|
|
|
|
# Kres
|
|
|
|
# Kres
|
|
|
|
logger.info("Parsing Kres: {}".format(args.kres_folder))
|
|
|
|
logger.info("Parsing Kres: {}".format(args.kres_folder))
|
|
|
|