|
|
|
@ -67,9 +67,15 @@ def _handle_kres_file_tpl(kres_file_tpl):
|
|
|
|
|
kres_col.update({"sid": sentence["sid"]}, sentence, upsert=True)
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
# skip if one of the sentences is already in DB
|
|
|
|
|
if kres_col.find({"sid": kres_data[0]["sid"]}).count() > 0:
|
|
|
|
|
logging.info("File {} already in DB ({}/{})".format(
|
|
|
|
|
kres_file, kres_file_idx, n_kres_files))
|
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
kres_data_1 = [_db_preprocess(x) for x in kres_data]
|
|
|
|
|
kres_col.insert_many(kres_data_1) # much much better (just make sure sid has a unique index)
|
|
|
|
|
logging.info("Handled {} ({}/{}) in {:.2f} s".format(
|
|
|
|
|
logging.info("Inserted data from {} ({}/{}) in {:.2f} s".format(
|
|
|
|
|
kres_file, kres_file_idx, n_kres_files, time.time() - tstart))
|
|
|
|
|
|
|
|
|
|
def _get_dbclient(args):
|
|
|
|
|