diff --git a/corpusparser/main.py b/corpusparser/main.py index d7699b9..2998a69 100644 --- a/corpusparser/main.py +++ b/corpusparser/main.py @@ -67,9 +67,15 @@ def _handle_kres_file_tpl(kres_file_tpl): kres_col.update({"sid": sentence["sid"]}, sentence, upsert=True) """ + # skip if one of the sentences is already in DB + if kres_col.find({"sid": kres_data[0]["sid"]}).count() > 0: + logging.info("File {} already in DB ({}/{})".format( + kres_file, kres_file_idx, n_kres_files)) + return + kres_data_1 = [_db_preprocess(x) for x in kres_data] kres_col.insert_many(kres_data_1) # much much better (just make sure sid has a unique index) - logging.info("Handled {} ({}/{}) in {:.2f} s".format( + logging.info("Inserted data from {} ({}/{}) in {:.2f} s".format( kres_file, kres_file_idx, n_kres_files, time.time() - tstart)) def _get_dbclient(args):