prevent duplicate entries in DB

corpusparser/main.py

@@ -67,9 +67,15 @@ def _handle_kres_file_tpl(kres_file_tpl):
             kres_col.update({"sid": sentence["sid"]}, sentence, upsert=True)
         """
 
+        # skip if one of the sentences is already in DB
+        if kres_col.find({"sid": kres_data[0]["sid"]}).count() > 0:
+            logging.info("File {} already in DB ({}/{})".format(
+                kres_file, kres_file_idx, n_kres_files))
+            return
+
         kres_data_1 = [_db_preprocess(x) for x in kres_data]
         kres_col.insert_many(kres_data_1)  # much much better (just make sure sid has a unique index)
-        logging.info("Handled {} ({}/{}) in {:.2f} s".format(
+        logging.info("Inserted data from {} ({}/{}) in {:.2f} s".format(
             kres_file, kres_file_idx, n_kres_files, time.time() - tstart))
 
 def _get_dbclient(args):