kristjan/cjvt-corpusparser
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

prevent duplicate entries in DB

Browse Source
This commit is contained in:
voje 2019-04-15 20:48:10 +02:00
parent f0b0abac1b
commit af4f6045bb
1 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
corpusparser/main.py
View File

@ -67,9 +67,15 @@ def _handle_kres_file_tpl(kres_file_tpl):
kres_col.update({"sid": sentence["sid"]}, sentence, upsert=True)
"""
# skip if one of the sentences is already in DB
if kres_col.find({"sid": kres_data[0]["sid"]}).count() > 0:
logging.info("File {} already in DB ({}/{})".format(
kres_file, kres_file_idx, n_kres_files))
return
kres_data_1 = [_db_preprocess(x) for x in kres_data]
kres_col.insert_many(kres_data_1) # much much better (just make sure sid has a unique index)
logging.info("Handled {} ({}/{}) in {:.2f} s".format(
logging.info("Inserted data from {} ({}/{}) in {:.2f} s".format(
kres_file, kres_file_idx, n_kres_files, time.time() - tstart))
def _get_dbclient(args):