update instead of insert, fixing sentence duplication in db

5 years ago · 2b7339ac5a
parent 77c599dded
commit 2b7339ac5a
1 changed files with 2 additions and 1 deletions
--- a/corpusparser/main.py
+++ b/corpusparser/main.py
@ -70,7 +70,8 @@ def data_to_valdb(sentence_generator, dbaddr, username, password, collection_nam
    col = valdb[collection_name]
    for s in sentence_generator:
        sdata = s[1]
-        col.insert_one(sdata)
+        # col.insert_one(sdata)
+        col.update({"sid": sdata["sid"]}, sdata, upsert=True)


 if __name__ == "__main__":