kristjan/cjvt-corpusparser
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

bugfix

Browse Source
This commit is contained in:
voje 2019-04-21 22:24:36 +02:00
parent 792c0b03fd
commit 2582314c4d
1 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
corpusparser/main.py
View File

@ -87,10 +87,10 @@ def _handle_kres_file_chunk(kres_file_chunk):
kres_data_1 = [_db_preprocess(x) for x in kres_data]
db_payload += kres_data_1
dbclient.valdb["kres"].insert_many(db_payload) # much much better (just make sure sid has a unique index)
logger.info("Inserted kres files chunk ({}/{}) in {:.2f} s".format(
kres_chunk_idx, n_chunks, time.time() - tstart))
dbclient.close()
dbclient.valdb["kres"].insert_many(db_payload, ordered=False) # much much better (just make sure sid has a unique index)
logger.info("Inserted kres files chunk ({}/{}) in {:.2f} s".format(
kres_chunk_idx, n_chunks, time.time() - tstart))
dbclient.close()
def _get_dbclient(args):
dbclient = MongoClient(
@ -163,6 +163,7 @@ if __name__ == "__main__":
)
kres_files = [x for x in Path(args.kres_folder).iterdir()]
kres_files = sorted(kres_files, key=lambda x: x.name)
kres_files_chunks = []
i = 0
while i < len(kres_files):