lkrsnik/cjvt-valency
1
0
Fork 0
forked from kristjan/cjvt-valency
Code Issues Pull Requests Releases Wiki Activity

Parameterized mongo_db + Added internal state saving for p1

Browse Source
This commit is contained in:
Luka
2020-09-22 19:31:31 +02:00
parent ae5f2869bc
commit 220529b777
2 changed files with 27 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
scripts/create_xml.py
View File

@@ -158,7 +158,7 @@ def hws_generator(collection, headword_text, RF, mongo):
yield frame_json
def get_sentences_of_interest(headword_category, collection, w_collection, RF, mongo, pbar):
def get_sentences_of_interest(headword_category, collection, w_collection, RF, mongo, pbar, status_collection, corpus_type):
sentences_of_interest = {}
# all_sentences = set()
sorted(headword_category, key=lambda x: x[0])
@@ -174,8 +174,14 @@ def get_sentences_of_interest(headword_category, collection, w_collection, RF, m
# last_processed_hw = 'aktivirati'
# last_processed_hw = 'aktivirati'
status_collection_update_list = []
# already_processed = False
for headword_id, (headword_text, category_text) in enumerate(headword_category):
# check whether element has been processed
if status_collection.count_documents({'corpus_type': corpus_type, 'headword_text': headword_text, 'part': 'p1'}):
pbar.update(1)
continue
# print(headword_text)
# if already_processed:
# if headword_text != last_processed_hw:
@@ -296,6 +302,7 @@ def get_sentences_of_interest(headword_category, collection, w_collection, RF, m
# requests = [{'_id': k, 'connections': v} for k, v in sentences_of_interest.items()]
# if 'GF0010453.1116.1' in sentences_of_interest:
# print('here')
status_collection.bulk_write(status_collection_update_list)
requests = [UpdateOne({'_id': k}, {'$set': v}, upsert=True) for k, v in sentences_of_interest.items()]
# print('print2:')
# print(time.time() - start_time)
@@ -305,7 +312,7 @@ def get_sentences_of_interest(headword_category, collection, w_collection, RF, m
# print('print3:')
# print(time.time() - start_time)
# start_time = time.time()
del status_collection_update_list
del requests
del sentences_of_interest
gc.collect()
@@ -339,17 +346,20 @@ def get_sentences_of_interest(headword_category, collection, w_collection, RF, m
# w_collection.insert_many(sentences_of_interest, ordered=False)
# except pymongo.errors.BulkWriteError as e:
# print(e.details['writeErrors'])
status_collection_update_list = []
sentences_of_interest = {}
# first_sentence = True
sentences_in_ram += 1
pbar.update(1)
status_collection_update_list.append(InsertOne({'corpus_type': corpus_type, 'headword_text': headword_text, 'part': 'p1'}))
# TODO uncomment
# if 'GF0010453.1116.1' in sentences_of_interest:
# a = sentences_of_interest['GF0010453.1116.1']
# print('here')
status_collection.bulk_write(status_collection_update_list)
requests = [UpdateOne({'_id': k}, {'$set': v}, upsert=True) for k, v in sentences_of_interest.items()]
result = w_collection.bulk_write(requests)
@@ -1467,7 +1477,10 @@ def main(args):
print('beginning chunk')
start_time = time.time()
# user:user:valdb:127.0.0.1
mongo = MongoClient(username='user', password='user', authSource='valdb')
[db_user, db_password, db_database, db_host] = args.mongo_db.split(':')
mongo = MongoClient(username=db_user, password=db_password, authSource=db_database)
db = mongo.valdb
collection_ssj = db['ssj']
@@ -1479,6 +1492,7 @@ def main(args):
w_collection_gigafida = db2['gigafida']
w_a_collection_ssj = db2['ssj' + '_all']
w_a_collection_gigafida = db2['gigafida' + '_all']
status_collection = db2['status']
valency_pattern_id_collection = db2['valency_pattern_ids']
@@ -1527,10 +1541,10 @@ def main(args):
# sentences_of_interest_stored = args.p1_processed
if not args.p1_processed:
with tqdm(total=len(headword_category)) as pbar:
get_sentences_of_interest(headword_category, collection_ssj, w_collection_ssj, RF, mongo, pbar)
get_sentences_of_interest(headword_category, collection_ssj, w_collection_ssj, RF, mongo, pbar, status_collection, 'ssj')
if not args.ignore_gigafida:
with tqdm(total=len(headword_category)) as pbar:
get_sentences_of_interest(headword_category, collection_gigafida, w_collection_gigafida, RF, mongo, pbar)
get_sentences_of_interest(headword_category, collection_gigafida, w_collection_gigafida, RF, mongo, pbar, status_collection, 'gigafida')
# sentences_of_interest = OrderedDict(sorted(sentences_of_interest.items()))
print(time.time() - start_time)
# num_sentences = 0
@@ -1568,6 +1582,7 @@ def main(args):
if __name__ == '__main__':
arg_parser = argparse.ArgumentParser(description='Export and validate collocation data from DDD database.')
arg_parser.add_argument('--sloleks_db', type=str, help='Database credentials')
arg_parser.add_argument('--mongo_db', type=str, help='Database credentials')
arg_parser.add_argument('--schema', type=str, help='XML schema')
arg_parser.add_argument('--infile', type=str, help='Input file')
arg_parser.add_argument('--outdir', type=str, help='Output directory')