lkrsnik/cjvt-valency
1
0
Fork 0
forked from kristjan/cjvt-valency
Code Issues Pull Requests Releases Wiki Activity

Adding processing improvemets.

Browse Source
This commit is contained in:
Luka
2020-09-23 13:02:31 +02:00
parent 220529b777
commit ce1fb46b4e
2 changed files with 45 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
scripts/create_xml.py
View File

@@ -26,7 +26,7 @@ import pymongo
# sys.path.insert(1, '/home/luka/Development/srl/cjvt_valency/src/pkg/valency')
# sys.path.insert(1, '/home/luka/Development/srl/cjvt_valency/src/pkg/cjvt-corpusparser')
from valency.Frame import frames_from_db_entry
from valency.Frame import frames_from_db_entry_headword
from valency.reduce_functions import reduce_functions
import argparse
@@ -140,11 +140,8 @@ def hws_generator(collection, headword_text, RF, mongo):
# print('tu2!')
frames = []
for ent in cur:
frames += frames_from_db_entry(ent) # pre-process this step for prod TODO
frames += frames_from_db_entry_headword(ent, headword_text) # pre-process this step for prod TODO
cur.close()
print('tu3!')
# filter by relevant hw
frames = [x for x in frames if x.hw == headword_text]
# if headword_text == 'brati':
# print('here')
@@ -197,11 +194,8 @@ def get_sentences_of_interest(headword_category, collection, w_collection, RF, m
# print('tu2!')
frames = []
for ent in cur:
frames += frames_from_db_entry(ent) # pre-process this step for prod TODO
frames += frames_from_db_entry_headword(ent, headword_text) # pre-process this step for prod TODO
cur.close()
# print('tu3!')
# filter by relevant hw
frames = [x for x in frames if x.hw == headword_text]
# if headword_text == 'brati':
# print('here')
@@ -302,7 +296,8 @@ def get_sentences_of_interest(headword_category, collection, w_collection, RF, m
# requests = [{'_id': k, 'connections': v} for k, v in sentences_of_interest.items()]
# if 'GF0010453.1116.1' in sentences_of_interest:
# print('here')
status_collection.bulk_write(status_collection_update_list)
if len(status_collection_update_list) > 0:
status_collection.bulk_write(status_collection_update_list)
requests = [UpdateOne({'_id': k}, {'$set': v}, upsert=True) for k, v in sentences_of_interest.items()]
# print('print2:')
# print(time.time() - start_time)
@@ -359,10 +354,12 @@ def get_sentences_of_interest(headword_category, collection, w_collection, RF, m
# if 'GF0010453.1116.1' in sentences_of_interest:
# a = sentences_of_interest['GF0010453.1116.1']
# print('here')
status_collection.bulk_write(status_collection_update_list)
if len(status_collection_update_list) > 0:
status_collection.bulk_write(status_collection_update_list)
requests = [UpdateOne({'_id': k}, {'$set': v}, upsert=True) for k, v in sentences_of_interest.items()]
result = w_collection.bulk_write(requests)
if len(requests) > 0:
result = w_collection.bulk_write(requests)
# sentences_of_interest = [{'_id': k, 'connections': v} for k, v in sentences_of_interest.items()]
# try:
@@ -611,10 +608,8 @@ def obtain_xml_data(collection, w_a_collection, headword_text, RF, mongo, patter
cur = collection.find({"headwords": headword_text})
frames = []
for ent in cur:
frames += frames_from_db_entry(ent) # pre-process this step for prod TODO
frames += frames_from_db_entry_headword(ent, headword_text)
cur.close()
# filter by relevant hw
frames = [x for x in frames if x.hw == headword_text]
ret_frames = RF(frames, mongo.db.sensemap)
json_ret = {"frames": []}