243 lines
7.4 KiB
Python
243 lines
7.4 KiB
Python
# Reduction function for frames.
|
|
# Input: list of Frame objects, output: list of Frame objects.
|
|
# App uses reduce_0, 1 and 5
|
|
|
|
from valency.frame import Frame, Slot
|
|
from copy import deepcopy as DC
|
|
import logging
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
SENSE_UNDEFINED = "nedefinirano"
|
|
|
|
## TIDI: use frame.py
|
|
## TODO: build a list of [Frame] with lists of [Slot]
|
|
|
|
|
|
def sorted_by_len_tids(frames):
|
|
return sorted(
|
|
frames,
|
|
key=lambda x: len(x.tids),
|
|
reverse=True
|
|
)
|
|
|
|
|
|
def reduce_0(frames, vallex=None):
|
|
# new request... frames should be sorded by
|
|
# functors list (basically reduce_1, just each
|
|
# sentence gets its own frame)
|
|
r1_frames = reduce_1(frames)
|
|
sorting_strings = []
|
|
separated_frames = []
|
|
for frame in r1_frames:
|
|
for tid in frame.tids:
|
|
tmp_frame = DC(frame)
|
|
tmp_frame.tids = [tid]
|
|
separated_frames.append(tmp_frame)
|
|
sorting_strings.append("".join(
|
|
[slot.functor for slot in tmp_frame.slots]
|
|
))
|
|
permutation = [x for _, x in sorted(
|
|
zip(sorting_strings, range(len(sorting_strings))))]
|
|
sorted_sep_frames = [separated_frames[i] for i in permutation]
|
|
return sorted_sep_frames
|
|
|
|
|
|
def reduce_1(frames, vallex=None):
|
|
# Combine frames with the same set of functors.
|
|
# The order of functors is not important.
|
|
frame_sets = [] # [set of functors, list of frames]
|
|
for frame in frames:
|
|
functors = [slot.functor for slot in frame.slots]
|
|
|
|
for fs in frame_sets:
|
|
if set(functors) == set(fs[0]):
|
|
fs[1].append(frame)
|
|
break
|
|
else:
|
|
# Python for else -> fires if loop has ended.
|
|
frame_sets.append([functors, [frame]])
|
|
|
|
ret_frames = []
|
|
for fs in frame_sets:
|
|
tids = []
|
|
slots = {}
|
|
# All possible slots in this frame.
|
|
for functor in fs[0]:
|
|
slots[functor] = Slot(functor=functor)
|
|
# Reduce slots from all frames. (Merge ACT from all frames, ...)
|
|
for frame in fs[1]:
|
|
tids += frame.tids
|
|
for sl in frame.slots:
|
|
slots[sl.functor].tids += sl.tids
|
|
slots_list = []
|
|
for k, e in slots.items():
|
|
slots_list.append(e)
|
|
rf = Frame(tids=tids, slots=slots_list)
|
|
rf.sort_slots()
|
|
ret_frames.append(rf)
|
|
return sorted_by_len_tids(ret_frames)
|
|
|
|
|
|
def reduce_3(raw_frames, vallex):
|
|
# sskj simple lesk ids
|
|
ssj_ids = [frame.tids[0] for frame in raw_frames]
|
|
db_results = list(vallex.db.sskj_simple_lesk.find(
|
|
{"ssj_id": {"$in": ssj_ids}}))
|
|
id_map = {}
|
|
for entry in db_results:
|
|
id_map.update({entry["ssj_id"]: {
|
|
"sense_id": entry.get("sense_id"),
|
|
"sense_desc": entry.get("sense_desc")
|
|
}})
|
|
return frames_from_sense_ids(raw_frames, id_map)
|
|
|
|
|
|
def reduce_4(raw_frames, vallex):
|
|
# kmeans ids
|
|
ssj_ids = [frame.tids[0] for frame in raw_frames]
|
|
db_results = list(vallex.db.kmeans.find(
|
|
{"ssj_id": {"$in": ssj_ids}}))
|
|
id_map = {}
|
|
for entry in db_results:
|
|
id_map.update({entry["ssj_id"]: {
|
|
"sense_id": entry["sense_id"]
|
|
}})
|
|
return frames_from_sense_ids(raw_frames, id_map)
|
|
|
|
|
|
def reduce_5(raw_frames, vallex):
|
|
USER_SENSE_COLL = "v2_sense_map"
|
|
headword = raw_frames[0].hw
|
|
ssj_ids_full = [frame.tids[0] for frame in raw_frames]
|
|
# v2_sense_map stores only sentence half of ssj_id
|
|
ssj_ids = [".".join(ssj_id.split(".")[:-1]) for ssj_id in ssj_ids_full]
|
|
db_results = list(vallex.db[USER_SENSE_COLL].find({
|
|
"ssj_id": {"$in": ssj_ids},
|
|
"hw": headword,
|
|
}))
|
|
id_map = {}
|
|
for entry in db_results:
|
|
id_map[entry["ssj_id"]] = entry["sense_id"]
|
|
|
|
ret_frames = frames_from_sense_ids(raw_frames, id_map)
|
|
|
|
# sort: frames with senses to top
|
|
senses_undefined = []
|
|
senses_defined = []
|
|
for frame in ret_frames:
|
|
if frame.sense_info["sense_id"] == SENSE_UNDEFINED:
|
|
senses_undefined.append(frame)
|
|
else:
|
|
senses_defined.append(frame)
|
|
ret_frames = senses_defined + senses_undefined
|
|
|
|
return ret_frames
|
|
|
|
|
|
def frames_from_sense_ids(raw_frames, id_map):
|
|
# id map = dict {
|
|
# ssj_id: sense_id
|
|
# }
|
|
# id_dict = dict {
|
|
# sense_id: [frame, ...]
|
|
# }
|
|
id_dict = {}
|
|
for frame in raw_frames:
|
|
# long version ssj_id (S123.t12)
|
|
frame_ssj_id = frame.tids[0]
|
|
frame_sense_id = id_map.get(frame_ssj_id)
|
|
if frame_sense_id is None:
|
|
# try short version ssj_id (S123)
|
|
frame_ssj_id = ".".join(frame_ssj_id.split(".")[:-1])
|
|
frame_sense_id = id_map.get(frame_ssj_id)
|
|
|
|
# set default if sense_id not found
|
|
if frame_sense_id is None:
|
|
frame_sense_id = SENSE_UNDEFINED
|
|
"""
|
|
sense_id = id_map.get(frame.tids[0])
|
|
if sense_id is not None:
|
|
sense_id = sense_id.get("sense_id")
|
|
else:
|
|
sense_id = "nedefinirano"
|
|
"""
|
|
if frame_sense_id not in id_dict:
|
|
id_dict[frame_sense_id] = []
|
|
id_dict[frame_sense_id].append(DC(frame))
|
|
|
|
ret_frames = []
|
|
for sense_id, frames in id_dict.items():
|
|
tids = []
|
|
reduced_slots = []
|
|
for frame in frames:
|
|
tids.extend(frame.tids)
|
|
for slot in frame.slots:
|
|
# if functor not in reduced slots,
|
|
# add new slot; else increase count
|
|
for rslot in reduced_slots:
|
|
if slot.functor == rslot.functor:
|
|
rslot.count += 1
|
|
rslot.tids.extend(slot.tids)
|
|
break
|
|
else:
|
|
# in case for loop didn't match a slot
|
|
reduced_slots.append(Slot(
|
|
functor=slot.functor,
|
|
tids=slot.tids,
|
|
count=1
|
|
))
|
|
reduced_frame = Frame(tids, slots=reduced_slots)
|
|
id_map_entry = (
|
|
id_map.get(tids[0]) or
|
|
id_map.get(".".join(tids[0].split(".")[:-1]))
|
|
)
|
|
if id_map_entry is None:
|
|
reduced_frame.sense_info = {
|
|
"sense_id": SENSE_UNDEFINED,
|
|
}
|
|
else:
|
|
reduced_frame.sense_info = {
|
|
"sense_id": id_map_entry
|
|
}
|
|
reduced_frame.sort_slots()
|
|
ret_frames.append(reduced_frame)
|
|
return ret_frames
|
|
|
|
|
|
reduce_functions = {
|
|
"reduce_0": {
|
|
"f": reduce_0,
|
|
"desc":
|
|
"Vsaka pojavitev glagola dobi svoj stavčni vzorec.",
|
|
"simple_name": "posamezni stavki"
|
|
},
|
|
"reduce_1": {
|
|
"f": reduce_1,
|
|
"desc":
|
|
"Združevanje stavčnih vzorcev z enako skupino udeleženskih vlog.",
|
|
"simple_name": "združeni stavki"
|
|
},
|
|
"reduce_3": {
|
|
"f": reduce_3,
|
|
"desc":
|
|
"Združevanje stavčnih vzorcev na osnovi pomenov povedi v SSKJ. "
|
|
"Pomeni so dodeljeni s pomočjo algoritma Simple Lesk.",
|
|
"simple_name": "SSKJ_pomeni"
|
|
},
|
|
"reduce_4": {
|
|
"f": reduce_4,
|
|
"desc":
|
|
"Združevanje stavčnih vzorcev na osnovi pomenov povedi "
|
|
"s pomočjo algoritma K-Means. Število predvidenih pomenov "
|
|
"podano na osnovi SSKJ.",
|
|
"simple_name": "KMeans_pomeni"
|
|
},
|
|
"reduce_5": {
|
|
"f": reduce_5,
|
|
"desc":
|
|
"Uporabniško dodeljeni pomeni povedi.",
|
|
"simple_name": "po meri"
|
|
}
|
|
}
|