ozbolt/luscenje_struktur
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

EMPTY COMMIT - removing trailing whitespace

Browse Source
This commit is contained in:
Ozbolt Menegatti 2019-06-08 11:42:57 +02:00
parent 3a22cd91c3
commit 797060f619
1 changed files with 53 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

104
wani.py
View File

@ -142,9 +142,9 @@ class Order(Enum):
def new(order): def new(order):
if order is not None: if order is not None:
if order == "to-from": if order == "to-from":
return Order.ToFrom return Order.ToFrom
elif order == "from-to": elif order == "from-to":
return Order.FromTo return Order.FromTo
else: else:
raise NotImplementedError("What kind of ordering is: {}".format(order)) raise NotImplementedError("What kind of ordering is: {}".format(order))
else: else:
@ -174,7 +174,7 @@ class ComponentRepresentation:
self.words = [] self.words = []
self.rendition_text = None self.rendition_text = None
self.agreement = [] self.agreement = []
def get_agreement(self): def get_agreement(self):
return [] return []
@ -195,7 +195,7 @@ class LemmaCR(ComponentRepresentation):
class LexisCR(ComponentRepresentation): class LexisCR(ComponentRepresentation):
def _render(self): def _render(self):
return self.data['lexis'] return self.data['lexis']
class WordFormAllCR(ComponentRepresentation): class WordFormAllCR(ComponentRepresentation):
def _render(self): def _render(self):
if len(self.words) == 0: if len(self.words) == 0:
@ -228,7 +228,7 @@ class WordFormAnyCR(ComponentRepresentation):
return None return None
else: else:
return text_forms[(word_msd, word_lemma)] return text_forms[(word_msd, word_lemma)]
class WordFormMsdCR(WordFormAnyCR): class WordFormMsdCR(WordFormAnyCR):
def __init__(self, *args): def __init__(self, *args):
super().__init__(*args) super().__init__(*args)
@ -251,7 +251,7 @@ class WordFormMsdCR(WordFormAnyCR):
return True return True
pass pass
def add_word(self, word): def add_word(self, word):
if self.lemma is None: if self.lemma is None:
self.lemma = word.lemma self.lemma = word.lemma
@ -259,7 +259,7 @@ class WordFormMsdCR(WordFormAnyCR):
if self.check_msd(word.msd): if self.check_msd(word.msd):
super().add_word(word) super().add_word(word)
def _render(self): def _render(self):
msd = self.word_renderer.get_lemma_msd(self.lemma, self.msd) msd = self.word_renderer.get_lemma_msd(self.lemma, self.msd)
WordLemma = namedtuple('WordLemmaOnly', 'msd most_frequent_text lemma text') WordLemma = namedtuple('WordLemmaOnly', 'msd most_frequent_text lemma text')
@ -272,10 +272,10 @@ class WordFormAgreementCR(WordFormMsdCR):
def __init__(self, data, word_renderer): def __init__(self, data, word_renderer):
super().__init__(data, word_renderer) super().__init__(data, word_renderer)
self.rendition_candidate = None self.rendition_candidate = None
def get_agreement(self): def get_agreement(self):
return self.data['other'] return self.data['other']
def match(self, word_msd): def match(self, word_msd):
existing = [(w.msd, w.text) for w in self.words] existing = [(w.msd, w.text) for w in self.words]
@ -289,7 +289,7 @@ class WordFormAgreementCR(WordFormMsdCR):
return True return True
return False return False
def confirm_match(self): def confirm_match(self):
self.rendition_text = self.rendition_candidate self.rendition_text = self.rendition_candidate
@ -304,8 +304,8 @@ class WordFormAgreementCR(WordFormMsdCR):
v1 = TAGSET[t1].index(agr_case) v1 = TAGSET[t1].index(agr_case)
# if none specified: nedolocnik, always agrees # if none specified: nedolocnik, always agrees
if v1 + 1 >= len(msd1): if v1 + 1 >= len(msd1):
continue continue
# first is uppercase, not in TAGSET # first is uppercase, not in TAGSET
m1 = msd1[v1 + 1] m1 = msd1[v1 + 1]
@ -315,8 +315,8 @@ class WordFormAgreementCR(WordFormMsdCR):
logging.warning("Cannot do agreement: {} for msd {} not found!".format(agr_case, msd2)) logging.warning("Cannot do agreement: {} for msd {} not found!".format(agr_case, msd2))
return False return False
v2 = TAGSET[t2].index(agr_case) v2 = TAGSET[t2].index(agr_case)
if v2 + 1 >= len(msd2): if v2 + 1 >= len(msd2):
continue continue
m2 = msd2[v2 + 1] m2 = msd2[v2 + 1]
# match! # match!
@ -324,7 +324,7 @@ class WordFormAgreementCR(WordFormMsdCR):
return False return False
return True return True
def render(self): def render(self):
pass pass
@ -333,7 +333,7 @@ class ComponentRendition:
def __init__(self): def __init__(self):
self.more = {} self.more = {}
self.representation_factory = ComponentRepresentation self.representation_factory = ComponentRepresentation
def add_feature(self, feature): def add_feature(self, feature):
if 'rendition' in feature: if 'rendition' in feature:
if feature['rendition'] == "lemma": if feature['rendition'] == "lemma":
@ -366,10 +366,10 @@ class ComponentRendition:
else: else:
return None return None
def cr_instance(self, word_renderer): def cr_instance(self, word_renderer):
return self.representation_factory(self.more, word_renderer) return self.representation_factory(self.more, word_renderer)
@staticmethod @staticmethod
def set_representations(matches, structure, word_renderer): def set_representations(matches, structure, word_renderer):
representations = {} representations = {}
@ -377,7 +377,7 @@ class ComponentRendition:
representations[c.idx] = [] representations[c.idx] = []
for rep in c.representation: for rep in c.representation:
representations[c.idx].append(rep.cr_instance(word_renderer)) representations[c.idx].append(rep.cr_instance(word_renderer))
for cid, reps in representations.items(): for cid, reps in representations.items():
for rep in reps: for rep in reps:
for agr in rep.get_agreement(): for agr in rep.get_agreement():
@ -514,7 +514,7 @@ class Restriction:
self.matcher = None self.matcher = None
self.present = None self.present = None
return return
restriction_type = restriction_tag.get('type') restriction_type = restriction_tag.get('type')
if restriction_type == "morphology": if restriction_type == "morphology":
self.type = RestrictionType.Morphology self.type = RestrictionType.Morphology
@ -620,7 +620,7 @@ class Component:
mn = self._match_next(word) mn = self._match_next(word)
if mn is None: if mn is None:
return None return None
to_ret = [m1] to_ret = [m1]
for cmatch in mn: for cmatch in mn:
# if good match but nothing to add, just continue # if good match but nothing to add, just continue
@ -660,7 +660,7 @@ class Component:
# need to get all links that match # need to get all links that match
for next, link, order in self.next_element: for next, link, order in self.next_element:
next_links = word.get_links(link) next_links = word.get_links(link)
to_ret.append([]) to_ret.append([])
# good flag # good flag
@ -700,7 +700,7 @@ class SyntacticStructure:
st = SyntacticStructure() st = SyntacticStructure()
st.id = xml.get('id') st.id = xml.get('id')
st.lbs = xml.get('LBS') st.lbs = xml.get('LBS')
assert(len(list(xml)) == 1) assert(len(list(xml)) == 1)
system = next(iter(xml)) system = next(iter(xml))
@ -731,7 +731,7 @@ class SyntacticStructure:
st.determine_core2w() st.determine_core2w()
return st return st
def determine_core2w(self): def determine_core2w(self):
ppb_components = [] ppb_components = []
for c in self.components: for c in self.components:
@ -777,7 +777,7 @@ class SyntacticStructure:
def load_structures(filename): def load_structures(filename):
with open(filename, 'r') as fp: with open(filename, 'r') as fp:
et = ElementTree.XML(fp.read()) et = ElementTree.XML(fp.read())
return build_structures(et), get_lemma_features(et) return build_structures(et), get_lemma_features(et)
def build_structures(et): def build_structures(et):
@ -807,7 +807,7 @@ def get_lemma_features(et):
rgx_str += position[1] rgx_str += position[1]
else: else:
raise RuntimeError("Strange rgx for lemma_feature...") raise RuntimeError("Strange rgx for lemma_feature...")
assert(rgx_str[0].isupper()) assert(rgx_str[0].isupper())
result[rgx_str[0]] = rgx_str.strip().replace(' ', '-') result[rgx_str[0]] = rgx_str.strip().replace(' ', '-')
@ -860,7 +860,7 @@ class Word:
self.links[link].extend(self.links[l]) self.links[link].extend(self.links[l])
return self.links[link] return self.links[link]
def most_frequent_text(self, word_renderer): def most_frequent_text(self, word_renderer):
return word_renderer.render(self.lemma, self.msd) return word_renderer.render(self.lemma, self.msd)
@ -873,13 +873,13 @@ class WordMsdRenderer:
self.lemma_msd = {} self.lemma_msd = {}
self.lemma_features = lemma_features self.lemma_features = lemma_features
self.memoized_msd_merges = {} self.memoized_msd_merges = {}
def add_words(self, words): def add_words(self, words):
self.all_words.extend(words) self.all_words.extend(words)
def num_all_words(self): def num_all_words(self):
return len(self.all_words) return len(self.all_words)
def generate_renders(self): def generate_renders(self):
data = defaultdict(lambda: defaultdict(list)) data = defaultdict(lambda: defaultdict(list))
for w in self.all_words: for w in self.all_words:
@ -898,15 +898,15 @@ class WordMsdRenderer:
for txt in texts: for txt in texts:
freq_words[(msd, txt)] += 1 freq_words[(msd, txt)] += 1
common_msd = self.merge_msd(common_msd, msd) common_msd = self.merge_msd(common_msd, msd)
self.lemma_msd[lemma] = common_msd self.lemma_msd[lemma] = common_msd
self.frequent_words[lemma] = [] self.frequent_words[lemma] = []
for (msd, txt), n in sorted(freq_words.items(), key=lambda x: -x[1]): for (msd, txt), n in sorted(freq_words.items(), key=lambda x: -x[1]):
self.frequent_words[lemma].append((msd, txt, n)) self.frequent_words[lemma].append((msd, txt, n))
lf = self.lemma_features lf = self.lemma_features
for lemma in self.lemma_msd.keys(): for lemma in self.lemma_msd.keys():
cmsd = self.lemma_msd[lemma] cmsd = self.lemma_msd[lemma]
@ -914,7 +914,7 @@ class WordMsdRenderer:
self.lemma_msd[lemma] = "".join( self.lemma_msd[lemma] = "".join(
l1 if l1 != "-" else l2 for l1, l2 in zip(lf[cmsd[0]], cmsd) l1 if l1 != "-" else l2 for l1, l2 in zip(lf[cmsd[0]], cmsd)
) )
def merge_msd(self, common_msd, new_msd): def merge_msd(self, common_msd, new_msd):
key = (common_msd, new_msd) key = (common_msd, new_msd)
if key in self.memoized_msd_merges: if key in self.memoized_msd_merges:
@ -931,12 +931,12 @@ class WordMsdRenderer:
value = "".join(merge_letter(l1, l2) for l1, l2 in zip(common_msd, new_msd)) value = "".join(merge_letter(l1, l2) for l1, l2 in zip(common_msd, new_msd))
self.memoized_msd_merges[key] = value self.memoized_msd_merges[key] = value
return value return value
def render(self, lemma, msd): def render(self, lemma, msd):
if lemma in self.rendered_words: if lemma in self.rendered_words:
if msd in self.rendered_words[lemma]: if msd in self.rendered_words[lemma]:
return self.rendered_words[lemma][msd][0] return self.rendered_words[lemma][msd][0]
def available_words(self, lemma, existing_texts): def available_words(self, lemma, existing_texts):
counted_texts = Counter(existing_texts) counted_texts = Counter(existing_texts)
for (msd, text), _n in counted_texts.most_common(): for (msd, text), _n in counted_texts.most_common():
@ -946,7 +946,7 @@ class WordMsdRenderer:
for msd, text, _ in self.frequent_words[lemma]: for msd, text, _ in self.frequent_words[lemma]:
if (msd, text) not in counted_texts: if (msd, text) not in counted_texts:
yield (msd, text) yield (msd, text)
def get_lemma_msd(self, lemma, word_msd): def get_lemma_msd(self, lemma, word_msd):
# should be here, since we collect every lemmas # should be here, since we collect every lemmas
lemma_msd = self.lemma_msd[lemma] lemma_msd = self.lemma_msd[lemma]
@ -1024,7 +1024,7 @@ class Writer:
@staticmethod @staticmethod
def make_output_writer(args): def make_output_writer(args):
return Writer(False, args.output, args.multiple_output, int(args.sort_by), args.sort_reversed) return Writer(False, args.output, args.multiple_output, int(args.sort_by), args.sort_reversed)
@staticmethod @staticmethod
def make_all_writer(args): def make_all_writer(args):
return Writer(True, args.all, False, -1, False) return Writer(True, args.all, False, -1, False)
@ -1047,7 +1047,7 @@ class Writer:
assert(len(cols) == self.length()) assert(len(cols) == self.length())
cols = ["C{}_{}".format(i + 1, thd) for i in range(MAX_NUM_COMPONENTS) for thd in cols] cols = ["C{}_{}".format(i + 1, thd) for i in range(MAX_NUM_COMPONENTS) for thd in cols]
cols = ["Structure_ID"] + cols + ["Colocation_ID"] cols = ["Structure_ID"] + cols + ["Colocation_ID"]
if not self.all: if not self.all:
cols += ["Joint_representative_form", "Frequency"] cols += ["Joint_representative_form", "Frequency"]
@ -1067,7 +1067,7 @@ class Writer:
return [word.lemma, word.lemma, "lemma_fallback"] return [word.lemma, word.lemma, "lemma_fallback"]
else: else:
return [word.lemma, representation, "ok"] return [word.lemma, representation, "ok"]
def sorted_rows(self, rows): def sorted_rows(self, rows):
if self.sort_by < 0 or len(rows) < 2: if self.sort_by < 0 or len(rows) < 2:
return rows return rows
@ -1075,7 +1075,7 @@ class Writer:
if len(rows[0]) <= self.sort_by: if len(rows[0]) <= self.sort_by:
logging.warning("Cannot sort by column #{}: Not enough columns!".format(len(rows[0]))) logging.warning("Cannot sort by column #{}: Not enough columns!".format(len(rows[0])))
return rows return rows
try: try:
int(rows[0][self.sort_by]) int(rows[0][self.sort_by])
key=lambda row: int(row[self.sort_by]) key=lambda row: int(row[self.sort_by])
@ -1103,7 +1103,7 @@ class Writer:
representation += " " + to_write[-2] representation += " " + to_write[-2]
# make them equal size # make them equal size
to_write.extend([""] * (MAX_NUM_COMPONENTS * self.length() - len(to_write))) to_write.extend([""] * (MAX_NUM_COMPONENTS * self.length() - len(to_write)))
to_write = [structure_id] + to_write + [cid] to_write = [structure_id] + to_write + [cid]
if not self.all: if not self.all:
@ -1144,7 +1144,7 @@ class Writer:
if self.multiple_output: if self.multiple_output:
fp_close(fp) fp_close(fp)
if not self.multiple_output: if not self.multiple_output:
fp_close(fp) fp_close(fp)
@ -1155,7 +1155,7 @@ class StructureMatch:
self.matches = [] self.matches = []
self.representations = {} self.representations = {}
def distinct_matches(self): def distinct_matches(self):
dm = set() dm = set()
keys = list(self.matches[0].keys()) keys = list(self.matches[0].keys())
@ -1179,7 +1179,7 @@ class ColocationIds:
if key not in self.data: if key not in self.data:
self.data[key] = StructureMatch(str(len(self.data) + 1), sid) self.data[key] = StructureMatch(str(len(self.data) + 1), sid)
self.data[key].append(match) self.data[key].append(match)
def get(self, key, n): def get(self, key, n):
return self.data[key][n] return self.data[key][n]
@ -1187,7 +1187,7 @@ class ColocationIds:
for sid, nms in matches.items(): for sid, nms in matches.items():
for nm in nms: for nm in nms:
self._add_match(nm[1], sid, nm[0]) self._add_match(nm[1], sid, nm[0])
def get_matches_for(self, structure_id, group): def get_matches_for(self, structure_id, group):
for _cid_tup, sm in self.data.items(): for _cid_tup, sm in self.data.items():
if sm.structure_id != structure_id: if sm.structure_id != structure_id:
@ -1204,7 +1204,7 @@ class ColocationIds:
for _1, sm in tqdm(self.data.items()): for _1, sm in tqdm(self.data.items()):
ComponentRendition.set_representations(sm, components_dict[sm.structure_id], word_renderer) ComponentRendition.set_representations(sm, components_dict[sm.structure_id], word_renderer)
idx += 1 idx += 1
def determine_colocation_dispersions(self): def determine_colocation_dispersions(self):
dispersions = defaultdict(int) dispersions = defaultdict(int)
for (structure_id, *word_tups) in self.data.keys(): for (structure_id, *word_tups) in self.data.keys():
@ -1219,7 +1219,7 @@ def match_file(words, structures):
for s in tqdm(structures): for s in tqdm(structures):
for w in words: for w in words:
mhere = s.match(w) mhere = s.match(w)
for match in mhere: for match in mhere:
colocation_id = [(idx, w.lemma) for idx, w in match.items()] colocation_id = [(idx, w.lemma) for idx, w in match.items()]
colocation_id = [s.id] + list(sorted(colocation_id, key=lambda x:x[0])) colocation_id = [s.id] + list(sorted(colocation_id, key=lambda x:x[0]))
colocation_id = tuple(colocation_id) colocation_id = tuple(colocation_id)
@ -1246,7 +1246,7 @@ def main(input_file, structures_file, args):
# make temporary directory to hold temporary files # make temporary directory to hold temporary files
with tempfile.TemporaryDirectory() as tmpdirname: with tempfile.TemporaryDirectory() as tmpdirname:
cmd = sys.argv cmd = sys.argv
for inpt in args.input: for inpt in args.input:
if inpt in cmd: if inpt in cmd:
cmd.remove(inpt) cmd.remove(inpt)
@ -1256,7 +1256,7 @@ def main(input_file, structures_file, args):
del cmd[pidx] del cmd[pidx]
del cmd[pidx] del cmd[pidx]
def func(n): def func(n):
cmdn = [sys.executable] + cmd + [args.input[n], "--match-to-file", "{}/{}.p".format(tmpdirname, n)] cmdn = [sys.executable] + cmd + [args.input[n], "--match-to-file", "{}/{}.p".format(tmpdirname, n)]
subprocess.check_call(cmdn) subprocess.check_call(cmdn)
return n return n
@ -1322,3 +1322,5 @@ if __name__ == '__main__':
start = time.time() start = time.time()
main(args.input, args.structures, args) main(args.input, args.structures, args)
logging.info("TIME: {}".format(time.time() - start)) logging.info("TIME: {}".format(time.time() - start))
# 2876, 2945 type