Updating for lemma representation of word_form. Also cleaning code, adding tqdm,...

wani.py

@@ -1,7 +1,7 @@
 from xml.etree import ElementTree
 import re
 from enum import Enum
-from collections import defaultdict
+from collections import defaultdict, namedtuple
 import sys
 import logging
 import argparse
@@ -12,6 +12,7 @@ import concurrent.futures
 import tempfile
 
 from msd_translate import MSD_TRANSLATE
+from tqdm import tqdm
 
 
 MAX_NUM_COMPONENTS = 5
@@ -217,7 +218,7 @@ class ComponentRendition:
         return self.rendition is rendition
     
     @staticmethod
-    def set_representations(matches, structure, word_renderer):
+    def set_representations(matches, structure, word_renderer, lemma_msds):
         representations = {
             c.idx: [[], None] if c.representation.isit(Rendition.WordForm) else [True, ""]
             for c in structure.components
@@ -225,24 +226,34 @@ class ComponentRendition:
         found_agreements = {}
         word_component_id = {}
 
-        # doprint = structure.id == '1' and matches[0]['1'].text.startswith('evrop') and matches[0]['2'].text.startswith('prv')
+        def render_all(component_id, lst, _bw):
-        doprint = False
+            rep = "/".join([w.text for w in set(lst)]) if len(lst) > 0 else None
+            matches.representations[component_id] = rep
         
-        def render_all(component_id, lst):
+        def render_form(component_id, lst, backup_word):
-            matches.representations[component_id] = "/".join([w.text for w in set(lst)])
+            if backup_word is not None:
+                lst.append(backup_word)
 
-        def render_form(component_id, lst):
+            lst_ctr = []
-            sorted_lst = sorted(set(lst), key=lst.count)
+            for word in lst:
-            for word in sorted_lst:
+                lst_ctr.append((word.msd, word.lemma))
+            sorted_lst = sorted(set(lst_ctr), key=lst.count)
+
+            if len(lst) > 3:
+                a = 3
+
+            for word_msd, word_lemma in sorted_lst:
                 if component_id in found_agreements:
                     other_component_id, other_word, agreements = found_agreements[component_id]
-                    print(word.lemma, other_word.lemma, component_id, other_component_id, word.msd, word.msd)
+                    agr = are_agreements_ok(word_msd, other_word.lemma, other_word.msd, agreements)
-                    agr = are_agreements_ok(word.msd, other_word.lemma, other_word.msd, agreements)
                     if agr is None:
                         continue
+
                     matches.representations[other_component_id] = agr
 
-                matches.representations[word_component_id[word.id]] = word.most_frequent_text(word_renderer)
+                if word_lemma is not None:
+                    matches.representations[component_id] = word_renderer.render(word_lemma, word_msd)
+
                 break
                 
                 # othw = are_agreements_ok(word, found_agreements)
@@ -255,10 +266,7 @@ class ComponentRendition:
                 if ow_msd[0] != w2_msd[0]:
                     continue
 
-                print(w1_msd, w2_msd)
                 if check_agreement(w1_msd, w2_msd, agreements):
-                    if doprint:
-                        print("GOOD :)")
                     return w2_txt
 
         def check_msd(word, selectors):
@@ -279,7 +287,6 @@ class ComponentRendition:
                 # if not in msd, some strange msd was tries, skipping...
                 if agr_case not in TAGSET[t1]:
                     logging.warning("Cannot do agreement: {} for msd {} not found!".format(agr_case, msd1))
-                    print("BAAAD")
                     return False
 
                 v1 = TAGSET[t1].index(agr_case)
@@ -293,7 +300,6 @@ class ComponentRendition:
                 t2 = msd2[0]
                 if agr_case not in TAGSET[t2]:
                     logging.warning("Cannot do agreement: {} for msd {} not found!".format(agr_case, msd2))
-                    print("BAAAD")
                     return False
                 v2 = TAGSET[t2].index(agr_case)
                 if v2 + 1 >= len(msd2): 
@@ -328,44 +334,26 @@ class ComponentRendition:
                     assert(rep.isit(Rendition.WordForm))
                     wf_type, more = rep.more
 
-                    if wf_type is WordFormSelection.All:
+                    if wf_type is WordFormSelection.Msd:
-                        add = True
-                        func = render_all
-                    elif wf_type is WordFormSelection.Msd:
                         add = check_msd(w, more)
                         func = render_form
+                    elif wf_type is WordFormSelection.All:
+                        add = True
+                        func = render_all
                     elif wf_type is WordFormSelection.Any:
                         add = True
                         func = render_form
                     else:
                         assert(wf_type is WordFormSelection.Agreement)
                         other_w, agreements = more
-                        found_agreements[other_w] = (w_id, w.lemma, agreements)
+                        found_agreements[other_w] = (w_id, w, agreements)
+
                         add = True
                         func = lambda *x: None
 
+                    representations[w_id][1] = func
                     if add:
                         representations[w_id][0].append(w)
-                        representations[w_id][1] = func
-
-        if doprint:
-            print(len(matches), len(found_agreements))
-
-        # for w1i, w2i, agreements in found_agreements:
-        #     w1, w2 = words[w1i], words[w2i]
-        #     if doprint:
-        #         print("? ", w1.msd, w2.msd, end="")
-
-        #     if w2i not in bad_words:
-        #             
-        #         if check_agreement(w1, w2, agreements):
-        #             representations[w1i][0].append(w1.text)
-        #             if doprint:
-        #                 print(" :)")
-        #         elif doprint:
-        #             print(" :(")
-        #     elif doprint:
-        #         print(" :((")
 
         # just need to set representation to first group,
         # but in correct order, agreements last!
@@ -379,19 +367,11 @@ class ComponentRendition:
 
         for w_id, w in representation_sorted_words:
             data = representations[w_id]
-            if doprint:
-                print([(r.text, r.lemma, r.msd) for r in data[0]])
-
             if type(data[1]) is str:
                 matches.representations[w_id] = None if data[0] else data[1]
-            elif len(data[0]) == 0:
-                matches.representations[w_id] = None
             else:
-                data[1](str(w_id), data[0])
+                backup_word = lemma_only_word(lemma_msds[w.msd[0]]) if w.msd[0] in lemma_msds else None
-            
+                data[1](str(w_id), data[0], backup_word)
-        if doprint:
-            print(matches.representations)
-            print('--')
             
     def __str__(self):
         return str(self.rendition)
@@ -468,7 +448,7 @@ def build_morphology_regex(restriction):
                 return False
         return True
 
-    return " ".join(rgx), matcher
+    return rgx, matcher
 
 
 def build_lexis_regex(restriction):
@@ -493,7 +473,8 @@ class Restriction:
         restriction_type = restriction_tag.get('type')
         if restriction_type == "morphology":
             self.type = RestrictionType.Morphology
-            self.present, self.matcher = build_morphology_regex(list(restriction_tag))
+            present, self.matcher = build_morphology_regex(list(restriction_tag))
+            self.present = " ".join(present)
         elif restriction_type == "lexis":
             self.type = RestrictionType.Lexis
             self.present, self.matcher = build_lexis_regex(list(restriction_tag))
@@ -822,10 +803,14 @@ class SyntacticStructure:
         #     to_ret.append((m, self.check_agreements(m)))
 
 
-def build_structures(filename):
+def load_structures(filename):
-    structures = []
     with open(filename, 'r') as fp:
         et = ElementTree.XML(fp.read())
+    
+    return build_structures(et), get_lemma_features(et)
+
+def build_structures(et):
+    structures = []
     for structure in et.iter('syntactic_structure'):
         to_append = SyntacticStructure.from_xml(structure)
         if to_append is None:
@@ -833,6 +818,29 @@ def build_structures(filename):
         structures.append(to_append)
     return structures
 
+def get_lemma_features(et):
+    lf = et.find('lemma_features')
+    if lf is None:
+        return {}
+
+    result = {}
+    for pos in lf.iter('POS'):
+        rgx_list, _ = build_morphology_regex(pos)
+        rgx_str = ""
+        for position in rgx_list:
+            if position == ".":
+                rgx_str += " "
+            elif len(position) == 1:
+                rgx_str += position
+            elif len(position) == 3 and position[0] == "[" and position[2] == "]":
+                rgx_str += position[1]
+            else:
+                raise RuntimeError("Strange rgx for lemma_feature...")
+        
+        assert(rgx_str[0].isupper())
+        result[rgx_str[0]] = rgx_str.strip().replace(' ', '-')
+
+    return result
 
 def get_msd(comp):
     d = dict(comp.items())
@@ -844,6 +852,10 @@ def get_msd(comp):
         logging.error(d, file=sys.stderr)
         raise NotImplementedError("MSD?")
 
+def lemma_only_word(msd):
+    WordLemma = namedtuple('WordLemmaOnly', 'msd most_frequent_text lemma')
+    return WordLemma(msd=msd, most_frequent_text=lambda *x: None, lemma=None)
+
 class Word:
     def __init__(self, xml, do_msd_translate):
         self.lemma = xml.get('lemma')
@@ -882,6 +894,7 @@ class WordMsdRenderer:
     def __init__(self):
         self.all_words = []
         self.rendered_words = {}
+        self.frequent_words = {}
     
     def add_words(self, words):
         self.all_words.extend(words)
@@ -893,19 +906,30 @@ class WordMsdRenderer:
 
         for lemma, ld in data.items():
             self.rendered_words[lemma] = {}
+            freq_words = defaultdict(int)
+
             for msd, texts in ld.items():
                 rep = max(set(texts), key=texts.count)
-                self.rendered_words[lemma][msd] = rep
+                self.rendered_words[lemma][msd] = (rep, len(texts))
+
+                for txt in texts:
+                    freq_words[(msd, txt)] += 1
+            
+            self.frequent_words[lemma] = []
+            for (msd, txt), n in sorted(freq_words.items(), key=lambda x: -x[1]):
+                self.frequent_words[lemma].append((msd, txt, n))
         
     def render(self, lemma, msd):
         if lemma in self.rendered_words:
             if msd in self.rendered_words[lemma]:
-                return self.rendered_words[lemma][msd]
+                return self.rendered_words[lemma][msd][0]
     
     def available_words(self, lemma):
-        if lemma in self.rendered_words:
+        if lemma in self.frequent_words:
-            for msd in self.rendered_words[lemma].keys():
+            # print("--")
-                yield (msd, self.rendered_words[lemma][msd])
+            for msd, text, _ in self.frequent_words[lemma]:
+                # print(lemma, msd, text, _)
+                yield (msd, text)
 
 def is_root_id(id_):
     return len(id_.split('.')) == 3
@@ -1010,7 +1034,6 @@ class Writer:
         elif self.all:
             return [word.id, word.text, word.lemma, word.msd]
         else:
-            # print("1", word)
             if representation is None:
                 return [word.lemma, word.lemma, "lemma_fallback"]
             else:
@@ -1137,17 +1160,21 @@ class ColocationIds:
                 if group:
                     break
 
-    def set_representations(self, structures, word_renderer):
+    def set_representations(self, structures, word_renderer, lemma_msds):
         components_dict = {structure.id: structure for structure in structures}
-        for _1, sm in self.data.items():
+        idx = 1
-            ComponentRendition.set_representations(sm, components_dict[sm.structure_id], word_renderer)
+        for _1, sm in tqdm(self.data.items()):
+            if idx == 120:
+                a = 3
+            ComponentRendition.set_representations(sm, components_dict[sm.structure_id], word_renderer, lemma_msds)
+            idx += 1
 
 
 def match_file(words, structures):
     matches = {s.id: [] for s in structures}
 
-    for idx, s in enumerate(structures):
+    for idx, s in tqdm(list(enumerate(structures))):
-        logging.info("{}/{}: {:7s}".format(idx, len(structures), s.id))
+        # logging.info("{}/{}: {:7s}".format(idx, len(structures), s.id))
         for w in words:
             mhere = s.match(w)
             logging.debug("  GOT: {}".format(len(mhere)))
@@ -1162,13 +1189,19 @@ def match_file(words, structures):
 
 
 def main(input_file, structures_file, args):
-    structures = build_structures(structures_file)
+    structures, lemma_msds = load_structures(structures_file)
     for s in structures:
         logging.debug(str(s))
 
     colocation_ids = ColocationIds()
     word_renderer = WordMsdRenderer()
 
+    # if True:
+    #     with open("match_word.p", "rb") as fp:
+    #         words, matches = pickle.load(fp)
+    #     colocation_ids.add_matches(matches)
+    #     word_renderer.add_words(words)
+
     if args.parallel:
         num_parallel = int(args.parallel)
 
@@ -1215,7 +1248,7 @@ def main(input_file, structures_file, args):
     # get word renders for lemma/msd
     word_renderer.generate_renders()
     # figure out representations!
-    colocation_ids.set_representations(structures, word_renderer)
+    colocation_ids.set_representations(structures, word_renderer, lemma_msds)
 
     if args.all:
         Writer.make_all_writer(args).write_out(structures, colocation_ids)