Added conversion from msd to universal dependencies based on Jaka's implementation

conversion_utils/jos_msds_and_properties.py

@@ -1,12 +1,21 @@
-import lxml.etree as lxml
 import re
 import pickle
+import lxml.etree as lxml
+from collections import defaultdict
 from importlib_resources import files
 
+from enum import Enum
+
 from conversion_utils.utils import xpath_find, get_xml_id
 
 JOS_SPECIFICATIONS_PICKLE_RESOURCE = 'jos_specifications.pickle'
 
+RESOURCES_DIR = "conversion_utils.resources"
+
+MSD_TO_FEATURES = "jos-msd2features.tbl"
+JOS_TO_UD_FEATURES_RULES = "jos2ud-features.tbl"
+JOS_TO_UPOS_RULES = "jos2ud-pos.tbl"
+
 ## Positions of lexeme-level features for each category
 LEXEME_FEATURE_MAP = {'noun':{1,2},
                       'verb':{1,2},
@@ -53,6 +62,14 @@ LEVEL_EXCEPTIONS = {('pronoun', 2, 'čezme'), ('zaimek', 2, 'čezme'),
                     ('pronoun', 8, 'se'), ('zaimek', 8, 'se'),
                     ('pronoun', 8, 'ti'), ('zaimek', 8, 'ti')}
 
+class MsdState(Enum):
+    FULL = 1
+    PARTIAL = 2
+    UNKNOWN = 3
+
+class MsdException(Exception):
+    pass
+
 
 class Specifications:
     """JOS specifications with list of all word categories."""
@@ -214,7 +231,33 @@ class Properties:
             and self.lexeme_feature_map == obj.lexeme_feature_map\
             and self.form_feature_map == obj.form_feature_map\
             and self.language == obj.language
-            
+          
+
+class UD:
+    """Universal Dependencies object.
+    
+    Can be converted to a valid UD features string.
+    """
+    
+    def __init__(self, pos, features_map):
+        self.pos = pos
+        self.features_map = features_map
+
+    def to_features_string(self):
+        return self._features_string()
+
+    def to_full_string(self):
+        return "UposTag=" + self.pos + "|" + self._features_string()
+
+    def _features_string(self):
+        return "|".join([f"{feature}={value}" for feature, value in self._sort_features(self.features_map)])
+
+    def _sort_features(self, features_map):
+        return sorted(features_map.items(), key=lambda x: x[0].lower(), reverse=False)
+
+    def __str__(self):
+        return f"pos={self.pos}, features_map={self.features_map}"
+
 
 class Msd:
     """JOS msd."""  
@@ -228,13 +271,7 @@ class Msd:
 
     def __eq__(self, obj):
         return isinstance(obj, Msd) and self.code == obj.code and self.language == obj.language
-
-
-class CustomException(Exception):
-    pass
-
-class MsdException(CustomException):
-    pass
+    
 
 class Converter:
     """Converter between Msd and Properties objects."""
@@ -256,25 +293,57 @@ class Converter:
                 self.specifications = parser.parse(xml_file_name)
             except:
                 exit('Could not parse specifications xml file provided.')
+     
+        self.mte_to_ud_features = self._parse_msd_ud_conversion(MSD_TO_FEATURES)
+        self.mte_to_ud_features_rules = self._parse_ud_rules(JOS_TO_UD_FEATURES_RULES)
+        self.mte_to_upos_rules = self._parse_ud_rules(JOS_TO_UPOS_RULES)
+
+    def _parse_msd_ud_conversion(self, file_name):
+        """Parse file with direct conversions from English Msd to Universal Dependencies."""
+        conversion_map = defaultdict()
+        with files(RESOURCES_DIR).joinpath(file_name).open("r", encoding="UTF-8") as conversion_file:
+            for line in conversion_file.readlines():
+                mte_msd_en, mte_features_en = line.strip("\n").split("\t")
+                mte_sl = self.translate_msd(Msd(mte_msd_en, "en"), "sl").code
+                conversion_map[mte_msd_en] = mte_features_en
+                conversion_map[mte_sl] = mte_features_en
+        return conversion_map
+
+    def _parse_ud_rules(self, file_name):
+        """Parse file with rules additional rules for converting from applied to conversion from English Msd to Universal Dependencies."""
+        all_rules = defaultdict(list)
+        with files(RESOURCES_DIR).joinpath(file_name).open("r", encoding="UTF-8") as rules_file:
+            for line in [l for l in rules_file.readlines() if l[0].isdigit()]:
+                priority, *current_rules = line.strip("\n").split("\t")
+                current_rules += [""] * (6 - len(current_rules))
+                all_rules[priority].append(current_rules)
+        return all_rules
 
     def is_valid_msd(self, msd):
         """Verify if the Msd code is in the standard JOS set."""
+        return msd.code in self.specifications.codes_map[msd.language]
+
+    def get_msd_state(self, msd):
+        """Determine if the Msd code is full, partial or unknown."""
         code_map = self.specifications.codes_map[msd.language]
         if msd.code in code_map:
-            return True
+            return MsdState.FULL
         for msd_code in code_map:
             if msd_code.startswith(msd.code):
-                return True
-        return False
+                return MsdState.PARTIAL
+        return MsdState.UNKNOWN
 
-    def check_valid_msd(self, msd, require_valid_flag):
+    def check_valid_msd(self, msd, require_valid_flag, allow_partial=True):
         """If the Msd code is not valid, raise an exception or give a warning."""
-        if (not self.is_valid_msd(msd)):
-            message = 'The msd {} is unknown'.format(msd.code)
-            if (require_valid_flag):
+        msd_state = self.get_msd_state(msd)
+        if msd_state == MsdState.UNKNOWN:
+            message = f"The msd '{msd.code}' is unknown"
+            if require_valid_flag:
                 raise MsdException(message)
             else:
                 print('[WARN] ' + message)
+        if msd_state == MsdState.PARTIAL and not allow_partial:
+            raise MsdException(f"Partial msd '{msd.code}' is not allowed. Full msd is required.")
 
     def msd_to_properties(self, msd, language, lemma=None, require_valid_flag=False, warn_level_flag=False):
         """Convert Msd to Properties.
@@ -364,6 +433,48 @@ class Converter:
         self.check_valid_msd(msd, require_valid_flag)
         return msd
 
+    def msd_to_ud(self, msd, lemma):
+        """Convert Msd to Universal Dependencies object.
+
+        Partial Msds are currently not supported.
+
+        Parameters:
+        msd(Msd): the Msd to convert
+        lemma(str): the lemma of the word form with the MSD
+        """
+
+        self.check_valid_msd(msd, False, allow_partial=False)
+        upos_category, *upos_features = self.mte_to_ud_features[msd.code].split()
+        final_upos = ""
+
+        for priority in sorted(self.mte_to_upos_rules, reverse=True):
+            for rule in self.mte_to_upos_rules[priority]:
+                rule_lemma, rule_category, rule_mte_features, _, rule_pos_ud, _ = rule
+
+                if (rule_category != upos_category
+                or (rule_lemma not in ("*", "*en") and lemma != rule_lemma)
+                or (rule_lemma == "*en" and not lemma.endswith("en"))
+                or (rule_mte_features != "*" and not all(f in upos_features for f in rule_mte_features.split("|")))):
+                    continue
+
+                final_upos = rule_pos_ud
+
+        for priority in sorted(self.mte_to_ud_features_rules):
+            for rule in self.mte_to_ud_features_rules[priority]:
+                rule_lemma, rule_category, rule_mte_features, rule_pos_ud, rule_ud_features, _ = rule
+
+                if (rule_lemma != "*" and lemma != rule_lemma
+                or (rule_category != "*" and rule_category != upos_category)
+                or (rule_pos_ud != "*" and rule_pos_ud != final_upos)):
+                    continue
+
+                upos_features = [rule_ud_features if f == rule_mte_features else f for f in upos_features]
+                if rule_mte_features == "*" and rule_ud_features != "-":
+                    upos_features.append(rule_ud_features)
+
+        ud_features = dict(f.split("=", 1) for f in "|".join(upos_features).split("|") if f not in {"", "-"})
+        return UD(final_upos, ud_features)
+
     def translate_msd(self, msd, language):
         return self.properties_to_msd(self.msd_to_properties(msd, language), language)