bug: matching srl_json files with kres_xml files

.gitignore

@@ -1,3 +1,3 @@
 data/samples/
-*/pycache/
+*/__pycache__/
 *egg-info/

README.md

@@ -1,4 +1,4 @@
-# cjvt-vezljivost
+# cjvt-valency
 
 ## Components
 

src/pkg/corpusparser/corpusparser/Parser.py

@@ -1,6 +1,7 @@
 from corpusparser import Sentence
 from pathlib import Path
 import re
+import json
 from lxml import etree
 
 # Read input file(.xml, .json; kres or ssj500k).  
@@ -26,15 +27,8 @@ class Parser():
         if self.corpus == "kres":
             return self.parse_jos_links_kres(sent_el)
         else:
-            return self.parse_jos_links_ssj(sent_el)
+            # 'syntax' is the linkgroup we're looking for
-
+            return self.parse_any_links_ssj(sent_el, "syntax")
-    def parse_ssj_target_arg(self, text):
-        # from: 0, to: 6
-        # <link ana="syn:modra" target="#ssj1.1.3 #ssj1.1.3.t6"/>
-        # from: 6, to: 7
-        # <link ana="syn:dol" target="#ssj1.1.3.t6 #ssj1.1.3.t7"/>
-        lst = [x.split(".")[-1] for x in text.split(" ")]
-        return [int(x[1:] if x[0] == "t" else 0) for x in lst]
 
     def parse_jos_links_kres(self, sent_el):
         lgrps = sent_el.findall(".//links")
@@ -49,31 +43,56 @@ class Parser():
             }]
         return res_links
 
-    def parse_jos_links_ssj(self, sent_el):
+    def parse_ssj_target_arg(self, text):
+        # from: 0, to: 6
+        # <link ana="syn:modra" target="#ssj1.1.3 #ssj1.1.3.t6"/>
+        # from: 6, to: 7
+        # <link ana="syn:dol" target="#ssj1.1.3.t6 #ssj1.1.3.t7"/>
+        lst = [x.split(".")[-1] for x in text.split(" ")]
+        return [int(x[1:] if x[0] == "t" else 0) for x in lst]
+
+    def parse_any_links_ssj(self, sent_el, links_type):
         lgrps = sent_el.findall(".//linkGrp")
-        if len(lgrps) < 1:
+        links = [x for x in lgrps if x.get("type") == links_type][0]
-            # print(etree.tostring(sent_el))
-            raise IOError("Can't find links.")
         res_links = []
-        for link in lgrps[0]:
+        for link in links:
-            print(link)
             tar = self.parse_ssj_target_arg(link.get("target"))
             res_links += [{
                 "from": tar[0],
                 "afun": link.get("ana").split(":")[1],
-                "to": [1],
+                "to": tar[1],
             }]
         return res_links
 
+    def parse_srl_links(self, sent_el, xml_file=None):
+        if self.corpus == "kres":
+            return self.parse_srl_links_kres(sent_el, xml_file)
+        else:
+            return self.parse_any_links_ssj(sent_el, "SRL")
+
+    def parse_srl_links_kres(self, sent_el, sent_srl_dict):
+        print(sent_srl_dict)
+        # find the correspointing json file with srl links
+        return "TODO"
+
     def parse(self):
         if self.corpus == "kres":
-            print("parse kres: TODO")
+            for xml_file in self.kres_folder.iterdir():
+                self.parse_xml_file(xml_file)
+                break  # TODO dev break
         else:
             self.parse_xml_file(self.ssj_file)
 
-    def parse_xml_file(self, filepath):
+    def parse_xml_file(self, xml_file):
-        res_dict = {}
+        srl_dict = {}
-        with filepath.open("rb") as fp:
+        if self.corpus == "kres":  
+            # in case of kres, read the SRL links form a separate json file
+            file_id = xml_file.name.split(".")[0]
+            json_file = self.kres_srl_folder / Path(file_id).with_suffix(".srl.json")
+            with json_file.open("r") as fp:
+                srl_dict = json.loads(fp.read())
+
+        with xml_file.open("rb") as fp:
             # remove namespaces
             bstr = fp.read()
 
@@ -89,6 +108,8 @@ class Parser():
         else:
             divs = root.findall(".//div")
 
+        res_dict = []  # TODO: try making an iterator instead
+
         # parse divs
         for div in divs:
             f_id = div.get("id")
@@ -137,15 +158,23 @@ class Parser():
                     sentence_id = "{}.{}.{}".format(f_id, p_id, s_id)
 
                     # make a generator instead of holding the whole corpus in memory
+                    # TODO -- match ids
+                    print("---")
+                    print(sorted(srl_dict.keys(), key=lambda x: x.split(".")[1])[:100])
+                    print(sentence_id)
+                    print(srl_dict.get(str(sentence_id)))
+                    print("---")
                     if sentence_id in res_dict:
                         raise KeyError("duplicated id: {}".format(sentence_id))
                     res_dict[sentence_id] = {
                         "sid": sentence_id,
                         "text": sentence_text,
                         "tokens": sentence_tokens,
-                            "jos_links": self.parse_jos_links(s)
+                        "jos_links": self.parse_jos_links(s),
+                        "srl_links": self.parse_srl_links(s, srl_dict[sentence_id]),
                     }
 
                     print(res_dict[sentence_id])
-                        break
+                    print("------------------------------------------------- END")
+                    return  # TODO dev break
         return res_dict

src/preflight/main_parse.py

@@ -9,8 +9,17 @@ if __name__ == "__main__":
 	args = parser.parse_args()	
 
 	# parse ssj
+	"""
 	ssj_parser = Parser(
 		corpus="ssj",
 		infiles=[args.ssj_file]
 	)
 	ssj_parser.parse()
+	"""
+
+	# parse kres
+	kres_parser = Parser(
+		corpus="kres",
+		infiles=[args.kres_folder, args.kres_srl_folder]
+	)
+	kres_parser.parse()