From da460f74f15aba9cfa5c9960421c180e87c677d8 Mon Sep 17 00:00:00 2001 From: voje Date: Wed, 13 Mar 2019 08:59:27 +0100 Subject: [PATCH] corpusparser finished python dict representation; TODO .json and DB --- data/debugging/dbg.txt | 6 +++ data/kres_srl | 2 +- data/kres_srl_t420 | 1 + src/pkg/corpusparser/corpusparser/Parser.py | 33 +++++++++------- src/pkg/corpusparser/corpusparser/README.md | 38 +++++++++++++++++++ src/pkg/corpusparser/corpusparser/Sentence.py | 3 -- src/pkg/corpusparser/corpusparser/__init__.py | 3 +- 7 files changed, 67 insertions(+), 19 deletions(-) create mode 100644 data/debugging/dbg.txt create mode 120000 data/kres_srl_t420 create mode 100644 src/pkg/corpusparser/corpusparser/README.md delete mode 100644 src/pkg/corpusparser/corpusparser/Sentence.py diff --git a/data/debugging/dbg.txt b/data/debugging/dbg.txt new file mode 100644 index 0000000..2b6909d --- /dev/null +++ b/data/debugging/dbg.txt @@ -0,0 +1,6 @@ +F0034713.20.1": [{"dep": "7", "arg": "REC", "from": "9"}, {"dep": "10", "arg": "ACT", "from": "9"}, {"dep": "13", "arg": "MWPRED", "from": "12"}, {"dep": "18", "arg": "MANN", "from": "19"}, {"dep": "20", "arg": "LOC", "from": "19"}] + +Sentence: +F0034713.20.0 +1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 +Sodobni ali preprosto neosveščeni potrošnik vse prerad zavrže stvar, ki se je malenkostno pokvarila in bi se jo zlahka dalo popraviti. \ No newline at end of file diff --git a/data/kres_srl b/data/kres_srl index f1acfc4..465d987 120000 --- a/data/kres_srl +++ b/data/kres_srl @@ -1 +1 @@ -/home/kristjan/kres_srl/final_json/ \ No newline at end of file +/home/voje/work_data/final_json \ No newline at end of file diff --git a/data/kres_srl_t420 b/data/kres_srl_t420 new file mode 120000 index 0000000..f1acfc4 --- /dev/null +++ b/data/kres_srl_t420 @@ -0,0 +1 @@ +/home/kristjan/kres_srl/final_json/ \ No newline at end of file diff --git a/src/pkg/corpusparser/corpusparser/Parser.py b/src/pkg/corpusparser/corpusparser/Parser.py index 1c5dd6b..847fa69 100644 --- a/src/pkg/corpusparser/corpusparser/Parser.py +++ b/src/pkg/corpusparser/corpusparser/Parser.py @@ -1,14 +1,16 @@ -from corpusparser import Sentence from pathlib import Path import re import json from lxml import etree +import logging + +logging.basicConfig(level=logging.INFO) # Read input file(.xml, .json; kres or ssj500k). # Create an iterator that outputs resulting sentences (python dict format). class Parser(): - def __init__(self, corpus, infiles): + def __init__(self, corpus, infiles, logger=None): if corpus == "kres": self.kres_folder = Path(infiles[0]) @@ -22,6 +24,7 @@ class Parser(): self.W_TAGS = ['w'] self.C_TAGS = ['c'] self.S_TAGS = ['S', 'pc'] + self.logger = logger or logging.getLogger(__name__) def parse_jos_links(self, sent_el): if self.corpus == "kres": @@ -71,13 +74,15 @@ class Parser(): return self.parse_any_links_ssj(sent_el, "SRL") def parse_srl_links_kres(self, sent_el, sent_srl_links): - print("HA") - if len(sent_srl_links) == 0: - print("HI") - return [] - print(sent_srl_links) + res_links = [] + for link in sent_srl_links: + res_links += [{ + "from": int(link["from"]), + "afun": link["arg"], + "to": int(link["dep"]), + }] # find the correspointing json file with srl links - return sent_srl_links + return res_links def parse(self): if self.corpus == "kres": @@ -166,7 +171,10 @@ class Parser(): raise KeyError("duplicated id: {}".format(sentence_id)) jos_links = self.parse_jos_links(s) srl_links = srl_dict.get(sentence_id) if self.corpus == "kres" else None - srl_links_parsed = self.parse_srl_links(s, srl_links) + if srl_links is None: + srl_links_parsed = None + else: + srl_links_parsed = self.parse_srl_links(s, srl_links) res_dict[sentence_id] = { "sid": sentence_id, "text": sentence_text, @@ -174,8 +182,7 @@ class Parser(): "jos_links": jos_links, "srl_links": srl_links_parsed } - print("------------------------------------------------- END") - print(res_dict[sentence_id]) - print("------------------------------------------------- END") - return # TODO dev break + if srl_links is None: + self.logger.info("srl_links missing:{}:{}".format( + sentence_id, res_dict[sentence_id]["text"])) return res_dict diff --git a/src/pkg/corpusparser/corpusparser/README.md b/src/pkg/corpusparser/corpusparser/README.md new file mode 100644 index 0000000..0d41d60 --- /dev/null +++ b/src/pkg/corpusparser/corpusparser/README.md @@ -0,0 +1,38 @@ +# corpusparser +A tool for parsing ssj500k and Kres into a unified .json format. + +## Input: +### ssj500k +To parse ssj500k, point to the monolythic `ssj500k-sl.body.xml` file (tested on ssj500k 2.1). + +### Kres +To parse Kres, point to folders: +* Kres folder, containig several (around 20K) .xml files (`F00XXXXX.xml.parsed.xml`). +* Kres SRL folder, containing SRL links for the corresponding F00...xml files (`F00XXXXX.srl.json`). + +## Internal data format +This is the internal python dict data format. It can be stored to file as `.json` or stored into a database for application usage. +```python +{ + 'sid': 'F0034713.5.0', + 'text': 'Mednarodni denarni sklad je odobril 30 milijard evrov vredno posojilo Grčiji. ', + 'tokens': [ + {'text': 'Mednarodni', 'lemma': 'mednaroden', 'msd': 'Ppnmeid', 'word': True, 'tid': 1}, + {'text': 'denarni', 'lemma': 'denaren', 'msd': 'Ppnmeid', 'word': True, 'tid': 2}, + {'text': 'sklad', 'lemma': 'sklad', 'msd': 'Somei', 'word': True, 'tid': 3}, + {'text': 'je', 'lemma': 'biti', 'msd': 'Gp-ste-n', 'word': True, 'tid': 4}, + {'text': 'odobril', 'lemma': 'odobriti', 'msd': 'Ggdd-em', 'word': True, 'tid': 5}, + {'text': '30', 'lemma': '30', 'msd': 'Kag', 'word': True, 'tid': 6}, + {'text': 'milijard', 'lemma': 'milijarda', 'msd': 'Sozmr', 'word': True, 'tid': 7}, # ... + ] + 'jos_links': [ + {'to': 1, 'from': 3, 'afun': 'dol'}, + {'to': 2, 'from': 3, 'afun': 'dol'}, + {'to': 3, 'from': 5, 'afun': 'ena'}, # ... + ] + 'srl_links': [ + {'to': 3, 'from': 5, 'afun': 'ACT'}, + {'to': 7, 'from': 5, 'afun': 'PAT'} + ] +} +``` \ No newline at end of file diff --git a/src/pkg/corpusparser/corpusparser/Sentence.py b/src/pkg/corpusparser/corpusparser/Sentence.py deleted file mode 100644 index aba9a4a..0000000 --- a/src/pkg/corpusparser/corpusparser/Sentence.py +++ /dev/null @@ -1,3 +0,0 @@ -class Sentence(): - def __init__(): - print("Sentence __init__(): TODO") diff --git a/src/pkg/corpusparser/corpusparser/__init__.py b/src/pkg/corpusparser/corpusparser/__init__.py index 213c88a..d993b8c 100644 --- a/src/pkg/corpusparser/corpusparser/__init__.py +++ b/src/pkg/corpusparser/corpusparser/__init__.py @@ -1,2 +1 @@ -from corpusparser.Parser import Parser -from corpusparser.Sentence import Sentence \ No newline at end of file +from corpusparser.Parser import Parser \ No newline at end of file