From 47bb4ce0adf6712e54f3bf4b829279e5f9dce2e7 Mon Sep 17 00:00:00 2001
From: voje <voje@iskratel.si>
Date: Tue, 12 Feb 2019 08:24:46 +0100
Subject: [PATCH] java_python env dockerfile, progress on parser

---
 dockerfiles/all/Dockerfile                    |  12 +++++++
 dockerfiles/all/README.md                     |  12 +++++++
 tools/parser/README.md                        |   3 ++
 tools/parser/__init__.pyc                     | Bin 147 -> 0 bytes
 .../__pycache__/__init__.cpython-37.pyc       | Bin 129 -> 0 bytes
 .../parser/__pycache__/parser.cpython-37.pyc  | Bin 2289 -> 0 bytes
 tools/parser/msdmap.py                        |   6 +++-
 tools/parser/parser.py                        |  30 ++++++++++++++----
 tools/parser/parser.pyc                       | Bin 325 -> 0 bytes
 9 files changed, 56 insertions(+), 7 deletions(-)
 create mode 100644 dockerfiles/all/Dockerfile
 create mode 100644 dockerfiles/all/README.md
 delete mode 100644 tools/parser/__init__.pyc
 delete mode 100644 tools/parser/__pycache__/__init__.cpython-37.pyc
 delete mode 100644 tools/parser/__pycache__/parser.cpython-37.pyc
 delete mode 100644 tools/parser/parser.pyc

diff --git a/dockerfiles/all/Dockerfile b/dockerfiles/all/Dockerfile
new file mode 100644
index 0000000..e6159e9
--- /dev/null
+++ b/dockerfiles/all/Dockerfile
@@ -0,0 +1,12 @@
+FROM ubuntu:16.04
+
+RUN apt-get update
+RUN apt-get install -y \
+vim \
+default-jdk \
+python3 \
+python3-pip
+
+RUN pip3 install lxml
+
+ENV PYTHONIOENCODING UTF-8
diff --git a/dockerfiles/all/README.md b/dockerfiles/all/README.md
new file mode 100644
index 0000000..166f5b1
--- /dev/null
+++ b/dockerfiles/all/README.md
@@ -0,0 +1,12 @@
+You might want to mount this whole repo into the docker container.  
+Also mount data locations.  
+
+Example container:
+```bash
+$ docker build . -t my_python
+$ docker run \
+    -it \
+    -v $(echo $(cd ../..; pwd)):/cjvt-srl-tagging \
+    python_java \
+    /bin/bash
+```
diff --git a/tools/parser/README.md b/tools/parser/README.md
index ef6ae42..a52097a 100644
--- a/tools/parser/README.md
+++ b/tools/parser/README.md
@@ -1,3 +1,6 @@
 ## msdmap.py
 Help conversion between english and slovenian MSD.  
 Hardcoded values from online documentation (html tables).  
+
+## Sources
+[1] (conll09 data format) https://nlpado.de/~sebastian/pub/papers/conll09_hajic.pdf
\ No newline at end of file
diff --git a/tools/parser/__init__.pyc b/tools/parser/__init__.pyc
deleted file mode 100644
index d333b89c362415aca1b483e4a42d569584230a4a..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 147
zcmZSn%**xXKv+yN0~9a<X$K%KW&si@3=F{<AQ3+eAi;n}6x#sB^fU5vQ}wfpGK)*H
z67%%aGfVW7v&u?zi;Hq}OA^!5GxO5*OY-w`iuDT;i;7c=^yA|*^D;}~<Mj$EOE`cA
S*yQG?l;)(`fvhP8Vg>+XNg?O}

diff --git a/tools/parser/__pycache__/__init__.cpython-37.pyc b/tools/parser/__pycache__/__init__.cpython-37.pyc
deleted file mode 100644
index ecc16a37ac1eb85d907302918a0abcc82dd970ce..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 129
zcmZ?b<>g`kf_s+nF(CReh=2h`Aj1KOi&=m~3PUi1CZpd<h9ZzKg7~GSpPW@zqFY>)
zqg#@go}QVPu3wU$pHr+~kXTflTBIKzpP83g5+AQuP<e~PCO1E&G$+*#WJWO%GXMa&
C4IG02

diff --git a/tools/parser/__pycache__/parser.cpython-37.pyc b/tools/parser/__pycache__/parser.cpython-37.pyc
deleted file mode 100644
index d2e9effc75991a73e8379622eced50fa9d0d9e81..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 2289
zcmah~&2Jnv6t_K|+1c5T?4~I+(3Vg>12o|P0xCd2Qz0mnk01q2TcYiHc9~={v+;OK
zy7fqiz!B()147j<H*Op_aX{jN_#1rX6plS`=mmbBO-rCc%#5F(pWoQe@8joJ8jXP9
z`T3zs7k+9J@|zvpdKj3O(DD@!K?F_677Z!2JWE)}gwv;?BRD7*uJHOSbVWt@pOVlM
zRS}@Ch?=OQ_XS%ei;X?3L>8%WBPFBgoS`2X+BI}FWP@&(+DSaKXbrT}X!#1P5Tu|n
zDVQP}R_wBC?8}D>M>}IWAq7`#!x6AWa{((p=T<UygeNMRPaKT+*wuXO34cN-v~c^R
zfOXm(S0-fBlN%x^sCN6b@Y1HJS$^z`IyAj&bmvFyt&nT<;`c>GSH@LQh1`eyI!3-|
zOlUd#MN!q&>FhWV%?TAV^P~u%Cn5JmYhUkDQPZ_M^y<PX8nCLl^9N>Y6S6`s{$9-J
znT@(?Y{Clpv7Wi(ua<6!_B;`@n+zIHV#EdcjA7rFZs6bMGo)zfrZ}W#bZeJdNx910
zglZ~+@-I<J4o_%0TXJ_zDD1y`YJU$t!5Yo&8>J<!(vn|qS#pgIz7|KYSL?uu|GD$L
zZrNRX$dUu=wBhp`x*;mozU>>^H`a)_XFRKY9q2mZ)z)pzuosh_IwbDJOnX9i=_eF&
zoM+#5CXcH*UE8H8LpC1$hHIwV&y($6*2wmCw2#*a)+O(dcgg!i+uS>bY?Gsz3;E7g
zlIAa*IRC+ki;tf$WY4CZP04OHOx4I;!2GG3bYbKURqy1phQ%V}XWu=u8ak`1uaCT?
z<z*3H9&z*oLznjc#8ZxPX;(5-Ne6xCCehYbCv>)Q5jvf;bCaG5d77n><d`1$SBs_c
z8@0u7ki^}Nio`-JPAyz57J2C8u`pakJIZ(}+l<oOxJjID=8K07&jwLyxQsf&Fd0F2
zcuCq~4W0Mje8YHA+Ra2{_-1q^H<ezTicXRkrynWf=GaOZ9;-+iM|JwfTU&qc<yTi_
z%i8Y5DfT;EN{qL<-}f@P)ltSj8(ldgWhPB+(2;qxKDB4o_`NuZ1|9WbSnm&`JYVl-
zaxlz|FQa^2#9d`rZxDL$Ypx{h&N2nOh%e{Bj`(uOd+RtW9fW*PGI_{zFkvH)QiZcb
z>(&k7%-_S(A0dm9(2WwSTDv_Li|>eh3}?Cd=yG@CvO1Z|<Rs47kJJ9L%CaP1E{&J_
z-O}KSsZNihqS!hi7a)XkyiMlQS<n{aw2tPZ1x^!fj?UR;wJ@%OvoxR#uLr3Q$x&8k
zEyx4#0Sg?g2foRy4r5O_@(Jv78}wWZDrhC>?SRo_46p;-P=$isrGjk|Bt3x67K4Su
zowrd8eBQ#vO5rFAhyXy8FrXMET$v}kwBV%_(5Lyn1|aO9M$bX%aFosyYtupr&W(H|
zcNtyj!;(?^bSHg8E!a$2D8)<DRCh!)kacKC3)dm<^oA+IDYLCbPac8DI0-Vf^4eQv
z-pJ!tb>5PDKo(saXdA?&F9&&8wdqnWZ@BH`NvmmNIVF!<?g5bVX!#rnfj{f;we{=~
z#EQ`qtkk&eRa<Jf)}~(l0gQk@cfQ3n0-%K7zABi8M**XJ(~%3?O<XZ-7gt<))wtG!
z6cXV9RvpOTdyL`RG1c&!@aOGp^%fPu^u_x4SKD>1#(#~l>VY|oCUt-!-EJZhRMaf5
zDWI!<PTJIv_9Ug}ON5d3o}32g!K+2}|Gr?)Ld0H)O<ZLnGc;Ia#vMo$y}TcDfSu;;
zLPG7yD`6!Ys&!Ot=$0T9a)omHS4>QxGL(FpBu_rKfo1X~5Q0*%CBYCL8zLWNVN1il
zWdVRT%hv`@K5e(JhCBgClu#xQLn4n^a-Ss&me_(R<6G(<Fr!Qon;*u1ek&8hBzn<e
Raoz;M?#%A$2mV=q<}ZfPFBbp+

diff --git a/tools/parser/msdmap.py b/tools/parser/msdmap.py
index 7404faa..929ed74 100644
--- a/tools/parser/msdmap.py
+++ b/tools/parser/msdmap.py
@@ -9,7 +9,7 @@ class Msdmap():
             ("prislov", "R", "Adverb", "R"),
             ("zaimek", "Z", "Pronoun", "P"),
             ("števnik", "K", "Numeral", "M"),
-            ("predlog", "D", "Preposition", "S"),
+            ("predlog", "D", "Adposition", "S"),
             ("veznik", "V", "Conjunction", "C"),
             ("členek", "L", "Particle", "Q"),
             ("medmet", "M", "Interjection", "I"),
@@ -290,6 +290,10 @@ class Msdmap():
         for pos in self.pos_val:
             if pos[7] == en_category and pos[col] == query:
                 return pos
+        print("---")
+        print(en_category)
+        print(col)
+        print(query)
         raise ValueError("Wrong part of speech value.")
 
     def msd_from_slo(self, msd):
diff --git a/tools/parser/parser.py b/tools/parser/parser.py
index 2a08c2b..0a91b5b 100644
--- a/tools/parser/parser.py
+++ b/tools/parser/parser.py
@@ -1,5 +1,6 @@
 from lxml import etree
 import re
+from parser.msdmap import Msdmap
 
 W_TAGS = ['w']
 C_TAGS = ['c']
@@ -16,13 +17,15 @@ S_TAGS = ['S', 'pc']
 def parse_tei(filepath):
     guess_corpus = None  # SSJ | KRES
     res_dict = {}
-    with open(filepath, "r") as fp:
+    with open(filepath, "rb") as fp:
         # remove namespaces
-        xmlstr = fp.read()
-        xmlstr = re.sub('\\sxmlns="[^"]+"', '', xmlstr, count=1)
-        xmlstr = re.sub(' xml:', ' ', xmlstr)
+        bstr = fp.read()
 
-        root = etree.XML(xmlstr.encode("utf-8"))
+        utf8str = bstr.decode("utf-8")
+        utf8str = re.sub('\\sxmlns="[^"]+"', '', utf8str, count=1)
+        utf8str = re.sub(' xml:', ' ', utf8str)
+
+        root = etree.XML(utf8str.encode("utf-8"))
 
         divs = []  # in ssj, there are divs, in Kres, there are separate files
         if "id" in root.keys():
@@ -104,16 +107,28 @@ def parse_links(s_el):
 
 
 def to_conll09(sentence_entry):
+
+    def fillpred(pos, feat):
+        if False:
+            # todo
+            return "Y"
+        return "_"
+
+    msdm = Msdmap()
     # works with kres, with parsed links
     out_str = ""
     for token in sentence_entry["tokens"]:
         if token[0] != "w":
             continue
+        msd = msdm.msd_from_slo(token[4])
+        fprd = fillpred("todo", "todo")
+        print(msd)
         print(token)
         print(sentence_entry["links"])
         t_id = token[1]
         print(t_id)
-        out_str += "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\n".format(
+        #            1   3   4   5   6   7   8   9  10  11  12  13  14
+        out_str += "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\n".format(
             t_id,  # id
             token[2],  # form
             token[3],  # lemma
@@ -126,6 +141,9 @@ def to_conll09(sentence_entry):
             sentence_entry["links"][t_id][2],  # phead
             sentence_entry["links"][t_id][1],  # deprel
             sentence_entry["links"][t_id][1],  # pdeprel
+            fprd,  # fillpred
+            (token[3] if fprd == "Y" else "_"),  # pred
+            "todo"  # apredn...
         )
     out_str += "\n"
     return out_str
diff --git a/tools/parser/parser.pyc b/tools/parser/parser.pyc
deleted file mode 100644
index 5ddae18ab3193dcd82bc96287f8c80e21cb84f52..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 325
zcmZSn%*(a(a9B(-0~9a<X$K%K<^>Wd3=Ay{3{gM^BSWwT6Ho+2oij)sCrBM5gS!V%
zF+`0YSZ_vZPENj#LP26tacYqUP!R(fQEUYi)6dAyP1VmX$}BF)O3c$w&n(eT&MGU>
zEiTH@ElEsI&&*5LFUil(Db|M=1f}!}Doc2P+QF8@L!>}1F9x}hfzeL`#0dhiscb0+
S&`6uy{FKt1R6DSRAUgo?&`8Gs