Working fix pre merge

ATEapi/Dockerfile

@@ -1,31 +1,26 @@
+# syntax = docker/dockerfile:experimental
 # For more information, please refer to https://aka.ms/vscode-docker-python
 FROM python:3.8
 
 EXPOSE 5000
 
+# Creates a non-root user with an explicit UID and adds permission to access the /app folder
+# For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers
+RUN adduser -u 5678 --disabled-password --gecos "" appuser
+USER appuser
+
 # Keeps Python from generating .pyc files in the container
-ENV PYTHONDONTWRITEBYTECODE=1
-
 # Turns off buffering for easier container logging
-ENV PYTHONUNBUFFERED=1
+ENV PYTHONDONTWRITEBYTECODE=1 \
-
+    PYTHONUNBUFFERED=1 \
-# RUN apk add --no-cache gcc musl-dev
+    PATH="/home/appuser/.local/bin:${PATH}"
-RUN pip install gunicorn
 
 # Install pip requirements
 COPY requirements.txt .
-RUN python -m pip install -r requirements.txt
+RUN python -m pip install --user -r requirements.txt
-
+RUN python -c "import classla; classla.download('sl')"
-WORKDIR /app
+# WORKDIR /app
 COPY . /app
 
-
-ADD https://kt-cloud.ijs.si/index.php/s/T4qtSKxbxgqr6c5/download/pytorch_model.bin ./model/term_extractor/
-
-# Creates a non-root user with an explicit UID and adds permission to access the /app folder
-# For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers
-RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /app
-USER appuser
-
 # During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug
-CMD ["gunicorn", "-t 0", "--bind", "0.0.0.0:5000", "main:app"]
+CMD ["gunicorn", "--bind", "0.0.0.0:5000", "--timeout", "128",  "app.main:app"]

ATEapi/README.md

@@ -23,7 +23,7 @@ Run the following command on the terminal:
 main.py
 ```
 
-A link will be show on the terminal so that you can access to the API and test requests.
+A link will be show on the terminal so that you can access to the API and test requests locally.
 ```python
  * Serving Flask app 'main'
  * Debug mode: on
@@ -35,10 +35,6 @@ Press CTRL+C to quit
  * Debugger PIN: 185-520-689
 ```
 
-We suggest you use POSTMAN or Hoppscotch to test the API with an uploaded file as the output whose format is CONLL-like (plese check out [temp_1.conllu](temp_1.conllu)). See the demonstration as the image below.
-
-![demo](./imgs/demo.png)
-
 ## 4. Docker version
 Run the following command:
 ```python
@@ -46,7 +42,60 @@ docker build -t ate .
 docker run -d -p 5000:5000 ate
 ```
 
-## 5. References
+## 5. Output format
+
+For each term, we return one entry for each term lemma, for which we return also the canonical form, pos tag, ranking, as well as an example of a term form and its msd. A response of the service is encoded in a JSON format composed of the following attributes:
+
+- `lemma`: a *string* presents the lemmatised form of the term (in the corpus, term forms with the same lemma are considered as the same terms).
+- `canonical`: a *string* presents the canonical form generated.
+- `frequency`: the *integer* representing the frequency of the lemmatised term in the corpus.
+- `ranking`: a *float* presents the termhood ranking.
+- `term_example_occurrence`: a *string* presents an example occurrence of the term in the corpus (a term can appear in different forms, e.g. different cases, and one example is returned). For the example term, we selected the one with the highest ranking score (see ranking attribute).
+- `term_example_pos`: a *string* presents the part-of-speech (POS) of each word in the term example occurrence.
+- `term_example_msd`: a *string* presents the msd of the term example occurrence.
+
+Please check out the format example below:
+
+```python
+[
+    {
+        "lemma": "plagiatorstvo",
+        "canonical": "plagiatorstvo",
+        "frequency": 2,
+        "ranking": 0.9977,
+        "term_example_occurrence": "plagiatorstvo",
+        "term_example_pos": "NOUN",
+        "term_example_msd": "Ncnsn"
+    },
+    {
+        "lemma": "obročen plačevanje",
+        "canonical": "obročno plačevanje",
+        "frequency": 1,
+        "ranking": 0.9976,
+        "term_example_occurrence": "obročnim plačevanjem",
+        "term_example_pos": "ADJ NOUN",
+        "term_example_msd": "Agpnsi Ncnsi"
+    },
+    {
+        "lemma": "delen odpis",
+        "canonical": "delni odpis",
+        "frequency": 1,
+        "ranking": 0.9971,
+        "term_example_occurrence": "delnega odpisa",
+        "term_example_pos": "ADJ NOUN",
+        "term_example_msd": "Agpmsg Ncmsg"
+    },
+    ...
+]
+```
+
+We suggest you use POSTMAN or Hoppscotch to test the API with an uploaded file as the output whose format is CONLL-like (plese check out [temp.conllu](temp.conllu)). See the demonstration as the image below.
+
+The output format:
+![demo](./imgs/demo.png)
+
+## 6. References
+
 The term extraction tool is an updated version of Tran et al. (2022), using the SloBERTa model.
 
 Hanh Thi Hong Tran, Matej Martinc, Andraz Repar, Antoine Doucet and Senja Pollak: A Transformer-based Sequence-labeling Approach to the Slovenian Cross-domain Automatic Term Extraction. Proc. of Jezikovne tehnologije in digitalna humanistika, 2022.

ATEapi/canonical_utils.py

@@ -1,108 +1,158 @@
+import argparse
+import csv
 import os
-import classla
+import string
-# classla.download('sl', logging_level='WARNING')
 
+import classla
 from lemmagen3 import Lemmatizer
 
-def lem_adj(gender, wrd):
+# classla.download("sl", logging_level="WARNING")
-    lem = Lemmatizer()
+classla_nlp_pipeline = classla.Pipeline(
-    if gender == 'm':
+    lang="sl",
-        lem.load_model(os.path.join('./model/lemmagen_models/kanon-adj-male.bin'))
+    processors="tokenize,pos,lemma,depparse",
-    elif gender == 'f':
+    tokenize_pretokenized=True,
-        lem.load_model(os.path.join('./model/lemmagen_models/kanon-adj-female.bin'))
+    logging_level="WARNING",
-    elif gender == 'n':
+    download_method=None
-        lem.load_model(os.path.join('./model/lemmagen_models/kanon-adj-neutral.bin'))
+)
 
-    form = lem.lemmatize(wrd)
+def _resolve_lemmagen_model_loc(model_name):
-    return form
+    basedir = os.path.dirname(__file__)
+    return os.path.join(basedir, "/app/model/lemmagen_models", model_name)
 
+_canon_lemmatizer = Lemmatizer()
+_canon_lemmatizer.load_model(_resolve_lemmagen_model_loc("kanon.bin"))
+canon_lemma = _canon_lemmatizer.lemmatize
 
-def process_nlp_pipeline(lang, text):
+ADJ_LEMMATIZER_LOC_MAP = {
-    nlp = classla.Pipeline(lang=lang, processors='tokenize,pos,lemma', tokenize_pretokenized=True, logging_level='WARNING')
+    ("m", "s"): _resolve_lemmagen_model_loc("kanon-adj-male.bin"),
-    doc = nlp(text)
+    ("m", "p"): _resolve_lemmagen_model_loc("kanon-adj-male-plural.bin"),
-    return doc
+    ("f", "s"): _resolve_lemmagen_model_loc("kanon-adj-female.bin"),
+    ("f", "p"): _resolve_lemmagen_model_loc("kanon-adj-female-plural.bin"),
+    ("n", "s"): _resolve_lemmagen_model_loc("kanon-adj-neutral.bin"),
+    ("n", "p"): _resolve_lemmagen_model_loc("kanon-adj-neutral-plural.bin"),
+}
 
+_ADJ_LEMMATIZER_CACHE = {}
+
+def lem_adj(gender, number, wrd):
+    lem_key = (gender, number)
+    if lem_key not in _ADJ_LEMMATIZER_CACHE:
+        assert lem_key in ADJ_LEMMATIZER_LOC_MAP
+        lemmatizer_model_loc = ADJ_LEMMATIZER_LOC_MAP[lem_key]
+        lemmatizer = Lemmatizer()
+        lemmatizer.load_model(lemmatizer_model_loc)
+        _ADJ_LEMMATIZER_CACHE[lem_key] = lemmatizer
+    lemmatizer = _ADJ_LEMMATIZER_CACHE[lem_key]
+    return lemmatizer.lemmatize(wrd)
 
 def get_adj_msd(head, word):
     feats = head.feats
     feats_dict = {}
-    feats = feats.strip().split('|')
+    feats = feats.strip().split("|")
     for f in feats:
-        f = f.strip().split('=')
+        f = f.strip().split("=")
         feats_dict[f[0]] = f[1]
-    gender = feats_dict['Gender']
+    gender = feats_dict["Gender"]
-    #print(gender)
+    if gender == "Masc" and len(word.xpos) == 6:
-    #gender = gender.strip().split('=')[1]
+        msd = word.xpos[:-1] + "ny"
-    if gender == 'Masc' and len(word.xpos) == 6:
+    elif gender == "Masc" and len(word.xpos) == 7:
-        msd = word.xpos[:-1]+'ny'
+        msd = word.xpos[:-1] + "y"
-    elif gender == 'Masc' and len(word.xpos) == 7:
+    elif gender == "Fem":
-        msd = word.xpos[:-1]+'y'
+        msd = word.xpos[:-1] + "n"
-    elif gender == 'Fem':
+    elif gender == "Neut":
-        msd = word.xpos[:-1]+'n'
+        msd = word.xpos[:-1] + "n"
-    elif gender == 'Neut':
-        msd = word.xpos[:-1]+'n'
     else:
-        msd = None
+        # msd = None
+        msd = "qqqqqq"  # hacky but it means that adverbs are just copied over to the canonical form
     return msd
 
+def _is_single_acronym(term):
+    # (single word, all uppercase and length less than 5 characters)
+    if len(term.words) == 1:
+        word = term.words[0].text
+        return len(word) < 5 and word.isupper()
+    return False
 
-def subfinder(mylist, pattern):
+def _join_term_words(term):
-    matches = []
+    return " ".join([w.text for w in term.words])
-    for i in range(len(mylist)):
-        if mylist[i].text.lower() == pattern[0] and [t.text.lower() for t in mylist[i:i+len(pattern)]] == pattern:
-            matches.append(mylist[i:i+len(pattern)])
-    return matches
-
-
-def find_canon(term):
-    head = None
-    pre = []
-    post = []
-    for word in term.words:
-        if word.upos == 'NOUN' or word.upos == 'PROPN':
-            head = word
-            break
-    if head is None:
-        if len(term.words) == 1:
-            head2 = term.words[0]
-            lem = Lemmatizer()
-            lem.load_model(os.path.join('./model/lemmagen_models/kanon.bin'))
-            head_form = lem.lemmatize(head2.text.lower())
-            return head_form
-        else:
-            return ' '.join([w.text for w in term.words])  # just return the input because we do not cover such case
-    else:
-        for word in term.words:
-            if word.id < head.id:
-                pre.append(word)
-            elif word.id > head.id:
-                post.append(word)
 
+def _process_pre(pre, head, gender, number):
     canon = []
     for el in pre:
         msd = get_adj_msd(head, el)
-        if msd is None:
+        if msd[0] == "A":
-            canon.append(el.lemma.lower())
+            form = lem_adj(gender, number, el.text.lower())
+            canon.append(form)
         else:
-            if msd[0] == 'A' and msd[3] == 'm':
+            canon.append(el.lemma.lower())
-                form = lem_adj('m', el.text.lower())
+    return canon
-                canon.append(form)
+
-            elif msd[0] == 'A' and msd[3] == 'f':
+def find_canon(term):
-                form = lem_adj('f', el.text.lower())
+    if _is_single_acronym(term):
-                canon.append(form)
+        return term.words[0].text
-            elif msd[0] == 'A' and msd[3] == 'n':
+
-                form = lem_adj('n', el.text.lower())
+    head = None
-                canon.append(form)
+    pre = []
+    post = []
+
+    for word in term.words:
+        if word.head == 0:
+            head = word
+        elif head is None:
+            pre.append(word)
+        else:
+            post.append(word)
+    ## special case where all words are proper nouns and each word is canonized independently
+    if all(w.upos == "PROPN" for w in term.words):
+        canon_name = [canon_lemma(w.text) for w in term.words]
+        return " ".join(canon_name)
+
+    if head is None:
+        if len(term.words) == 1:
+            head2 = term.words[0]
+            return canon_lemma(head2.text.lower())
+        else:
+            # just return the input because we do not cover such case
+            return _join_term_words(term)
+    if head.upos == "VERB":  # if the term is not a noun phrase
+        # just return the input because we do not cover such case
+        return _join_term_words(term)
+    if head.upos == "ADJ":
+        if len(term.words) == 1:  # for single word adjectives, return male form
+            return lem_adj("m", "s", term.words[0].text.lower())
+        else:
+            # just return the input because we do not cover such case
+            return _join_term_words(term)
+
+    gender = head.xpos[2]
+    number = head.xpos[3]
+    ending = head.lemma[-1]
+    if gender == "f" and number == "p" and ending in "ie":  # sani, hlače
+        canon = _process_pre(pre, head, gender, number)
+        canon.append(head.lemma)
+    elif gender == "m" and number == "p" and ending == "i":  # možgani
+        canon = _process_pre(pre, head, gender, number)
+        canon.append(head.lemma)
+    elif gender == "n" and number == "p" and ending == "a":  # vrata
+        canon = _process_pre(pre, head, gender, number)
+        canon.append(head.lemma)
+    else:
+        canon = _process_pre(pre, head, gender, "s")
+        head_form = canon_lemma(head.text.lower())
+        canon.append(head_form)
 
-    lem = Lemmatizer()
-    lem.load_model(os.path.join('./model/lemmagen_models/kanon.bin'))
-    head_form = lem.lemmatize(head.text.lower())
-    canon.append(head_form)
     for el in post:
         canon.append(el.text)
-    return ' '.join(canon)
+    return " ".join(canon)
 
 def process(forms):
-    text = '\n'.join(forms)
+    text = "\n".join(forms)
-    doc = process_nlp_pipeline('sl', text)
+    doc = classla_nlp_pipeline(text)
-    return [find_canon(sent) for sent in doc.sentences]
+    canonical_forms = []
+    for term in doc.sentences:
+        try:
+            canonical_form = find_canon(term)
+        except Exception:
+            canonical_form = _join_term_words(term)
+        canonical_forms.append(canonical_form)
+    return canonical_forms

ATEapi/main.py

@@ -3,8 +3,9 @@ import torch
 import torch.nn.functional as F
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 
-from utils import *
+from .utils import *
-from canonical_utils import *
+from .canonical_utils import *
+
 from flask import Flask, request, jsonify
 from transformers import AutoTokenizer, AutoModelForTokenClassification 
 
@@ -12,12 +13,13 @@ from transformers import AutoTokenizer, AutoModelForTokenClassification
 app = Flask(__name__)
 
 label_list=["n", "B-T", "T"]
-tokenizer = AutoTokenizer.from_pretrained('./model/term_extractor/')
+tokenizer = AutoTokenizer.from_pretrained('/app/model/term_extractor/')
-model = AutoModelForTokenClassification.from_pretrained('./model/term_extractor/', num_labels=len(label_list)).to(device)
+model = AutoModelForTokenClassification.from_pretrained('/app/model/term_extractor/', num_labels=len(label_list)).to(device)
 
 @app.route('/predict',methods=['POST'])
 def predict():
     frame = read_conll(request.files['file'])
+    # print(frame)
     sequences = [' '.join(x) for x in frame.word]
     lemma, pos, msd = frame.lemma, frame.pos, frame.msd
     preds = []
@@ -36,30 +38,33 @@ def predict():
         final_preds.append(p)
         final_probs.append(p1)
     predicted_terms, prob_terms, lemma_terms, pos_terms, msd_terms = extract_terms_full(final_preds, final_probs, texts, lemma, pos, msd)
-    df = pd.DataFrame({'terms':predicted_terms,
+    if len(predicted_terms) == 0:
-                    'raw_prob':prob_terms,
+        return jsonify({'term_example_occurrence': 'No terms found'})
-                    'lemma':lemma_terms,
+    else:
-                    'pos':pos_terms,
+        df = pd.DataFrame({'term_example_occurrence':predicted_terms,
-                    'msd':msd_terms})
+                        'raw_prob':prob_terms,
-    df = df.drop_duplicates(subset=['lemma','pos'], keep='first')
+                        'lemma':lemma_terms,
-    # print(df.head(5))
+                        'term_example_pos':pos_terms,
-    df['prob'] = pd.Series(dtype='float')
+                        'term_example_msd':msd_terms})
-    for i in range(len(df)):
+        df = df.drop_duplicates(subset=['lemma','term_example_pos'], keep='first')
-        temp = [float(x) for x in df['raw_prob'].iloc[i].split(' ')]
+        
-        df['prob'].iloc[i] = round(sum(temp)/len(temp),4)
+        df['ranking'] = pd.Series(dtype='float')
+        for i in range(len(df)):
+            temp = [float(x) for x in df['raw_prob'].iloc[i].split(' ')]
+            df['ranking'].iloc[i] = round(sum(temp)/len(temp),4)
 
-    df = df.sort_values(by=['lemma','prob'], ascending=True)
+        df = df.sort_values(by=['lemma','ranking'], ascending=True)
-    df = df.drop_duplicates(subset=['lemma'], keep='last')
+        df = df.drop_duplicates(subset=['lemma'], keep='last')
-    df['canonical'] = process(df['terms'])
+        df['canonical'] = process(df['term_example_occurrence']) 
-    df = df[['terms', 'canonical', 'lemma','pos','msd','prob']].rename(columns={'prob':'ranking'})
+        corpus = ' '.join([' '.join(x) for x in lemma])
-    # sort by ranking 
+        df['frequency'] = [corpus.count(x) for x in df['lemma']]
-    # print(df.head(5))
+        df = df[[ 'lemma', 'canonical', 'frequency','ranking','term_example_occurrence', 'term_example_pos','term_example_msd']]
-    df = df[df['pos'] != 'PUNCT']
+        df = df[df['term_example_pos'] != 'PUNCT']
-    df = df.query("terms.str.len() > 2")
+        df = df.query("term_example_occurrence.str.len() > 2")
-    df = df.sort_values('ranking', ascending=False).drop_duplicates(subset=['terms','lemma'], keep = 'first').sort_index()
+        df = df.drop_duplicates(subset=['term_example_occurrence','lemma'], keep = 'first')
-    print(df.head(5))
+        df = df.sort_values(by=['ranking'], ascending=False)
-    return df.to_json(orient='records')
+        # print(df.head(5))
-    # return jsonify(df.to_dict(orient='records'))
+        return df.to_json(orient='records')
 
 
 if __name__ == '__main__':

ATEapi/requirements.txt

@@ -1,7 +1,8 @@
-classla==1.1.0
+classla==1.2.0
 Flask==2.2.2
 lemmagen3==3.3.2
-numpy==1.21.6
+numpy==1.23.0
 pandas==1.3.5
-torch
+torch==1.12.0
 transformers==4.20.1
+gunicorn==20.1.0

README.md

@@ -37,7 +37,7 @@ Natančen opis in izvorni repozitorij dela projekta se nahaja na: https://github
 
 ### Podatkovna baza korpusa OSS 
 
-Podatkovna baza korpusa OSS vsebuje obdelana besedila iz korpusa OSS ter hkrati tudi iskalne indekse, ki omogočajo hitrejše poizvedbe po podatkih. Prav tako so v podatkovni bazi predizračunani hevristični indeksi za iskanje terminoloških kandidatov ter procedure za obdelavo uporabniških besedil na enak način. Ker je sama vsebina podatkovne baze prevelika, so dodane samo skripte za kreiranje strukture tabele, sam korpus pa je v bazo potrebno po instalaciji uvoziti in zagnati njegovo obdelavo.
+Podatkovna baza korpusa OSS vsebuje obdelana besedila iz korpusa OSS ter hkrati tudi iskalne indekse, ki omogočajo hitrejše poizvedbe po podatkih. Prav tako so v podatkovni bazi predizračunani hevristični indeksi za iskanje terminoloških kandidatov ter procedure za obdelavo uporabniških besedil na enak način. Ker je sama vsebina podatkovne baze prevelika, so dodane samo skripte za kreiranje strukture tabele, sam korpus pa je v bazo potrebno po instalaciji uvoziti in zagnati njegovo obdelavo. Ker je za obdelavo potrebnih precej sistemskih sredstev, bodo obdelani podatki tudi ločeno objavljeni.
 
 Uporabljena je odprtokodna podatkovna baza MariaDB z ColumnStore hrambo. 
 

canonical_forms/Dockerfile.prod

@@ -58,14 +58,15 @@ RUN pip install --no-cache /wheels/*
 # copy entrypoint-prod.sh
 COPY ./entrypoint.prod.sh $APP_HOME
 
+
 # copy project
 COPY . $APP_HOME
-
+RUN chmod u+x $APP_HOME/entrypoint.prod.sh
 # chown all the files to the app user
 RUN chown -R app:app $APP_HOME
 
 # a workaround for permission errors for the classla_volume
-RUN mkdir $HOME/classla_resources && chown app:app $HOME/classla_resources
+# RUN mkdir $HOME/classla_resources && chown app:app $HOME/classla_resources
 
 # change to the app user
 USER app

canonical_forms/canonizer.py

@@ -7,7 +7,7 @@ import classla
 from lemmagen3 import Lemmatizer
 
 
-classla.download("sl", logging_level="WARNING")
+#classla.download("sl", logging_level="WARNING")
 classla_nlp_pipeline = classla.Pipeline(
     lang="sl",
     processors="tokenize,pos,lemma,depparse",

canonical_forms/entrypoint.prod.sh

@@ -4,6 +4,6 @@
 sed -i 's/signal(/pass;#signal(/g' /usr/local/lib/python3.8/site-packages/classla/submodules/reldi_tokeniser/tokeniser.py
 
 # pre-download models
-python -c "import classla;classla.download('sl', logging_level='INFO')"
+#python -c "import classla;classla.download('sl', logging_level='INFO')"
 
 exec "$@"

canonical_forms/entrypoint.sh

@@ -5,6 +5,6 @@ echo "Patching tokeniser.py..."
 sed -i 's/signal(/pass;#signal(/g' /usr/local/lib/python3.8/site-packages/classla/submodules/reldi_tokeniser/tokeniser.py
 
 # pre-download models
-python -c "import classla;classla.download('sl', logging_level='INFO')"
+# python -c "import classla;classla.download('sl', logging_level='INFO')"
 
 exec "$@"

canonical_forms/requirements.txt

@@ -1,7 +1,7 @@
 Flask==2.1.*
 flask-restx
 gunicorn==20.1.0
-classla==1.0.1
+classla==1.1.0
 lemmagen3==3.3.1
 flask-socketio==5.1.0
 simple-websocket==0.2.0

definition_extraction/Dockerfile

@@ -0,0 +1,21 @@
+# pull official base image
+FROM python:3.8
+
+# set work directory
+WORKDIR /usr/src/app
+
+# set environment variables
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+
+# install dependencies
+RUN pip install --upgrade pip
+COPY ./requirements.txt /usr/src/app/requirements.txt
+RUN pip install -r requirements.txt
+#RUN --mount=type=cache,target=/root/.cache/pip pip install -r requirements.txt
+
+# copy project
+COPY . /usr/src/app/
+
+# run entrypoint.sh
+ENTRYPOINT ["/usr/src/app/entrypoint.sh"]

definition_extraction/Dockerfile.prod

@@ -0,0 +1,58 @@
+###########
+# BUILDER #
+###########
+
+# pull official base image
+FROM python:3.8 as builder
+
+# set work directory
+WORKDIR /usr/src/app
+
+# set environment variables
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+
+# install python dependencies
+COPY ./requirements.txt .
+RUN pip wheel --no-cache-dir --no-deps --wheel-dir /usr/src/app/wheels -r requirements.txt
+
+
+#########
+# FINAL #
+#########
+
+# pull official base image
+FROM python:3.8
+
+# create directory for the app user
+RUN mkdir -p /home/app
+
+# create the app user
+RUN addgroup --system app && adduser --system --group app
+
+
+# create the appropriate directories
+ENV HOME=/home/app
+ENV APP_HOME=/home/app/web
+RUN mkdir $APP_HOME
+WORKDIR $APP_HOME
+
+COPY --from=builder /usr/src/app/wheels /wheels
+COPY --from=builder /usr/src/app/requirements.txt .
+RUN pip install --upgrade pip
+RUN pip install --no-cache /wheels/*
+
+COPY ./entrypoint.prod.sh $APP_HOME
+
+# copy project
+COPY . $APP_HOME
+RUN chmod u+x $APP_HOME/entrypoint.prod.sh
+
+# chown all the files to the app user
+RUN chown -R app:app $APP_HOME
+
+# change to the app user
+USER app
+
+# run entrypoint.prod.sh
+ENTRYPOINT ["/home/app/web/entrypoint.prod.sh"]

definition_extraction/JeStaSoPatterns_finishOK.txt

@@ -0,0 +1,9 @@
+{{{\S+_____\S+_____N\S+\s+\S+_____\S+_____Va-r3[psd]-n\s+\S+_____\S+_____N\S+\s*}}}	#N je/sta/so N
+{{{\S+_____\S+_____N...ny?n?\s+\S+_____\S+_____Va-r3[psd]-n\s+\S+_____\S+_____N...ny?n?\s*}}}	#Sam-nom je/sta/so Sam-nom
+{{{(\S+_____\S+_____A\S+\s+)*\S+_____\S+_____N...ny?n?\s+((\S+_____\S+_____A\S+\s+)*\S+_____\S+_____N\S+\s+)*([a-z]+_____\S+_____S\S+\s+(\S+_____\S+_____A\S+\s+)*(\S+_____\S+_____N\S+\s+)+)*\S+_____\S+_____Va-r3[psd]-n\s*(\S+_____\S+_____A\S+\s+)*(\S+_____\S+_____N...ny?n?\s*)}}}
+{{{(\S+_____\S+_____A\S+\s+)*\S+_____\S+_____N...ny?n?\s+((\S+_____\S+_____A\S+\s+)*\S+_____\S+_____N\S+\s+)*([a-z]+_____\S+_____S\S+\s+(\S+_____\S+_____A\S+\s+)*(\S+_____\S+_____N\S+\s+)+)*\S+_____\S+_____Va-r3[psd]-n\s*(\S+_____\S+_____A\S+\s+)*(\S+_____\S+_____N...ny?n?\s+)}}}
+{{{^(\S+_____\S+_____A\S+\s+)*\S+_____\S+_____N...ny?n?\s+((\S+_____\S+_____A\S+\s+)*\S+_____\S+_____N\S+\s+)*([a-z]+_____\S+_____S\S+\s+(\S+_____\S+_____A\S+\s+)*(\S+_____\S+_____N\S+\s+)+)*\S+_____\S+_____Va-r3[psd]-n\s*(\S+_____\S+_____A\S+\s+)*(\S+_____\S+_____N...ny?n?\s*)}}}
+{{{^(\S+_____\S+_____A\S+\s+)*\S+_____\S+_____N...ny?n?\s+((\S+_____\S+_____A\S+\s+)*\S+_____\S+_____N\S+\s+)*([a-z]+_____\S+_____S\S+\s+(\S+_____\S+_____A\S+\s+)*(\S+_____\S+_____N\S+\s+)+)*\S+_____\S+_____Va-r3[psd]-n\s*(\S+_____\S+_____A\S+\s+)*(\S+_____\S+_____N...ny?n?\s+)}}}
+{{{(\S+_____\S+_____N...ny?n?\s+)((\S+_____\S+_____A\S+\s+)*((\S+_____\S+_____A\S+\s+)+((in)|(ali))_____\S+_____Cc\s*(\S+_____\S+_____A\S+\s+))*\S+_____\S+_____N\S+\s+)*([a-z]+_____\S+_____S\S+\s+(\S+_____\S+_____A\S+\s+)*((in)|(ali)_____\S+_____C\S*\s*\S+_____\S+_____A\S+\s*)*(\S+_____\S+_____N\S+\s+)+)*(angl?\._____\S+_____\S+\s+(\S+_____\S+_____[^V]\S*\s+)+)*\S+_____\S+_____Va-r3[psd]-n\s*(\S+_____\S+_____A\S+\s+)*((\S+_____\S+_____A\S+\s+)+((in)|(ali))_____\S+_____Cc\s*(\S+_____\S+_____A\S+\s+))*\S+_____\S+_____N...ny?n?\s*}}}
+
+

definition_extraction/SentEx_patternsF.pl

@@ -0,0 +1,284 @@
+#!/usr/bin/perl - w
+
+#script to extract occurrences of definitions in a File
+#output: definition candidates sorted by patterns
+#usage: perl SentEx_patternsF.pl Korpus Patternlist
+
+
+
+use utf8;
+binmode STDIN, 'utf8';
+binmode STDOUT, 'utf8';
+
+$file = shift(@ARGV);
+$pat = shift(@ARGV);
+if ($file=~/\//){
+($filestem) = $file=~ /.*\/(.+)\..../;
+}
+else {
+($filestem) = $file=~ /(.+)\..../;
+}
+
+print STDERR "F:$filestem\n";
+
+print STDERR "\n\nDo you want to evaluate against recall test set? If yes enter recall test set file else press return!\n";
+# $recalltestset=<>;
+$recalltestset="";
+&recalltestset;
+
+
+open(PAT, "<:utf8", "$pat") || die "Cannot open file:$!";
+open(OUT, ">:utf8", "ALLPATTERNEVALUATION_$filestem"."\.txt") || die "Cannot open outfile:$!";
+
+push @predefinedpatterns, "0"; #zato da se potem zacne z 1 ne 0
+while (<PAT>){
+chomp;
+if (length>0){
+  if (m/^\#/){}
+else {
+print STDERR "[$_]\n\n";
+s/{{{(.*)}}}/$1/;
+push @predefinedpatterns, "$1";
+
+}
+}
+}
+$k=$#predefinedpatterns;
+for ($i=1;$i<=$k; $i++){
+$count=0;
+$yes=0;
+$no=0;
+$undef=0;
+$allpozit=0;
+$allneg=0;
+$stop=0;
+$eval=0;
+@comments=();
+
+print STDERR "PATLIST$i:". $predefinedpatterns[$i]."\n";
+
+#open(OUT1, ">:utf8", "PAT_lema".$i."_$filestem."."xml") || die "Cannot open outfile1:$!";
+open(OUT2, ">:utf8", "PAT_".$i."_$filestem."."txt") || die "Cannot open outfile2:$!";
+
+open(FILE, "<:utf8", "$file") || die "Cannot open file:$!";
+@positivesbyeachpattern=();
+
+  $/ = "\/>\n";#"\n\n";#"<S/>";
+  while (<FILE>) {  
+    $s = $_;
+
+      if ($s=~/defvalue=\"Y/){
+	$allpozit++;
+      }
+      if ($s=~/defvalue=\"N/){
+	$allneg++;
+      }
+    if ($s =~/.*?(<id_sp.*?\>).*?\n/){
+
+($ids)= $s =~/.*?(<id_sp.*?\>).*?\n/g; 
+
+
+}
+
+$s =~s/.*?<id_sp.*?\>.*?\n//gs; 
+($sentmeta)= $s =~ /(<S sid_sp.*)/;
+#print STDERR "$sentmeta";
+@wordforms = $s =~ /(.*?)\t.*?\n/gs;
+    $string = join(" ", @wordforms);
+
+    @lines = split(/\n/, $s);
+
+    @msds = ();
+   @token_type_msd = ();
+    foreach $line (@lines) {
+
+     if ($line =~ /((.*?\t+TOK\t)|(angl?\.?\t+TOK_ABBR\t))/){
+
+
+  ($token,$type,$msd) = $line =~ /(.+?)\s*\t+.+\t+(.+?)\s*\t+(.+?)\s*\t*\s*$/;
+#$msd=~s/(.*?)\s+/$1/;
+$token_type_msd=$token."_____".$type."_____".$msd;
+ #print STDERR $token_type_msd."\n";
+#print "$token_type_msd"."\n";
+          push (@token_type_msd, $token_type_msd);
+          }
+ 
+
+      
+    } 
+    $msd_string = join(" ", @token_type_msd); 
+#print STDERR  $msd_string."]\n";
+#  print $msd_string."\n\n\n";
+#print "PAT$i:","$predefinedpatterns[$i]"."\n";
+if ($msd_string =~ /$predefinedpatterns[$i]/){    #v resnici token type msd
+#print STDERR  $msd_string."]\n\n";
+#print STDERR "\nYES PREVIOUS MATCHED\n";
+if ($stop==0){#da ga samo enkrat sprinta
+print OUT2 "\n>>>>".$i.":\t"."{{{$predefinedpatterns[$i]}}}"."\n";
+$stop=1;
+}
+      #print OUT1 "$s"."\n";
+      print OUT2 "$string###$sentmeta"."\n";
+$sentmeta=~/S sid_sp=\"(\d+)\"/;
+$idnb=$1;
+$union{$idnb}="$string###$sentmeta"."\n";
+#print OUT2 "MSD string: $msd_string"."\n";
+if ($sentmeta=~/defvalue=\"Y/){
+$yes++;
+$unionYES{$idnb}++;
+push @positivesbyeachpattern, $idnb;
+
+}
+elsif ($sentmeta=~/defvalue=\"N/){
+$no++;
+$unionNO{$idnb}++;
+}
+else {
+$undef++;
+$unionUNDEF{$idnb}++;
+}
+$count++;
+
+    }
+
+}
+
+
+
+
+
+
+
+
+$eval=$count-$undef;
+
+
+if ($count==0) {
+$count=0.000001;
+push @comments, "\$undefWas0";
+}
+if ($allpozit==0) {
+$allpozit=0.000001;
+push @comments, "\$allpozitWas0";
+}
+
+if ($eval==0) {
+###print OUT "EVAL IS ".$eval." and changed to=0.00001 \n";
+$eval=0.00001;
+###print OUT "nEWEVAL IS ".$eval.":::\n";
+push @comments, "\$evalWas0";
+}    
+
+print OUT "PATTERN$i:\t{{{$predefinedpatterns[$i]}}}\n";
+print OUT "TOTAL EXAMPLES: ".$count."\n";
+print OUT "TOTAL EVALUATED: ".$eval."\n";
+print OUT "POZITIVES:". $yes."\n";
+print OUT "NEGATIVES:". $no."\n";
+print OUT "UNDEFINED:". $undef."\n";
+print OUT "ALLPOZ:".$allpozit."\n";
+print OUT "ALLNEG:".$allneg."\n";
+print OUT "PRECISION_eval:". $yes/$eval."\n";
+#print OUT "PRECISION_noneval:". $yes/$count."\n";
+#print OUT "RECALL:".$yes/$allpozit."\n";
+#print OUT "POZ//UNDEF:\t"."$yes//".$undef."\n";
+#print OUT "PREC//RECALL:".$yes/$eval."//".$yes/$allpozit;
+
+#print OUT "POZITIVESBYPATTER>$i: "."@positivesbyeachpattern";
+
+#presek  @recalltestsetids in @positivesbyeachpattern
+  foreach $element (@positivesbyeachpattern,  @recalltestsetids) { $count{$element}++ }
+    foreach $element (keys %count) {
+	push @{ $count{$element} > 1 ? \@intersection : \@difference }, $element;
+    }  
+print OUT "INTERSECTION:"."number elements:[".@intersection."]:"."@intersection"."\n";
+&computerecall;
+print OUT "RECALL ON RECALL TEST SET:".@intersection."/". @recalltestsetids."=$divided\n";
+print OUT "(recall test set is:$recalltestset)\n";
+
+
+#print OUT "Ints reason : "."PozByPatt:"."@positivesbyeachpattern"."recallset"." @recalltestsetids"."\n";
+#$ints=@intersection."\n";
+#print STDERR "\n\nRECALL: $ints"."\\"."$number_recall"."\n";
+
+#print OUT "WorstPrec (if all non eval were ng:".$yes/$count;
+print OUT "\n\n======\n\n";
+#print OUT "COMMENTS:"."@commetns";
+
+%count=();
+@intersection=();
+
+}
+
+open (OUTUNION, ">:utf8", "outunion.txt");
+foreach $key (sort sortAsc keys %union){
+print OUTUNION "$union{$key}";
+
+}
+@unionpositivesbyeachpattern= keys %unionYES;
+  foreach $element (@unionpositivesbyeachpattern,  @recalltestsetids) { $count4union{$element}++ }
+    foreach $element (keys %count4union) {
+	push @{ $count4union{$element} > 1 ? \@intersection4union : \@difference4union }, $element;
+    }  
+&computerecall;
+
+print OUT "\n\n\nALL PATTERNS, i.e. UNION\n";
+
+
+
+print OUT "TOTAL EXAMPLES in UNION: ";
+print OUT scalar  keys %union; $scalarall= scalar keys %union;
+print OUT "\n";
+print OUT "POZITIVES:";
+print OUT scalar keys %unionYES; $scalaryes= scalar keys %unionYES;
+print OUT "\n";
+print OUT "NEGATIVES:";
+print OUT scalar keys %unionNO;
+print OUT "\n";
+print OUT "UNDEFINED:";
+print OUT scalar keys %unionUNDEF; $scalarundefined= scalar keys %unionUNDEF;
+print OUT "\n";
+$scalardefined=$scalarall-$scalarundefined;
+# print OUT "PRECISION_eval:". $scalaryes/ $scalardefined."\n";
+print OUT "INTERSECTION:"."number elements4union:[".@intersection4union."]:"."@intersection4union"."\n";
+  if ($chosenrecall>0){
+$iu=@intersection4union;
+$rt=@recalltestsetids;
+$dividedunion=$iu/$rt;
+print OUT "RECALL ON RECALL TEST SET:".@intersection4union."/". @recalltestsetids."=$dividedunion\n";
+print OUT "(recall test set is:$recalltestset)\n";
+}
+
+  sub sortAsc {
+
+           $a<=>$b;
+         }
+
+sub recalltestset {
+print STDERR "you entered:[".$recalltestset."]\n";
+# chomp $recalltestset;
+$recalltestset=~s/\n*\r*$//g;
+print STDERR "CHOMPED:: [".$recalltestset."]\n";
+
+if ($recalltestset=~/.../){
+$chosenrecall=1;
+open(RCSET, "<:utf8", "$recalltestset") || die "Cannot open RCSET file:$!";
+while (<RCSET>){
+  if (m/S sid_sp=\"(\d+)\"/){
+push @recalltestsetids, $1;
+}
+}
+
+}
+else {print STDERR "\nok no test set for recall!\n";}
+
+}
+
+sub computerecall {
+  if ($chosenrecall>0){
+$is=@intersection;
+$rs=@recalltestsetids;
+$divided=$is/$rs;
+}
+}
+
+print STDERR "total number of recall testset:  ".@recalltestsetids."\n";

definition_extraction/__init__.py

@@ -0,0 +1 @@
+ 

definition_extraction/app.py

@@ -0,0 +1,84 @@
+import os
+import json
+import tempfile
+
+from flask import Flask
+from flask_restx import Api, Resource, fields
+
+from werkzeug.middleware.proxy_fix import ProxyFix
+from werkzeug.datastructures import FileStorage
+from werkzeug.exceptions import BadRequest
+
+try:
+    from . import extract_defsent as extractor
+except ImportError:
+    import extract_defsent as extractor
+
+
+UPLOAD_FOLDER = 'uploads'
+MAX_TERMS_STRLEN = 100000
+
+app = Flask(__name__)
+app.wsgi_app = ProxyFix(app.wsgi_app)
+api = Api(app, version='1.0', title='Definition sentence extraction',
+          description='''A simple API for the extraction of definition sentence candidates from a given CoNLL-U file. A list of candidate terms (can also be empty) is used to filter input sentences.
+                         **NOTE**: Because the service accepts a file and an additional parameter (terms), the request's _Content-Type_ cannot be ```application/json``` but ```multipart/form-data```.
+                         As a result, the terms parameter must be a string in the POST request form data and must contain valid JSON.
+                        ''')
+app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
+ns = api.namespace('DefExAPI', description='Definition extraction API namespace')
+
+
+def terms_as_json(value):
+    '''Parses and validates terms stored in JSON string'''
+    if len(value) > MAX_TERMS_STRLEN:
+        raise ValueError(f'Security error: string too long! Must be of length <= {MAX_TERMS_STRLEN}.')
+    try:
+        lem_terms = json.loads(value)
+        if 'lemmatized_terms' not in lem_terms or not isinstance(lem_terms['lemmatized_terms'], list):
+            raise Exception('''Invalid JSON format for terms, must be like {"lemmatized_terms": ["first term", "second term", ...]}.''')
+        tlist = [str(x) for x in lem_terms['lemmatized_terms']]
+    except Exception as e:
+        raise ValueError(f'Error while parsing terms JSON string: {str(e)}')
+    else:
+        return tlist
+
+
+# Swagger documentation
+terms_as_json.__schema__ = {'type': str, 'format': 'JSON'}
+
+
+parser = api.parser()
+parser.add_argument('terms', type=terms_as_json, location='form', required=False)
+parser.add_argument('conllu_file', type=FileStorage, location='files', required=True)
+
+
+#@api.route('/with-parser/', endpoint='with-parser')
+@ns.route('/definition_sentence_extraction')
+@ns.expect(parser, validate=True)
+@ns.doc(params={'terms': f'''This is a string contaning an _optional_ list of **lemmatized terms** for which we want to extract potential definition sentence candidates.
+                This string should contain a valid JSON where terms are under key **lemmatized_terms**, e.g.:
+                ```{{"lemmatized_terms": ["first term", "second term", ...]}}```
+                **NOTE**: For security reasons, the length of the string is limited to {MAX_TERMS_STRLEN} characters.''',
+                'conllu_file': '''This is a mandatory parameter containing a valid CoNLL-U file.'''
+                })
+class DefinitionSentenceExtractionService(Resource):
+    def post(self):
+        args = parser.parse_args()
+        terms = args['terms'] if args['terms'] is not None else []
+        conllu_filestorage = args['conllu_file']
+
+        tempdir = tempfile.TemporaryDirectory()
+        tmpfile = os.path.join(tempdir.name, 'input.conllu')
+        with open(tmpfile, 'w') as fp:
+            fp.write(conllu_filestorage.read().decode())
+        try:
+            sentences = extractor.mp_extract(tmpfile, terms)
+        except Exception as e:
+            raise BadRequest(str(e))
+        tempdir.cleanup()
+        return {'definition_candidates': sentences}
+
+
+#if __name__ == '__main__':
+    #app.run(debug=True)

definition_extraction/entrypoint.prod.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec "$@"

definition_extraction/entrypoint.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec "$@"

definition_extraction/extract_defsent.py

@@ -0,0 +1,205 @@
+import sys
+import os
+import shutil
+import atexit
+import tempfile
+import subprocess
+import argparse
+import json
+import time
+import re
+import multiprocessing as mp
+
+import conllu
+
+
+termex_perl = 'SentEx_patternsF.pl'
+termex_pattern1 = 'JeStaSoPatterns_finishOK.txt'
+input_fname = 'input.lines'
+output_union = 'outunion.txt'
+
+
+
+#def cleanup(tempdir):
+    #if isinstance(tempdir, tempfile.TemporaryDirectory):
+        #print(f'INFO: Removing temporary folder: {tempdir.name}', file=sys.stderr)
+        #tempdir.cleanup()
+
+
+def words_search(words_string, sentence_string):
+    #words_string = " " + words_string.strip() + " "
+    #sentence_string = " " + sentence_string.strip() + " "
+    words_string = " " + words_string + " "
+    sentence_string = " " + sentence_string + " "
+    return sentence_string.find(words_string)
+
+
+# this search works for searching single whitespace joined words in other single whitespace joined words
+# See here (https://docs.python.org/3/library/re.html) for the definition of \b
+# NOTE: this is slow!
+def find_words_in_string(words, string):
+    if re.search(r"\b" + re.escape(words) + r"\b", string):
+        return True
+    return False
+
+
+# returns all non-overlapping starting sublist positions
+def sublist(lst, sub):
+    elt = sub[0]
+    pos = 0
+    allpos = []
+    while lst != []:
+        if elt in lst:
+            idx = lst.index(elt)
+            pos += idx
+            if lst[idx: idx + len(sub)] == sub:
+                allpos.append(pos)
+                lst = lst[idx + len(sub):]
+                pos += len(sub)
+                # return pos
+            else:
+                lst = lst[idx + 1:]
+                pos += 1
+        else:
+            break
+            # return -1
+    return allpos
+    # return -1
+
+
+# returns first sublist position or -1
+def is_sublist(lst, sub):
+    elt = sub[0]
+    pos = 0
+    while lst != []:
+        if elt in lst:
+            idx = lst.index(elt)
+            pos += idx
+            if lst[idx: idx + len(sub)] == sub:
+                return pos
+            else:
+                lst = lst[idx + 1:]
+                pos += 1
+        else:
+            return -1
+    return -1
+
+
+def split_conllu_file(conllu_file, n):
+    tempdir = tempfile.TemporaryDirectory()
+    #print('-->', tempdir.name)
+
+    parts = [[] for i in range(n)]
+    with open(conllu_file, 'r', encoding="utf-8") as ifp:
+        for i, sent in enumerate(conllu.parse_incr(ifp)):
+            parts[i%n].append(sent)
+
+    outfiles = [os.path.join(tempdir.name, f'{i}.conllu') for i in range(n)]
+    for fname, sentences in zip(outfiles, parts):
+        with open(fname, 'w') as fp:
+            for s in sentences:
+                fp.write(s.serialize())
+    return tempdir, outfiles
+
+
+def read_terms_json_file(fname):
+    with open(fname) as fp:
+        lem_terms = json.load(fp)
+
+    if 'lemmatized_terms' not in lem_terms or not isinstance(lem_terms['lemmatized_terms'], list):
+        raise Exception('''Invalid JSON format for terms, must be like {"lemmatized_terms": ["first term", "second term", ...]}.''')
+    return [str(x).strip() for x in lem_terms['lemmatized_terms']]
+
+
+
+def extract_definition_sentences(conllu_file, terms=[]):
+    # initialize temp directory
+    tempdir = tempfile.TemporaryDirectory()
+
+    st = time.time()
+
+    # prepare input data in correct format
+    with open(conllu_file, 'r', encoding="utf-8") as ifp:
+        with open(os.path.join(tempdir.name, input_fname), 'w') as ofp:
+            for i, sent in enumerate(conllu.parse_incr(ifp)):
+                if terms != []:
+                    lem_sent = ' '.join([tok['lemma'] for tok in sent])
+                    for term in terms:
+                        if term in lem_sent: # first test: string search, can lead to false results (substrings, not whole words)
+                            term_tokens = term.split()
+                            lem_tokens = [tok['lemma'] for tok in sent]
+                            if is_sublist(lem_tokens, term_tokens) != -1:  #slower exact testing
+                                # if words_search(term, lem_sent) != -1:
+                                # if is_sublist(lem_tokens, term) != -1:
+                                # if find_words_in_string(term, lem_sent):
+                                # if term in lem_sent:
+                                #if len(term)>1:
+                                # print('--->', term, lem_tokens)
+                                for tok in sent:
+                                    print(f"{tok['form']}\tTOK\t{tok['lemma']}\t{tok['xpos']}", file=ofp)
+                                sid = sent.metadata.get('sent_id', i)
+                                print(f'''\t\t\t<S sid_sp="{sid}" aid_sp="{conllu_file}" defvalue="" term="{term}"/>\n''', file=ofp)
+                else:
+                    for tok in sent:
+                        print(f"{tok['form']}\tTOK\t{tok['lemma']}\t{tok['xpos']}", file=ofp)
+                    sid = sent.metadata.get('sent_id', i)
+                    print(f'''\t\t\t<S sid_sp="{sid}" aid_sp="{conllu_file}" defvalue=""/>\n''', file=ofp)
+
+    #print('Preparation and filtering time: ', time.time() - st)
+    st = time.time()
+
+    for fn in [termex_perl, termex_pattern1]:
+        shutil.copyfile(fn, os.path.join(tempdir.name, fn))
+
+    # run extractor
+    p = subprocess.run(['perl', termex_perl, input_fname, termex_pattern1],
+                       stderr=subprocess.DEVNULL,
+                       cwd=tempdir.name)
+
+    if p.returncode != 0:
+        raise IOError('Term extraction process failed, check its perl script.')
+
+    resultfile = os.path.join(tempdir.name, output_union)
+    if not os.path.exists(resultfile):
+        raise IOError('Output file with union of results does not exist: {resultfile}')
+
+    #print('Extraction time: ', time.time() - st)
+
+    lines = open(resultfile).read()
+    print(lines)
+    return [{
+            "definicija":line.split('###')[0].strip(),
+            "term": re.search('term="(.*)"', line).group(1)
+            } for line in lines.split('\n') if line.strip()]
+
+
+def mp_extract(conllu, terms, ncores=os.cpu_count()):
+    defs = []
+    conllu_tempdir, conllu_part_files = split_conllu_file(conllu, ncores)
+    with mp.Pool(ncores) as pool:
+        params = zip(conllu_part_files, [terms] * len(conllu_part_files))
+        for i, result in enumerate(pool.starmap(extract_definition_sentences, params)):
+            print(result)
+            defs.extend(result)
+
+    #return sorted(list(set(defs)))
+    return defs;
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('CoNLLU_file', help='Input file in CoNLL-U format')
+    parser.add_argument('-n', '--ncpu', type=int, help='Number of CPU cores to use. Leave empty to use all cores.')
+    parser.add_argument('-t', '--terms', help='Optional input file with lemmatized terms for filtering the input file, formatted like this: {"lemmatized_terms": ["first term", "second term", ...]} ')
+    args = parser.parse_args()
+
+    #sents = extract_definition_sentences(args.CoNLLU_file)
+    #print('\n'.join(list(set(sents))))
+
+    ncores = args.ncpu if args.ncpu is not None else os.cpu_count()
+    lem_terms = read_terms_json_file(args.terms) if args.terms else []
+
+    defs = mp_extract(args.CoNLLU_file, lem_terms, ncores)
+    for s in defs:
+        print(s)

definition_extraction/requirements.txt

@@ -0,0 +1,6 @@
+conllu==4.5.*
+gunicorn==20.1.*
+
+werkzeug==2.1.2
+flask==2.1.3
+flask-restx==0.5.1

docker-compose.yml

@@ -5,34 +5,35 @@ services:
     restart: always
     image: flask-server
     ports:
-      - "8080:8080"
+      - "8081:8080"
-    volumes: 
+    volumes: # when pushing to server, remove the first dot here in front of /mnt
+      - /mnt/ssd/ds_ftp/classla_OS2022:/usr/src/app/classla_OS2022
       - ../classla/classla_resources:/root/classla_resources
+      - ../jobs_database_rsdo5:/usr/src/app/DB
     environment:
       - PYTHONUNBUFFERED=1
       - MDB_DATABASE=conllus_150k
-      - MDB_HOST=mariadb
+      - MDB_HOST=164.8.252.72
-      - MDB_PORT=3306
+      - MDB_PORT=55655
       - MDB_USER=rsdo5
       - MDB_PASSWORD=rsdo5mysql
-  
   tika2:
     image: apache/tika:1.28.4-full
     entrypoint: [ "/bin/sh", "-c", "java -jar /tika-server-1.28.4.jar -h 0.0.0.0 -p 9999" ]
     restart: always
-    ports:
+    #ports:
-      - "9999:9999"
+    #  - "9999"
-  
   ate-api:
     build: ./ATEapi
     restart: always
+ #   ports:
+    #  - "5000:5000"
     volumes:
       - ../classla/classla_resources:/home/appuser/classla_resources
-  
   canonizer:
-    container_name: canonizer_service
+    container_name: canonizer_service_t
     build:
-      context: ./canonical_forms/web
+      context: ./canonical_forms/
       dockerfile: Dockerfile.prod
     command: gunicorn --bind 0.0.0.0:5000 app:app
     volumes:
@@ -41,18 +42,15 @@ services:
       - FLASK_APP=app.py
       - FLASK_ENV=production
       - APP_FOLDER=/home/app/web
-
+  definitions:
-  mariadb:
+    container_name: definition_extraction
-    image: mariadb/columnstore
+    build:
-    restart: always
+      context: ./definition_extraction/
-    container_name: mariadbcs
+      dockerfile: Dockerfile.prod
-    ports:
+    command: gunicorn --bind 0.0.0.0:5000 app:app
-      - "3306:3306"
     volumes:
-      - mdbdata:/var/lib/mysql
+      - ../classla/classla_resources:/home/app/classla_resources
-      - mdbcs-data:/var/lib/columnstore
+    environment:
-      - mdbcs-etc:/etc/columnstore
+      - FLASK_APP=app.py
-volumes:
+      - FLASK_ENV=production
-    mdbdata:
+      - APP_FOLDER=/home/app/web
-    mdbcs-data:
-    mdbcs-etc:

swagger_server/controllers/extract_controller.py

@@ -8,21 +8,25 @@ from swagger_server.models.izlusci_async_body import IzlusciAsyncBody  # noqa: E
 from swagger_server.models.izlusci_sync_body import IzlusciSyncBody  # noqa: E501
 from swagger_server.requets_db.models.vrsta import JobManager
 from swagger_server.utils import cl_utils
+from swagger_server.utils import txt_utils
 from swagger_server.util import get_random_filename, create_random_file_in_tmp_folder
 import requests
 from werkzeug.utils import secure_filename
 
 ATEapi_endpoint = "http://ate-api:5000/predict"
 
+
 # endpoint below to be used only for development purposes (don't need to run docker)
 # ATEapi_endpoint = "http://localhost:5000/predict"
 
 
-def do_izlusci(conllus, prepovedane_besede):
+def do_izlusci(conllus, prepovedane_besede,definicije=False):
     tmp_file_path = ""
+    print(definicije);
     try:
         big_conllu = cl_utils.multipla_conllus_to_one_from_conllus_arr(conllus)
         tmp_file_path = create_random_file_in_tmp_folder(big_conllu, ".conllu")
+
         fp = open(tmp_file_path, 'rb')
         try:
             files = [
@@ -30,26 +34,38 @@ def do_izlusci(conllus, prepovedane_besede):
             ]
             res = requests.post(ATEapi_endpoint, files=files)
             data = json.loads(res.text)
+        
+           
+            ret = {'terminoloski_kandidati': [
+                {
+                    'POSoznake': tk['term_example_pos'],
+                    'kandidat': tk['lemma'],  # more to bit lemma al terms?
+                    'kanonicnaoblika': tk['canonical'],
+                    'ranking': tk['ranking'],
+                    'podporneutezi': [
+                        0.0,  # ????????
+                        0.0  # ??????
+                    ],
+                    'pogostostpojavljanja': [tk['frequency'], 0]  # ???????
+                }
+                for tk in data if tk['canonical'] not in prepovedane_besede
+            ]}
+
+          
+
+
+            if definicije:
+                print("grem po definicije!!!")
+                ret = txt_utils.extract_definition_sentences(tmp_file_path,ret);    
+               
+        except Exception as e: print(e)
         finally:
             fp.close()
             os.remove(tmp_file_path)
 
-        ret = {'terminoloski_kandidati': [
-            {
-                'POSoznake': tk['msd'],
-                'kandidat': tk['terms'],  # more to bit lemma al terms?
-                'kanonicnaoblika': tk['canonical'],
-                'ranking': tk['ranking'],
-                'podporneutezi': [
-                    0.0,  # ????????
-                    0.0  # ??????
-                ],
-                'pogostostpojavljanja': [0, 0]  # ???????
-            }
-            for tk in data if tk['terms'] not in prepovedane_besede
-        ]}
         return ret, 200
     except Exception as e:
+        print(e)
         return str(e), 500
 
 
@@ -84,5 +100,5 @@ def get_candidates_sync(body):  # noqa: E501
     """
     if connexion.request.is_json:
         body = IzlusciSyncBody.from_dict(connexion.request.get_json())  # noqa: E501
-
+    print(body)
-    return do_izlusci(body.conllus, body.prepovedane_besede)
+    return do_izlusci(body.conllus, body.prepovedane_besede,body.definicije)

swagger_server/controllers/jobs_controller.py

@@ -6,28 +6,54 @@ import traceback
 import peewee
 import asyncio
 import concurrent.futures as cf
-
+from flask import Response
 from swagger_server.controllers.extract_controller import do_izlusci
 from swagger_server.models.job_response import JobResponse  # noqa: E501
 from swagger_server.requets_db.models.vrsta import (Job)
 from threading import Thread
-from swagger_server.utils import cl_utils
+from swagger_server.utils import cl_utils, db_utils
 from swagger_server.utils import txt_utils
 from werkzeug.datastructures import FileStorage
 import threading
 import time
 
-CLASSLA_CONCURANCE_LIMIT = 3
+CLASSLA_SMALL_SIZE_LIMIT = 500 * 1e3  # 500 KB, aka.: 500 * 10^3
-DOC2TEXT_CONCURANCE_LIMIT = 3
-ATEAPI_CONCURANCE_LIMIT = 2
 
-classla_sem = threading.Semaphore(CLASSLA_CONCURANCE_LIMIT)
+DOC2TEXT_SMALL_SIZE_LIMIT = 10 * 1e6  # 10 MB
-doc2text_sem = threading.Semaphore(DOC2TEXT_CONCURANCE_LIMIT)
+
-ateapi_sem = threading.Semaphore(ATEAPI_CONCURANCE_LIMIT)
+ATEAPI_SMALL_SIZE_LIMIT = 2 * 1e6  # 2 MB
+
+########################################
+CLASSLA_CONCURANCE_LIMIT_BIG = 2
+CLASSLA_CONCURANCE_LIMIT_SMALL = 2
+
+DOC2TEXT_CONCURANCE_LIMIT_BIG = 2
+DOC2TEXT_CONCURANCE_LIMIT_SMALL = 2
+
+ATEAPI_CONCURANCE_LIMIT_BIG = 2
+ATEAPI_CONCURANCE_LIMIT_SMALL = 2
+
+IZLUSCI_PO_ISKANJU_CONCURANCE_LIMIT = 4
+########################################
+classla_sem_big = threading.Semaphore(CLASSLA_CONCURANCE_LIMIT_BIG)
+classla_sem_small = threading.Semaphore(CLASSLA_CONCURANCE_LIMIT_SMALL)
+
+doc2text_sem_big = threading.Semaphore(DOC2TEXT_CONCURANCE_LIMIT_BIG)
+doc2text_sem_small = threading.Semaphore(DOC2TEXT_CONCURANCE_LIMIT_SMALL)
+
+ateapi_sem_big = threading.Semaphore(ATEAPI_CONCURANCE_LIMIT_BIG)
+ateapi_sem_small = threading.Semaphore(ATEAPI_CONCURANCE_LIMIT_SMALL)
+
+izluscipoiskanju_sem = threading.Semaphore(IZLUSCI_PO_ISKANJU_CONCURANCE_LIMIT)
+
+running_threads = {}  # <--- dict currently not used, was trying to figure out how to cancel workers mid execution,
+
+
+# no luck with that yet
 
 
 def delete_job(job_id):  # noqa: E501
-    """Izbriše job
+    """Izbriše job
 
      # noqa: E501
 
@@ -36,7 +62,14 @@ def delete_job(job_id):  # noqa: E501
 
     :rtype: str
     """
-    return 'Endpoint currently disabled'
+    try:
+        job = Job.get_by_id(job_id)
+        if job.started_on is not None and job.finished_on is None:
+            return Response("Cancelling ongoing jobs currently not implemented.", 400)
+        job.delete_instance()
+        return f"Job with the ID {job_id} was removed."
+    except peewee.DoesNotExist:
+        return Response("Job with this ID does not exist", 404)
 
 
 def get_job_status(job_id):  # noqa: E501
@@ -56,8 +89,19 @@ def get_job_status(job_id):  # noqa: E501
         if job.started_on is not None and job.finished_on is None:
             return JobResponse(job_status="currently processing", created_on=job.created_on,
                                started_on=job.started_on), 200
-        if job.started_on is not None and job.finished_on is not None:
+        if job.started_on is not None and job.finished_on is not None and not job.job_output.startswith("ERROR -"):
-            return JobResponse(job_status="finished processing", created_on=job.created_on, started_on=job.started_on,
+            res = job.job_output
+            if job.job_type in [4, 5]:
+                try:
+                    res = json.loads(job.job_output)
+                except:
+                    pass
+            return JobResponse(job_status="finished processing (OK)", created_on=job.created_on,
+                               started_on=job.started_on,
+                               finished_on=job.finished_on, job_result=res), 200
+        if job.started_on is not None and job.finished_on is not None and job.job_output.startswith("ERROR -"):
+            return JobResponse(job_status="finished processing (ERROR)", created_on=job.created_on,
+                               started_on=job.started_on,
                                finished_on=job.finished_on, job_result=job.job_output), 200
     except peewee.DoesNotExist:
         return "Job with this ID does not exist", 404
@@ -75,22 +119,65 @@ def clear_up_unfinished_jobs():
 
 
 async def try_do_jobs():
-    with cf.ThreadPoolExecutor(max_workers=3) as ex:
+    with cf.ThreadPoolExecutor(max_workers=4) as ex:
         ex.submit(try_do_jobs_classla)
         ex.submit(try_do_jobs_doc2text)
         ex.submit(try_do_jobs_ateapi)
+        ex.submit(try_do_jobs_izluscipoiskanju)
 
 
-### Job looping
+# to pe je pod ex.submit
+# sub = ex.submit(execute_ateapi_job, job)
+# running_threads[job.id] = sub
+# time.sleep(1)
+# running_threads[job.id]
+# preveri ce je done: ```running_threads[job.id].done()``` (vrne true false)
+
+# was_canceled = running_threads[job.id].cancel()
+# Todo: Mogoce kaksna druga opcija? Ampak verjetno ne, ne vidim (še?) kak prekicat ONGOING job
+# To zgoraj preklice samo job, ki se se ni zacel, kar pa ni za ta use case uporabno.
+
+
+### Picking jobs for looping
+def try_do_jobs_izluscipoiskanju():
+    while True:
+        try:
+            if izluscipoiskanju_sem._value > 0:
+                unfinished_jobs = Job.select() \
+                    .where(Job.finished_on.is_null(), Job.started_on.is_null(), Job.job_type == 5) \
+                    .limit(izluscipoiskanju_sem._value)
+                with cf.ThreadPoolExecutor(max_workers=IZLUSCI_PO_ISKANJU_CONCURANCE_LIMIT) as ex:
+                    [ex.submit(execute_izluscipoiskanju_job, job, izluscipoiskanju_sem) for job in unfinished_jobs]
+        except Exception as e:
+            print(f"Exception in try_do_jobs_izluscipoiskanju")
+            traceback.print_exc()
+        finally:
+            time.sleep(3)
+
+
+### Picking jobs for looping
 def try_do_jobs_ateapi():
     while True:
         try:
-            if ateapi_sem._value > 0:
+            unfinished_jobs_big = []
-                unfinished_jobs = Job.select() \
+            unfinished_jobs_small = []
-                    .where(Job.finished_on.is_null(), Job.started_on.is_null(), Job.job_type == 4) \
+            if ateapi_sem_big._value > 0:
-                    .limit(ateapi_sem._value)
+                unfinished_jobs_big.extend(Job.select().where(Job.finished_on.is_null(), Job.started_on.is_null(),
-                with cf.ThreadPoolExecutor(max_workers=ATEAPI_CONCURANCE_LIMIT) as ex:
+                                                              Job.job_type == 4,
-                    [ex.submit(execute_ateapi_job, job) for job in unfinished_jobs]
+                                                              Job.input_size > ATEAPI_SMALL_SIZE_LIMIT) \
+                                           .limit(ateapi_sem_big._value))
+            if ateapi_sem_small._value > 0:
+                unfinished_jobs_small.extend(Job.select().where(Job.finished_on.is_null(), Job.started_on.is_null(),
+                                                                Job.job_type == 4,
+                                                                Job.input_size <= ATEAPI_SMALL_SIZE_LIMIT) \
+                                             .limit(ateapi_sem_small._value))
+
+            if len(unfinished_jobs_big) > 0:
+                with cf.ThreadPoolExecutor(max_workers=ATEAPI_CONCURANCE_LIMIT_BIG) as ex:
+                    [ex.submit(execute_ateapi_job, job, ateapi_sem_big) for job in unfinished_jobs_big]
+            if len(unfinished_jobs_small) > 0:
+                with cf.ThreadPoolExecutor(max_workers=ATEAPI_CONCURANCE_LIMIT_SMALL) as ex:
+                    [ex.submit(execute_ateapi_job, job, ateapi_sem_small) for job in unfinished_jobs_small]
         except Exception as e:
             print(f"Exception in try_do_jobs_ateapi")
             traceback.print_exc()
@@ -98,27 +185,51 @@ def try_do_jobs_ateapi():
             time.sleep(3)
 
 
-### Job looping
+### Picking jobs for looping
 def try_do_jobs_classla():
     time.sleep(15)  # wait for tokenizers to load for classla ...
     while True:
         try:
-            if cl_utils.nlp_loaded:
+            if not cl_utils.nlp_loaded:
-                if classla_sem._value > 0:
+                raise Exception("NLP utils not loaded yet.")
-                    unfinished_jobs_txt = Job.select() \
-                        .where(Job.finished_on.is_null(), Job.job_type == 2,
-                               Job.input_file.is_null(False)) \
-                        .limit(classla_sem._value)
 
-                    unfinished_jobs_no_txt = Job.select() \
+            unfinished_jobs_big = []
-                        .where(Job.finished_on.is_null(), Job.started_on.is_null(), Job.job_type == 2,
+            unfinished_jobs_small = []
-                               Job.input_file.is_null()) \
-                        .limit(classla_sem._value)
 
-                    unfinished_jobs = [j for j in unfinished_jobs_txt] + [j for j in unfinished_jobs_no_txt]
+            if classla_sem_big._value > 0:
-                    unfinished_jobs = unfinished_jobs[:classla_sem._value]
+                unfinished_jobs_txt = Job.select() \
-                    with cf.ThreadPoolExecutor(max_workers=CLASSLA_CONCURANCE_LIMIT) as ex:
+                    .where(Job.finished_on.is_null(), Job.job_type == 2,
-                        [ex.submit(execute_classla_job, job) for job in unfinished_jobs]
+                           Job.input_file.is_null(False), Job.input_size > CLASSLA_SMALL_SIZE_LIMIT) \
+                    .limit(classla_sem_big._value)
+
+                unfinished_jobs_no_txt = Job.select() \
+                    .where(Job.finished_on.is_null(), Job.started_on.is_null(), Job.job_type == 2,
+                           Job.input_file.is_null(), Job.input_size > CLASSLA_SMALL_SIZE_LIMIT) \
+                    .limit(classla_sem_big._value)
+
+                unfinished_jobs_big = [j for j in unfinished_jobs_txt] + [j for j in unfinished_jobs_no_txt]
+                unfinished_jobs_big = unfinished_jobs_big[:classla_sem_big._value]
+
+            if classla_sem_small._value > 0:
+                unfinished_jobs_txt = Job.select() \
+                    .where(Job.finished_on.is_null(), Job.job_type == 2,
+                           Job.input_file.is_null(False), Job.input_size <= CLASSLA_SMALL_SIZE_LIMIT) \
+                    .limit(classla_sem_small._value)
+
+                unfinished_jobs_no_txt = Job.select() \
+                    .where(Job.finished_on.is_null(), Job.started_on.is_null(), Job.job_type == 2,
+                           Job.input_file.is_null(), Job.input_size <= CLASSLA_SMALL_SIZE_LIMIT) \
+                    .limit(classla_sem_small._value)
+
+                unfinished_jobs_small = [j for j in unfinished_jobs_txt] + [j for j in unfinished_jobs_no_txt]
+                unfinished_jobs_small = unfinished_jobs_small[:classla_sem_small._value]
+
+            if len(unfinished_jobs_big) > 0:
+                with cf.ThreadPoolExecutor(max_workers=CLASSLA_CONCURANCE_LIMIT_BIG) as ex:
+                    [ex.submit(execute_classla_job, job, doc2text_sem_big) for job in unfinished_jobs_big]
+            if len(unfinished_jobs_small) > 0:
+                with cf.ThreadPoolExecutor(max_workers=CLASSLA_CONCURANCE_LIMIT_SMALL) as ex:
+                    [ex.submit(execute_classla_job, job, doc2text_sem_small) for job in unfinished_jobs_small]
 
         except Exception as e:
             print(f"Exception in try_do_jobs_classla")
@@ -127,16 +238,29 @@ def try_do_jobs_classla():
             time.sleep(3)
 
 
-### Job looping
+### Picking jobs for looping
 def try_do_jobs_doc2text():
     while True:
         try:
-            if doc2text_sem._value > 0:
+            unfinished_jobs_big = []
-                unfinished_jobs = Job.select() \
+            unfinished_jobs_small = []
-                    .where(Job.finished_on.is_null(), Job.started_on.is_null(), Job.job_type << [1, 12, 3, 32]) \
+            if doc2text_sem_big._value > 0:
-                    .limit(doc2text_sem._value)
+                unfinished_jobs_big.extend(Job.select().where(Job.finished_on.is_null(), Job.started_on.is_null(),
-                with cf.ThreadPoolExecutor(max_workers=DOC2TEXT_CONCURANCE_LIMIT) as ex:
+                                                              Job.job_type << [1, 12, 3, 32],
-                    [ex.submit(execute_doc2text_job, job) for job in unfinished_jobs]
+                                                              Job.input_size > DOC2TEXT_SMALL_SIZE_LIMIT) \
+                                           .limit(doc2text_sem_big._value))
+            if doc2text_sem_small._value > 0:
+                unfinished_jobs_small.extend(Job.select().where(Job.finished_on.is_null(), Job.started_on.is_null(),
+                                                                Job.job_type << [1, 12, 3, 32],
+                                                                Job.input_size <= DOC2TEXT_SMALL_SIZE_LIMIT) \
+                                             .limit(doc2text_sem_small._value))
+
+            if len(unfinished_jobs_big) > 0:
+                with cf.ThreadPoolExecutor(max_workers=DOC2TEXT_CONCURANCE_LIMIT_BIG) as ex:
+                    [ex.submit(execute_doc2text_job, job, doc2text_sem_big) for job in unfinished_jobs_big]
+            if len(unfinished_jobs_small) > 0:
+                with cf.ThreadPoolExecutor(max_workers=DOC2TEXT_CONCURANCE_LIMIT_SMALL) as ex:
+                    [ex.submit(execute_doc2text_job, job, doc2text_sem_small) for job in unfinished_jobs_small]
         except Exception as e:
             print(f"Exception in try_do_jobs_doc2text")
             traceback.print_exc()
@@ -144,13 +268,14 @@ def try_do_jobs_doc2text():
             time.sleep(3)
 
 
-async def prep_jobs(tasks):
+# async def prep_jobs(tasks):
-    await asyncio.gather(*tasks)
+#     await asyncio.gather(*tasks)
 
 
-def execute_doc2text_job(job: Job):
+####### JOB EXECUTION LOGIC
+def execute_doc2text_job(job: Job, sem: threading.Semaphore):
     try:
-        doc2text_sem.acquire()
+        sem.acquire()
         del_file = False
         job.started_on = datetime.datetime.utcnow()
         job.save()
@@ -158,7 +283,7 @@ def execute_doc2text_job(job: Job):
         tmp_file_path = job.input_file
         if not os.path.exists(tmp_file_path):
             job.finished_on = datetime.datetime.utcnow()
-            job.job_output = "ERROR - Temporary file went missing, couldn't properly finish job"
+            job.job_output = "ERROR - Temporary file went missing, couldn't properly finish job. Please try executing the job again."
             job.save()
             return
 
@@ -190,34 +315,85 @@ def execute_doc2text_job(job: Job):
         job.started_on = None
         job.save()
     finally:
-        doc2text_sem.release()
+        sem.release()
 
 
-def execute_classla_job(job: Job):
+####### JOB EXECUTION LOGIC
+def execute_classla_job(job: Job, sem: threading.Semaphore):
     try:
-        classla_sem.acquire()
+        sem.acquire()
         job.started_on = datetime.datetime.utcnow()
         job.save()
-        conllu, _ = cl_utils.raw_text_to_conllu(job.job_input)
+        conllu, status = cl_utils.raw_text_to_conllu(job.job_input)
+        if status != 200:
+            conllu = f'ERROR - {conllu}'
         job.job_output = conllu
         job.finished_on = datetime.datetime.utcnow()
         job.save()
+    except:
+        job.job_output = "ERROR - Something unexpected went wrong. Logs have been saved. Please contact the api admin if the problem persists."
+        job.finished_on = datetime.datetime.utcnow()
+        job.save()
+        print(f"Unexpected error at job {job.id}")
     finally:
-        classla_sem.release()
+        sem.release()
 
 
-def execute_ateapi_job(job: Job):
+####### JOB EXECUTION LOGIC
+def execute_ateapi_job(job: Job, sem: threading.Semaphore):
     try:
-        ateapi_sem.acquire()
+        sem.acquire()
         job.started_on = datetime.datetime.utcnow()
         job.save()
         info = json.loads(job.job_input)
-        ret_json, _ = do_izlusci(info['conllus'], info['prepovedane_besede'])
+        _res = do_izlusci(info['conllus'], info['prepovedane_besede'],info['definicije'])
-        job.job_output = ret_json
+        if type(_res) is tuple:
+            ret_json = _res[0]
+        else:
+            try:
+                if type(_res.response) is dict:
+                    ret_json = str(_res.response)
+                else:
+                    try:
+                        ret_json = _res.response[0].decode('utf-8')
+                    except:
+                        ret_json = "ERROR - Unknown exception."
+
+                if _res.status_code != 200:
+                    ret_json = f'ERROR - {ret_json}'
+            except:
+                ret_json = "ERROR - Unknown exception."
+        job.job_output = json.dumps(ret_json, ensure_ascii=False)
         job.finished_on = datetime.datetime.utcnow()
         job.save()
+    except:
+        job.job_output = "ERROR - Something unexpected went wrong. Logs have been saved. Please contact the api admin if the problem persists."
+        job.finished_on = datetime.datetime.utcnow()
+        job.save()
+        print(f"Unexpected error at job {job.id}")
     finally:
-        ateapi_sem.release()
+        sem.release()
+
+
+####### JOB EXECUTION LOGIC
+def execute_izluscipoiskanju_job(job: Job, sem: threading.Semaphore):
+    try:
+        sem.acquire()
+        job.started_on = datetime.datetime.utcnow()
+        job.save()
+        info = json.loads(job.job_input)
+        terKand = db_utils.vrni_oss_terminoloske_kandidate(info['leta'], info['vrste'], info['kljucne_besede'],
+                                                           info['prepovedane_besede'], info['udk'],info['definicije'])
+        job.job_output = json.dumps(terKand, ensure_ascii=False)
+        job.finished_on = datetime.datetime.utcnow()
+        job.save()
+    except:
+        job.job_output = "ERROR - Something unexpected went wrong. Logs have been saved. Please contact the api admin if the problem persists."
+        job.finished_on = datetime.datetime.utcnow()
+        job.save()
+        print(f"Unexpected error at job {job.id}")
+    finally:
+        sem.release()
 
 
 clear_up_unfinished_jobs()

swagger_server/controllers/oss_controller.py

@@ -1,55 +1,88 @@
+import json
+
+import connexion
+
+from swagger_server.requets_db.models.vrsta import JobManager
 from swagger_server.utils import db_utils
 from swagger_server import util
 from flask import send_file
 
 
-def get_conllus(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501
+def get_conllus(leta=None, vrste=None, kljucne_besede=None, udk=None):  # noqa: E501
-    """Vrne seznam CoNNL-U-jev glede na iskalne pogoje
+    """"Vrne seznam CoNNL-U-jev glede na iskalne pogoje
 
      # noqa: E501
 
-    :param leta: 
+    :param leta:
     :type leta: List[int]
-    :param vrste: 
+    :param vrste:
-    :type vrste: List[str]
+    :type vrste: List[int]
-    :param kljucnebesede: 
+    :param kljucne_besede:
-    :type kljucnebesede: List[str]
+    :type kljucne_besede: List[str]
-    :param cerifpodrocja: 
+    :param udk:
-    :type cerifpodrocja: List[int]
+    :type udk: List[str]
 
     :rtype: List[str]
     """
-    if not kljucnebesede:
+    if not kljucne_besede:
         return "Manjkajo kljucne besede", 400
-    #zaenkrat ne potrebujemo te storitve
+    files = db_utils.get_files_by_udc(kljucne_besede)
-    files = db_utils.get_files_by_udc(kljucnebesede)
     if not files:
         return 'Nobena datoteka ne ustreza iskalnemu pogoju', 404
     return ' '.join(files), 200
 
 
-def get_extracted_words(leta=None, vrste=None, kljucnebesede=None, udk=None):  # noqa: E501
+def get_extracted_words(leta=None, vrste=None, kljucne_besede=None, prepovedane_besede=None, udk=None,definicije=False):  # noqa: E501
-    """Vrne terminloške kandidate glede na 
+    """Vrne terminloške kandidate glede na ... (sync)
 
      # noqa: E501
 
-    :param leta: 
+    :param leta:
     :type leta: List[int]
-    :param vrste: 
+    :param vrste:
-    :type vrste: List[str]
+    :type vrste: List[int]
-    :param kljucnebesede: 
+    :param kljucne_besede:
-    :type kljucnebesede: List[str]
+    :type kljucne_besede: List[str]
-    :param cerifpodrocja: 
+    :param prepovedane_besede:
-    :type cerifpodrocja: List[int]
+    :type prepovedane_besede: List[str]
+    :param udk:
+    :type udk: List[str]
 
     :rtype: List[TerminoloskiKandidat]
     """
-    files = db_utils.vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, udk)
+    terKand = db_utils.vrni_oss_terminoloske_kandidate(leta, vrste, kljucne_besede, prepovedane_besede, udk,definicije)
-    return files, 200
+    return terKand, 200
 
 
+def get_extracted_words_async(leta=None, vrste=None, kljucne_besede=None, prepovedane_besede=None,
+                              udk=None,definicije=False):  # noqa: E501
+    """Vrne terminloške kandidate glede na ... (async)
 
-def get_files(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501
+     # noqa: E501
+
+    :param leta:
+    :type leta: List[int]
+    :param vrste:
+    :type vrste: List[int]
+    :param kljucne_besede:
+    :type kljucne_besede: List[str]
+    :param prepovedane_besede:
+    :type prepovedane_besede: List[str]
+    :param udk:
+    :type udk: List[str]
+
+    :rtype: str
+    """
+    job, is_old_job = JobManager.create_job(5, json.dumps(
+        {'leta': leta, 'vrste': vrste, 'kljucne_besede': kljucne_besede, 'prepovedane_besede': prepovedane_besede,
+         'udk': udk,'definicije':definicije}))
+    if job is None:
+        return "Something went wrong", 500
+    ret = {'check_job_url': f'{connexion.request.url_root}/job/{job.id}'}
+    return ret, 200
+
+
+def get_files(leta=None, vrste=None, kljucne_besede=None, udk=None):  # noqa: E501
     """Vrne seznam binarnih zapisov v originalnem formatu glede na iskalne pogoje
 
      # noqa: E501
@@ -57,66 +90,64 @@ def get_files(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501
     :param leta: 
     :type leta: List[int]
     :param vrste: 
-    :type vrste: List[str]
+    :type vrste: List[int]
-    :param kljucnebesede: 
+    :param kljucne_besede:
-    :type kljucnebesede: List[str]
+    :type kljucne_besede: List[str]
-    :param cerifpodrocja: 
+    :param udk:
-    :type cerifpodrocja: List[int]
+    :type udk: List[str]
 
-    :rtype: List[List[bytearray]]
+    :rtype: str
     """
-    if not kljucnebesede:
+    if not kljucne_besede:
         return "Manjkajo kljucne besede", 400
-    #zaenkrat ne potrebujemo te storitve
+    files = db_utils.get_files_by_udc(kljucne_besede)
-    files = db_utils.get_files_by_udc(kljucnebesede)
     if not files:
         return 'Nobena datoteka ne ustreza iskalnemu pogoju', 404
     return ' '.join(files), 200
 
 
-def get_number_texts(leta=None, vrste=None, kljucnebesede=None, udk=None):  # noqa: E501
+def get_number_texts(leta=None, vrste=None, kljucne_besede=None, udk=None):  # noqa: E501
-    """Vrne število besedil glede na iskalne pogoje
+    """Vrne število besedil glede na iskalne pogoje
 
      # noqa: E501
 
     :param leta: 
     :type leta: List[int]
     :param vrste: 
-    :type vrste: List[str]
+    :type vrste: List[int]
-    :param kljucnebesede: 
+    :param kljucne_besede:
-    :type kljucnebesede: List[str]
+    :type kljucne_besede: List[str]
-    :param cerifpodrocja: 
+    :param udk:
-    :type udc: List[int]
+    :type udk: List[str]
 
     :rtype: int
     """
-    #if not kljucnebesede:
+    # if not kljucnebesede:
     #    return "Manjkajo kljucne besede", 400
 
-    files = db_utils.vrni_oss_dokumente(leta, vrste, kljucnebesede, udk)
+    files = db_utils.vrni_oss_dokumente(leta, vrste, kljucne_besede, udk)
     return len(files), 200
 
 
-def get_texts(leta, vrste, kljucnebesede, cerifpodrocja):  # noqa: E501
+def get_texts(leta=None, vrste=None, kljucne_besede=None, udk=None):  # noqa: E501
     """Vrne seznam besedil glede na iskalne pogoje
 
      # noqa: E501
 
-    :param leta: 
+    :param leta:
     :type leta: List[int]
-    :param vrste: 
+    :param vrste:
-    :type vrste: List[str]
+    :type vrste: List[int]
-    :param kljucnebesede: 
+    :param kljucne_besede:
-    :type kljucnebesede: List[str]
+    :type kljucne_besede: List[str]
-    :param cerifpodrocja: 
+    :param udk:
-    :type cerifpodrocja: List[int]
+    :type udk: List[str]
 
     :rtype: List[str]
     """
-    if not kljucnebesede:
+    if not kljucne_besede:
         return "Manjkajo kljucne besede", 400
-    #zaenkrat ne potrebujemo te storitve
+    files = db_utils.get_files_by_udc(kljucne_besede)
-    files = db_utils.get_files_by_udc(kljucnebesede)
     if not files:
         return 'Nobena datoteka ne ustreza iskalnemu pogoju', 404
     return ' '.join(files), 200
@@ -166,7 +197,7 @@ def oss_besedilo_po_id_get(file_id):  # noqa: E501
     """
     try:
         f = util.get_original_file_path_by_id(file_id)
-        print(f) # for debugging purposes on the server, delete this later
+        print(f)  # for debugging purposes on the server, delete this later
         return send_file(util.get_original_file_path_by_id(file_id), download_name=f'{file_id}.xml')
     except FileNotFoundError as e:
         return "The file with this ID doesn't exist.", 404

swagger_server/models/izlusci_async_body.py

@@ -14,7 +14,7 @@ class IzlusciAsyncBody(Model):
 
     Do not edit the class manually.
     """
-    def __init__(self, conllus: List[str]=None, prepovedane_besede: List[str]=None):  # noqa: E501
+    def __init__(self, conllus: List[str]=None, prepovedane_besede: List[str]=None, definicje: bool=False):  # noqa: E501
         """IzlusciAsyncBody - a model defined in Swagger
 
         :param conllus: The conllus of this IzlusciAsyncBody.  # noqa: E501
@@ -24,15 +24,18 @@ class IzlusciAsyncBody(Model):
         """
         self.swagger_types = {
             'conllus': List[str],
-            'prepovedane_besede': List[str]
+            'prepovedane_besede': List[str],
+            'definicije':bool
         }
 
         self.attribute_map = {
             'conllus': 'conllus',
-            'prepovedane_besede': 'prepovedaneBesede'
+            'prepovedane_besede': 'prepovedaneBesede',
+            'definicije':'definicije'
         }
         self._conllus = conllus
         self._prepovedane_besede = prepovedane_besede
+        self._definicije=definicje
 
     @classmethod
     def from_dict(cls, dikt) -> 'IzlusciAsyncBody':
@@ -45,6 +48,30 @@ class IzlusciAsyncBody(Model):
         """
         return util.deserialize_model(dikt, cls)
 
+
+
+    @property
+    def definicije(self) -> bool:
+        """Gets the conllus of this IzlusciSyncBody.
+
+
+        :return: The conllus of this IzlusciSyncBody.
+        :rtype: List[str]
+        """
+        return self._definicije
+
+    @definicije.setter
+    def definicije(self, definicije: bool):
+        """Sets the conllus of this IzlusciSyncBody.
+
+
+        :param conllus: The conllus of this IzlusciSyncBody.
+        :type conllus: List[str]
+        """
+
+        self._definicije = definicije
+
+
     @property
     def conllus(self) -> List[str]:
         """Gets the conllus of this IzlusciAsyncBody.

swagger_server/models/izlusci_sync_body.py

@@ -14,7 +14,7 @@ class IzlusciSyncBody(Model):
 
     Do not edit the class manually.
     """
-    def __init__(self, conllus: List[str]=None, prepovedane_besede: List[str]=None):  # noqa: E501
+    def __init__(self, conllus: List[str]=None, prepovedane_besede: List[str]=None, definicje: bool=False):  # noqa: E501
         """IzlusciSyncBody - a model defined in Swagger
 
         :param conllus: The conllus of this IzlusciSyncBody.  # noqa: E501
@@ -24,15 +24,18 @@ class IzlusciSyncBody(Model):
         """
         self.swagger_types = {
             'conllus': List[str],
-            'prepovedane_besede': List[str]
+            'prepovedane_besede': List[str],
+            'definicije':bool
         }
 
         self.attribute_map = {
             'conllus': 'conllus',
-            'prepovedane_besede': 'prepovedaneBesede'
+            'prepovedane_besede': 'prepovedaneBesede',
+            'definicije':'definicije'
         }
         self._conllus = conllus
         self._prepovedane_besede = prepovedane_besede
+        self._definicije = definicje
 
     @classmethod
     def from_dict(cls, dikt) -> 'IzlusciSyncBody':
@@ -66,6 +69,27 @@ class IzlusciSyncBody(Model):
 
         self._conllus = conllus
 
+    @property
+    def definicije(self) -> bool:
+        """Gets the conllus of this IzlusciSyncBody.
+
+
+        :return: The conllus of this IzlusciSyncBody.
+        :rtype: List[str]
+        """
+        return self._definicije
+
+    @definicije.setter
+    def definicije(self, definicije: bool):
+        """Sets the conllus of this IzlusciSyncBody.
+
+
+        :param conllus: The conllus of this IzlusciSyncBody.
+        :type conllus: List[str]
+        """
+
+        self._definicije = definicije
+
     @property
     def prepovedane_besede(self) -> List[str]:
         """Gets the prepovedane_besede of this IzlusciSyncBody.

swagger_server/models/terminoloski_kandidat.py

@@ -14,7 +14,7 @@ class TerminoloskiKandidat(Model):
 
     Do not edit the class manually.
     """
-    def __init__(self, kandidat: str=None, kanonicnaoblika: str=None, po_soznake: str=None, nosilnautez: float=None, podporneutezi: List[float]=None, pogostostpojavljanja: List[int]=None):  # noqa: E501
+    def __init__(self, kandidat: str=None,  definicija: str=None, kanonicnaoblika: str=None, po_soznake: str=None, nosilnautez: float=None, podporneutezi: List[float]=None, pogostostpojavljanja: List[int]=None):  # noqa: E501
         """TerminoloskiKandidat - a model defined in Swagger
 
         :param kandidat: The kandidat of this TerminoloskiKandidat.  # noqa: E501
@@ -32,6 +32,7 @@ class TerminoloskiKandidat(Model):
         """
         self.swagger_types = {
             'kandidat': str,
+            'definicja': str,
             'kanonicnaoblika': str,
             'po_soznake': str,
             'nosilnautez': float,
@@ -41,6 +42,7 @@ class TerminoloskiKandidat(Model):
 
         self.attribute_map = {
             'kandidat': 'kandidat',
+            'definicija': 'definicija',
             'kanonicnaoblika': 'kanonicnaoblika',
             'po_soznake': 'POSoznake',
             'nosilnautez': 'nosilnautez',
@@ -48,6 +50,7 @@ class TerminoloskiKandidat(Model):
             'pogostostpojavljanja': 'pogostostpojavljanja'
         }
         self._kandidat = kandidat
+        self._definicija = definicija
         self._kanonicnaoblika = kanonicnaoblika
         self._po_soznake = po_soznake
         self._nosilnautez = nosilnautez
@@ -86,6 +89,27 @@ class TerminoloskiKandidat(Model):
 
         self._kandidat = kandidat
 
+    @property
+    def definicija(self) -> str:
+        """Gets the definicija of this TerminoloskiKandidat.
+
+
+        :return: The definicija of this TerminoloskiKandidat.
+        :rtype: str
+        """
+        return self._definicija
+
+    @definicija.setter
+    def definicija(self, definicija: str):
+        """Sets the kandidat of this TerminoloskiKandidat.
+
+
+        :param kandidat: The kandidat of this TerminoloskiKandidat.
+        :type kandidat: str
+        """
+
+        self._definicija = definicija
+
     @property
     def kanonicnaoblika(self) -> str:
         """Gets the kanonicnaoblika of this TerminoloskiKandidat.

swagger_server/swagger/swagger.yaml

@@ -343,7 +343,7 @@ paths:
     get:
       tags:
       - oss
-      summary: 'Vrne terminloške kandidate glede na '
+      summary: 'Vrne terminloške kandidate glede na iskalne pogoje'
       operationId: get_extracted_words
       parameters:
       - name: leta
@@ -384,6 +384,12 @@ paths:
           type: array
           items:
             type: string
+      - name: definicije
+        in: query
+        required: false
+        style: form
+        schema:
+          type: boolean
       responses:
         "200":
           description: OK
@@ -651,6 +657,8 @@ components:
           type: string
         kanonicnaoblika:
           type: string
+        definition:
+          type: string
         POSoznake:
           type: string
         nosilnautez:
@@ -753,6 +761,8 @@ components:
           type: array
           items:
             type: string
+        definicije:
+          type: boolean
     izlusciAsync_body:
       type: object
       properties:
@@ -764,6 +774,8 @@ components:
           type: array
           items:
             type: string
+        definicije:
+          type: boolean
     datotekaVBesediloSync_body:
       required:
       - file

swagger_server/utils/db_utils.py

@@ -3,12 +3,13 @@ import os
 import sys
 import requests
 import json
+import time
 
 
 database_info = {
     'database': os.getenv("MDB_DATABASE", "oss"),
     'host': os.getenv("MDB_HOST", "localhost"),
-    'port': int(os.getenv("PORT", 3306)) ,
+    'port': int(os.getenv("MDB_PORT", 3306)) ,
     'user': os.getenv("MDB_USER", "root"),
     'password': os.getenv("MDB_PASSWORD", "root"),
 }
@@ -96,13 +97,13 @@ def vrni_oss_dokumente(leta, vrste, kljucnebesede, udk):
     return ret
 
 
-def vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, udk):
+def vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, prepovedane_besede, udk,definicije=False):
     ret = []
   
     try:
         print(database_info)
         conn = mariadb.connect(**database_info)
-        cur = conn.cursor()
+        cur = conn.cursor(dictionary=True)
         
         
         
@@ -142,7 +143,7 @@ def vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, udk):
         print(sql)
         print(params)
 
-        sqltk=f"""Select ngram,upos,avg(tfidf) as tfidf, sum(tf) as tf from (
+        sqltk=f"""Select ngram,upos,convert(avg(tfidf),FLOAT) as tfidf, convert(sum(tf),INT) as tf from (
                     SELECT tf.ngram, tf.upos,(0.5+0.5*(tf.tf/d.maxtf))*log(152000/df.df)*(-1*log(1-((dff.df)/(1+df.df)))) as tfidf, tf.tf as tf
                     FROM ngrams_upos_tf tf, documents d,
                         (
@@ -160,34 +161,55 @@ def vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, udk):
                     order by tfidf desc
                     limit 1000;"""
                     #
+        #sqltk=f"""select ngram,upos,convert(1.0,float) as tfidf,%s as tf from ngrams_upos_tf limit 10;"""
+
         print (sqltk)
-        
+        #še prepovedane besede ven
+        start_time = time.time()
         cur.execute(sqltk,params)
         terms=cur.fetchall()
+        print("Čas poizbedbe je %.2f sekund" % (time.time() - start_time))
+        print (terms);
         #ret = list(cur)
         can = {'forms':[
-            ngram 
+            ngram["ngram"]
             for ngram in terms
             ]
         }
-        res = requests.post(ATEapi_endpoint, json=can)
+        print (can);
-        data = res.json().canonical_forms
+        res = requests.post(canonapi_endpoint, json=can)
+        
+        data = res.json()
+        print (data);
+        print (data.get("canonical_forms"));
+        print (terms);
+        print(zip(data.get("canonical_forms"),terms))
+
+
+
 
         ret = {'terminoloski_kandidati': [
             {
-                'POSoznake': x.upos,
+                'POSoznake': x.get("upos"),
-                'kandidat': x.ngram,  # more to bit lemma al terms?
+                'kandidat': x.get("ngram"),  # more to bit lemma al terms?
+                'definicija': None,
                 'kanonicnaoblika': d,
-                'ranking': x.tfidf,
+                'ranking': x.get('tfidf'),
                 'podporneutezi': [
                     0.0,  # ????????
                     0.0  # ??????
                 ],
-                'pogostostpojavljanja': [tf, 0]  # ???????
+                'pogostostpojavljanja': [x.get('tf'), 0]  # ???????
             }
-            for d,x in zip(data,cur) 
+            for (d,x) in zip(data.get("canonical_forms"),terms) 
         ]}
-            
+    
+        #if definicije
+        #idi z variablo sql po id-je dokumentov, preberi conlluje iz diska
+        #naredi en vlki conllu
+        #pokliči metodo
+
+
     except mariadb.Error as e:
         print(f"Error connecting to MariaDB Platform: {e}")
     

swagger_server/utils/txt_utils.py

@@ -7,15 +7,51 @@ import xml.etree.ElementTree as ET
 from PyPDF2 import PdfReader
 from swagger_server.utils import cl_utils
 import cv2
+import json 
 import numpy as np
 import magic
 import re
 #to še mora v env 
 tika_server = "http://tika2:9999/tika"
+definicije_endpoint = "http://definitions:5000/DefExAPI/definition_sentence_extraction"
+
 
 # endpoint below to be used only for development purposes (don't need to run docker)
 # tika_server = "http://rsdo.lhrs.feri.um.si:9998/tika"
 
+#rabim conllu -> file
+# lematizirane besede ->lematized terms
+#file je touple z vsebino
+#torej ('temp_1.conllu', fp, 'application/octet-stream')
+def extract_definition_sentences(filePath="", lemmatized_terms=[]):
+
+    try:
+        fp = open(filePath, 'rb')
+        can = {'lemmatized_terms':[
+            w["kandidat"]
+            for w in lemmatized_terms["terminoloski_kandidati"]
+            ]
+        }
+        terms=json.dumps(can)
+        headers = {'accept': 'application/json'}
+        #,'Content-Type': 'multipart/form-data'}
+
+        res = requests.post(definicije_endpoint,headers=headers, files={'terms': (None, terms),'conllu_file': fp})
+        data = res.json()
+        print(data);
+        for i in lemmatized_terms["terminoloski_kandidati"]:
+            i["definicija"]=next((x["definicija"] for x in data["definition_candidates"] if x["term"] == i["kandidat"]), None)   
+        #apend to lematized terms
+        print(lemmatized_terms)
+    except Exception as e: print(e)
+    finally:
+        fp.close();
+
+        
+
+    return lemmatized_terms
+
+
 
 def extract_text_prepResp(file, content_type=""):
     content_type = file.content_type