ATE api update 2

ATEapi/Dockerfile

@@ -3,26 +3,23 @@ FROM python:3.8
 
 EXPOSE 5000
 
+# Creates a non-root user with an explicit UID and adds permission to access the /app folder
+# For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers
+RUN adduser -u 5678 --disabled-password --gecos "" appuser
+USER appuser
+
 # Keeps Python from generating .pyc files in the container
-ENV PYTHONDONTWRITEBYTECODE=1
-
 # Turns off buffering for easier container logging
-ENV PYTHONUNBUFFERED=1
+ENV PYTHONDONTWRITEBYTECODE=1 \
-
+    PYTHONUNBUFFERED=1 \
-# RUN apk add --no-cache gcc musl-dev
+    PATH="/home/appuser/.local/bin:${PATH}"
-RUN pip install gunicorn
 
 # Install pip requirements
 COPY requirements.txt .
-RUN python -m pip install -r requirements.txt
+RUN --mount=type=cache,target=/home/appuser/.cache/pip,uid=5678 python -m pip install --user -r requirements.txt
-
+RUN python -c "import classla; classla.download('sl')"
-WORKDIR /app
+# WORKDIR /app
 COPY . /app
 
-# Creates a non-root user with an explicit UID and adds permission to access the /app folder
-# For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers
-RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /app
-USER appuser
-
 # During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug
-CMD ["gunicorn", "--bind", "0.0.0.0:5000", "main:app"]
+CMD ["gunicorn", "--bind", "0.0.0.0:5000", "--timeout", "128",  "app.main:app"]

ATEapi/canonical_utils.py

@@ -1,108 +1,158 @@
+import argparse
+import csv
 import os
-import classla
+import string
-classla.download('sl', logging_level='WARNING')
 
+import classla
 from lemmagen3 import Lemmatizer
 
-def lem_adj(gender, wrd):
+# classla.download("sl", logging_level="WARNING")
-    lem = Lemmatizer()
+classla_nlp_pipeline = classla.Pipeline(
-    if gender == 'm':
+    lang="sl",
-        lem.load_model(os.path.join('./model/lemmagen_models/kanon-adj-male.bin'))
+    processors="tokenize,pos,lemma,depparse",
-    elif gender == 'f':
+    tokenize_pretokenized=True,
-        lem.load_model(os.path.join('./model/lemmagen_models/kanon-adj-female.bin'))
+    logging_level="WARNING",
-    elif gender == 'n':
+    download_method=None
-        lem.load_model(os.path.join('./model/lemmagen_models/kanon-adj-neutral.bin'))
+)
 
-    form = lem.lemmatize(wrd)
+def _resolve_lemmagen_model_loc(model_name):
-    return form
+    basedir = os.path.dirname(__file__)
+    return os.path.join(basedir, "/app/model/lemmagen_models", model_name)
 
+_canon_lemmatizer = Lemmatizer()
+_canon_lemmatizer.load_model(_resolve_lemmagen_model_loc("kanon.bin"))
+canon_lemma = _canon_lemmatizer.lemmatize
 
-def process_nlp_pipeline(lang, text):
+ADJ_LEMMATIZER_LOC_MAP = {
-    nlp = classla.Pipeline(lang=lang, processors='tokenize,pos,lemma', tokenize_pretokenized=True, logging_level='WARNING')
+    ("m", "s"): _resolve_lemmagen_model_loc("kanon-adj-male.bin"),
-    doc = nlp(text)
+    ("m", "p"): _resolve_lemmagen_model_loc("kanon-adj-male-plural.bin"),
-    return doc
+    ("f", "s"): _resolve_lemmagen_model_loc("kanon-adj-female.bin"),
+    ("f", "p"): _resolve_lemmagen_model_loc("kanon-adj-female-plural.bin"),
+    ("n", "s"): _resolve_lemmagen_model_loc("kanon-adj-neutral.bin"),
+    ("n", "p"): _resolve_lemmagen_model_loc("kanon-adj-neutral-plural.bin"),
+}
 
+_ADJ_LEMMATIZER_CACHE = {}
+
+def lem_adj(gender, number, wrd):
+    lem_key = (gender, number)
+    if lem_key not in _ADJ_LEMMATIZER_CACHE:
+        assert lem_key in ADJ_LEMMATIZER_LOC_MAP
+        lemmatizer_model_loc = ADJ_LEMMATIZER_LOC_MAP[lem_key]
+        lemmatizer = Lemmatizer()
+        lemmatizer.load_model(lemmatizer_model_loc)
+        _ADJ_LEMMATIZER_CACHE[lem_key] = lemmatizer
+    lemmatizer = _ADJ_LEMMATIZER_CACHE[lem_key]
+    return lemmatizer.lemmatize(wrd)
 
 def get_adj_msd(head, word):
     feats = head.feats
     feats_dict = {}
-    feats = feats.strip().split('|')
+    feats = feats.strip().split("|")
     for f in feats:
-        f = f.strip().split('=')
+        f = f.strip().split("=")
         feats_dict[f[0]] = f[1]
-    gender = feats_dict['Gender']
+    gender = feats_dict["Gender"]
-    #print(gender)
+    if gender == "Masc" and len(word.xpos) == 6:
-    #gender = gender.strip().split('=')[1]
+        msd = word.xpos[:-1] + "ny"
-    if gender == 'Masc' and len(word.xpos) == 6:
+    elif gender == "Masc" and len(word.xpos) == 7:
-        msd = word.xpos[:-1]+'ny'
+        msd = word.xpos[:-1] + "y"
-    elif gender == 'Masc' and len(word.xpos) == 7:
+    elif gender == "Fem":
-        msd = word.xpos[:-1]+'y'
+        msd = word.xpos[:-1] + "n"
-    elif gender == 'Fem':
+    elif gender == "Neut":
-        msd = word.xpos[:-1]+'n'
+        msd = word.xpos[:-1] + "n"
-    elif gender == 'Neut':
-        msd = word.xpos[:-1]+'n'
     else:
-        msd = None
+        # msd = None
+        msd = "qqqqqq"  # hacky but it means that adverbs are just copied over to the canonical form
     return msd
 
+def _is_single_acronym(term):
+    # (single word, all uppercase and length less than 5 characters)
+    if len(term.words) == 1:
+        word = term.words[0].text
+        return len(word) < 5 and word.isupper()
+    return False
 
-def subfinder(mylist, pattern):
+def _join_term_words(term):
-    matches = []
+    return " ".join([w.text for w in term.words])
-    for i in range(len(mylist)):
-        if mylist[i].text.lower() == pattern[0] and [t.text.lower() for t in mylist[i:i+len(pattern)]] == pattern:
-            matches.append(mylist[i:i+len(pattern)])
-    return matches
-
-
-def find_canon(term):
-    head = None
-    pre = []
-    post = []
-    for word in term.words:
-        if word.upos == 'NOUN' or word.upos == 'PROPN':
-            head = word
-            break
-    if head is None:
-        if len(term.words) == 1:
-            head2 = term.words[0]
-            lem = Lemmatizer()
-            lem.load_model(os.path.join('./model/lemmagen_models/kanon.bin'))
-            head_form = lem.lemmatize(head2.text.lower())
-            return head_form
-        else:
-            return ' '.join([w.text for w in term.words])  # just return the input because we do not cover such case
-    else:
-        for word in term.words:
-            if word.id < head.id:
-                pre.append(word)
-            elif word.id > head.id:
-                post.append(word)
 
+def _process_pre(pre, head, gender, number):
     canon = []
     for el in pre:
         msd = get_adj_msd(head, el)
-        if msd is None:
+        if msd[0] == "A":
-            canon.append(el.lemma.lower())
+            form = lem_adj(gender, number, el.text.lower())
+            canon.append(form)
         else:
-            if msd[0] == 'A' and msd[3] == 'm':
+            canon.append(el.lemma.lower())
-                form = lem_adj('m', el.text.lower())
+    return canon
-                canon.append(form)
+
-            elif msd[0] == 'A' and msd[3] == 'f':
+def find_canon(term):
-                form = lem_adj('f', el.text.lower())
+    if _is_single_acronym(term):
-                canon.append(form)
+        return term.words[0].text
-            elif msd[0] == 'A' and msd[3] == 'n':
+
-                form = lem_adj('n', el.text.lower())
+    head = None
-                canon.append(form)
+    pre = []
+    post = []
+
+    for word in term.words:
+        if word.head == 0:
+            head = word
+        elif head is None:
+            pre.append(word)
+        else:
+            post.append(word)
+    ## special case where all words are proper nouns and each word is canonized independently
+    if all(w.upos == "PROPN" for w in term.words):
+        canon_name = [canon_lemma(w.text) for w in term.words]
+        return " ".join(canon_name)
+
+    if head is None:
+        if len(term.words) == 1:
+            head2 = term.words[0]
+            return canon_lemma(head2.text.lower())
+        else:
+            # just return the input because we do not cover such case
+            return _join_term_words(term)
+    if head.upos == "VERB":  # if the term is not a noun phrase
+        # just return the input because we do not cover such case
+        return _join_term_words(term)
+    if head.upos == "ADJ":
+        if len(term.words) == 1:  # for single word adjectives, return male form
+            return lem_adj("m", "s", term.words[0].text.lower())
+        else:
+            # just return the input because we do not cover such case
+            return _join_term_words(term)
+
+    gender = head.xpos[2]
+    number = head.xpos[3]
+    ending = head.lemma[-1]
+    if gender == "f" and number == "p" and ending in "ie":  # sani, hlače
+        canon = _process_pre(pre, head, gender, number)
+        canon.append(head.lemma)
+    elif gender == "m" and number == "p" and ending == "i":  # možgani
+        canon = _process_pre(pre, head, gender, number)
+        canon.append(head.lemma)
+    elif gender == "n" and number == "p" and ending == "a":  # vrata
+        canon = _process_pre(pre, head, gender, number)
+        canon.append(head.lemma)
+    else:
+        canon = _process_pre(pre, head, gender, "s")
+        head_form = canon_lemma(head.text.lower())
+        canon.append(head_form)
 
-    lem = Lemmatizer()
-    lem.load_model(os.path.join('./model/lemmagen_models/kanon.bin'))
-    head_form = lem.lemmatize(head.text.lower())
-    canon.append(head_form)
     for el in post:
         canon.append(el.text)
-    return ' '.join(canon)
+    return " ".join(canon)
 
 def process(forms):
-    text = '\n'.join(forms)
+    text = "\n".join(forms)
-    doc = process_nlp_pipeline('sl', text)
+    doc = classla_nlp_pipeline(text)
-    return [find_canon(sent) for sent in doc.sentences]
+    canonical_forms = []
+    for term in doc.sentences:
+        try:
+            canonical_form = find_canon(term)
+        except Exception:
+            canonical_form = _join_term_words(term)
+        canonical_forms.append(canonical_form)
+    return canonical_forms

ATEapi/main.py

@@ -3,8 +3,9 @@ import torch
 import torch.nn.functional as F
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 
-from utils import *
+from .utils import *
-from canonical_utils import *
+from .canonical_utils import *
+
 from flask import Flask, request, jsonify
 from transformers import AutoTokenizer, AutoModelForTokenClassification 
 
@@ -12,12 +13,13 @@ from transformers import AutoTokenizer, AutoModelForTokenClassification
 app = Flask(__name__)
 
 label_list=["n", "B-T", "T"]
-tokenizer = AutoTokenizer.from_pretrained('./model/term_extractor/')
+tokenizer = AutoTokenizer.from_pretrained('/app/model/term_extractor/')
-model = AutoModelForTokenClassification.from_pretrained('./model/term_extractor/', num_labels=len(label_list)).to(device)
+model = AutoModelForTokenClassification.from_pretrained('/app/model/term_extractor/', num_labels=len(label_list)).to(device)
 
 @app.route('/predict',methods=['POST'])
 def predict():
     frame = read_conll(request.files['file'])
+    # print(frame)
     sequences = [' '.join(x) for x in frame.word]
     lemma, pos, msd = frame.lemma, frame.pos, frame.msd
     preds = []

ATEapi/requirements.txt

@@ -3,5 +3,6 @@ Flask==2.2.2
 lemmagen3==3.3.2
 numpy==1.23.0
 pandas==1.3.5
-torch
+torch==1.12.0
 transformers==4.20.1
+gunicorn==20.1.0