From e9345ee8876087b4e8343d07c0b78d5416ade074 Mon Sep 17 00:00:00 2001 From: Kikimanox Date: Fri, 11 Nov 2022 15:45:00 +0100 Subject: [PATCH] ATE api update 2 --- ATEapi/Dockerfile | 27 +++-- ATEapi/canonical_utils.py | 210 +++++++++++++++++++++++--------------- ATEapi/main.py | 10 +- ATEapi/requirements.txt | 3 +- 4 files changed, 150 insertions(+), 100 deletions(-) diff --git a/ATEapi/Dockerfile b/ATEapi/Dockerfile index 3d32290..aae166c 100644 --- a/ATEapi/Dockerfile +++ b/ATEapi/Dockerfile @@ -3,26 +3,23 @@ FROM python:3.8 EXPOSE 5000 +# Creates a non-root user with an explicit UID and adds permission to access the /app folder +# For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers +RUN adduser -u 5678 --disabled-password --gecos "" appuser +USER appuser + # Keeps Python from generating .pyc files in the container -ENV PYTHONDONTWRITEBYTECODE=1 - # Turns off buffering for easier container logging -ENV PYTHONUNBUFFERED=1 - -# RUN apk add --no-cache gcc musl-dev -RUN pip install gunicorn +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PATH="/home/appuser/.local/bin:${PATH}" # Install pip requirements COPY requirements.txt . -RUN python -m pip install -r requirements.txt - -WORKDIR /app +RUN --mount=type=cache,target=/home/appuser/.cache/pip,uid=5678 python -m pip install --user -r requirements.txt +RUN python -c "import classla; classla.download('sl')" +# WORKDIR /app COPY . /app -# Creates a non-root user with an explicit UID and adds permission to access the /app folder -# For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers -RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /app -USER appuser - # During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug -CMD ["gunicorn", "--bind", "0.0.0.0:5000", "main:app"] +CMD ["gunicorn", "--bind", "0.0.0.0:5000", "--timeout", "128", "app.main:app"] diff --git a/ATEapi/canonical_utils.py b/ATEapi/canonical_utils.py index 0b7f9ae..e51359c 100644 --- a/ATEapi/canonical_utils.py +++ b/ATEapi/canonical_utils.py @@ -1,108 +1,158 @@ +import argparse +import csv import os -import classla -classla.download('sl', logging_level='WARNING') +import string +import classla from lemmagen3 import Lemmatizer -def lem_adj(gender, wrd): - lem = Lemmatizer() - if gender == 'm': - lem.load_model(os.path.join('./model/lemmagen_models/kanon-adj-male.bin')) - elif gender == 'f': - lem.load_model(os.path.join('./model/lemmagen_models/kanon-adj-female.bin')) - elif gender == 'n': - lem.load_model(os.path.join('./model/lemmagen_models/kanon-adj-neutral.bin')) +# classla.download("sl", logging_level="WARNING") +classla_nlp_pipeline = classla.Pipeline( + lang="sl", + processors="tokenize,pos,lemma,depparse", + tokenize_pretokenized=True, + logging_level="WARNING", + download_method=None +) - form = lem.lemmatize(wrd) - return form +def _resolve_lemmagen_model_loc(model_name): + basedir = os.path.dirname(__file__) + return os.path.join(basedir, "/app/model/lemmagen_models", model_name) +_canon_lemmatizer = Lemmatizer() +_canon_lemmatizer.load_model(_resolve_lemmagen_model_loc("kanon.bin")) +canon_lemma = _canon_lemmatizer.lemmatize -def process_nlp_pipeline(lang, text): - nlp = classla.Pipeline(lang=lang, processors='tokenize,pos,lemma', tokenize_pretokenized=True, logging_level='WARNING') - doc = nlp(text) - return doc +ADJ_LEMMATIZER_LOC_MAP = { + ("m", "s"): _resolve_lemmagen_model_loc("kanon-adj-male.bin"), + ("m", "p"): _resolve_lemmagen_model_loc("kanon-adj-male-plural.bin"), + ("f", "s"): _resolve_lemmagen_model_loc("kanon-adj-female.bin"), + ("f", "p"): _resolve_lemmagen_model_loc("kanon-adj-female-plural.bin"), + ("n", "s"): _resolve_lemmagen_model_loc("kanon-adj-neutral.bin"), + ("n", "p"): _resolve_lemmagen_model_loc("kanon-adj-neutral-plural.bin"), +} +_ADJ_LEMMATIZER_CACHE = {} + +def lem_adj(gender, number, wrd): + lem_key = (gender, number) + if lem_key not in _ADJ_LEMMATIZER_CACHE: + assert lem_key in ADJ_LEMMATIZER_LOC_MAP + lemmatizer_model_loc = ADJ_LEMMATIZER_LOC_MAP[lem_key] + lemmatizer = Lemmatizer() + lemmatizer.load_model(lemmatizer_model_loc) + _ADJ_LEMMATIZER_CACHE[lem_key] = lemmatizer + lemmatizer = _ADJ_LEMMATIZER_CACHE[lem_key] + return lemmatizer.lemmatize(wrd) def get_adj_msd(head, word): feats = head.feats feats_dict = {} - feats = feats.strip().split('|') + feats = feats.strip().split("|") for f in feats: - f = f.strip().split('=') + f = f.strip().split("=") feats_dict[f[0]] = f[1] - gender = feats_dict['Gender'] - #print(gender) - #gender = gender.strip().split('=')[1] - if gender == 'Masc' and len(word.xpos) == 6: - msd = word.xpos[:-1]+'ny' - elif gender == 'Masc' and len(word.xpos) == 7: - msd = word.xpos[:-1]+'y' - elif gender == 'Fem': - msd = word.xpos[:-1]+'n' - elif gender == 'Neut': - msd = word.xpos[:-1]+'n' + gender = feats_dict["Gender"] + if gender == "Masc" and len(word.xpos) == 6: + msd = word.xpos[:-1] + "ny" + elif gender == "Masc" and len(word.xpos) == 7: + msd = word.xpos[:-1] + "y" + elif gender == "Fem": + msd = word.xpos[:-1] + "n" + elif gender == "Neut": + msd = word.xpos[:-1] + "n" else: - msd = None + # msd = None + msd = "qqqqqq" # hacky but it means that adverbs are just copied over to the canonical form return msd +def _is_single_acronym(term): + # (single word, all uppercase and length less than 5 characters) + if len(term.words) == 1: + word = term.words[0].text + return len(word) < 5 and word.isupper() + return False -def subfinder(mylist, pattern): - matches = [] - for i in range(len(mylist)): - if mylist[i].text.lower() == pattern[0] and [t.text.lower() for t in mylist[i:i+len(pattern)]] == pattern: - matches.append(mylist[i:i+len(pattern)]) - return matches - - -def find_canon(term): - head = None - pre = [] - post = [] - for word in term.words: - if word.upos == 'NOUN' or word.upos == 'PROPN': - head = word - break - if head is None: - if len(term.words) == 1: - head2 = term.words[0] - lem = Lemmatizer() - lem.load_model(os.path.join('./model/lemmagen_models/kanon.bin')) - head_form = lem.lemmatize(head2.text.lower()) - return head_form - else: - return ' '.join([w.text for w in term.words]) # just return the input because we do not cover such case - else: - for word in term.words: - if word.id < head.id: - pre.append(word) - elif word.id > head.id: - post.append(word) +def _join_term_words(term): + return " ".join([w.text for w in term.words]) +def _process_pre(pre, head, gender, number): canon = [] for el in pre: msd = get_adj_msd(head, el) - if msd is None: - canon.append(el.lemma.lower()) + if msd[0] == "A": + form = lem_adj(gender, number, el.text.lower()) + canon.append(form) else: - if msd[0] == 'A' and msd[3] == 'm': - form = lem_adj('m', el.text.lower()) - canon.append(form) - elif msd[0] == 'A' and msd[3] == 'f': - form = lem_adj('f', el.text.lower()) - canon.append(form) - elif msd[0] == 'A' and msd[3] == 'n': - form = lem_adj('n', el.text.lower()) - canon.append(form) + canon.append(el.lemma.lower()) + return canon + +def find_canon(term): + if _is_single_acronym(term): + return term.words[0].text + + head = None + pre = [] + post = [] + + for word in term.words: + if word.head == 0: + head = word + elif head is None: + pre.append(word) + else: + post.append(word) + ## special case where all words are proper nouns and each word is canonized independently + if all(w.upos == "PROPN" for w in term.words): + canon_name = [canon_lemma(w.text) for w in term.words] + return " ".join(canon_name) + + if head is None: + if len(term.words) == 1: + head2 = term.words[0] + return canon_lemma(head2.text.lower()) + else: + # just return the input because we do not cover such case + return _join_term_words(term) + if head.upos == "VERB": # if the term is not a noun phrase + # just return the input because we do not cover such case + return _join_term_words(term) + if head.upos == "ADJ": + if len(term.words) == 1: # for single word adjectives, return male form + return lem_adj("m", "s", term.words[0].text.lower()) + else: + # just return the input because we do not cover such case + return _join_term_words(term) + + gender = head.xpos[2] + number = head.xpos[3] + ending = head.lemma[-1] + if gender == "f" and number == "p" and ending in "ie": # sani, hlače + canon = _process_pre(pre, head, gender, number) + canon.append(head.lemma) + elif gender == "m" and number == "p" and ending == "i": # možgani + canon = _process_pre(pre, head, gender, number) + canon.append(head.lemma) + elif gender == "n" and number == "p" and ending == "a": # vrata + canon = _process_pre(pre, head, gender, number) + canon.append(head.lemma) + else: + canon = _process_pre(pre, head, gender, "s") + head_form = canon_lemma(head.text.lower()) + canon.append(head_form) - lem = Lemmatizer() - lem.load_model(os.path.join('./model/lemmagen_models/kanon.bin')) - head_form = lem.lemmatize(head.text.lower()) - canon.append(head_form) for el in post: canon.append(el.text) - return ' '.join(canon) + return " ".join(canon) def process(forms): - text = '\n'.join(forms) - doc = process_nlp_pipeline('sl', text) - return [find_canon(sent) for sent in doc.sentences] + text = "\n".join(forms) + doc = classla_nlp_pipeline(text) + canonical_forms = [] + for term in doc.sentences: + try: + canonical_form = find_canon(term) + except Exception: + canonical_form = _join_term_words(term) + canonical_forms.append(canonical_form) + return canonical_forms diff --git a/ATEapi/main.py b/ATEapi/main.py index 2a6463e..67d6c0d 100644 --- a/ATEapi/main.py +++ b/ATEapi/main.py @@ -3,8 +3,9 @@ import torch import torch.nn.functional as F device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') -from utils import * -from canonical_utils import * +from .utils import * +from .canonical_utils import * + from flask import Flask, request, jsonify from transformers import AutoTokenizer, AutoModelForTokenClassification @@ -12,12 +13,13 @@ from transformers import AutoTokenizer, AutoModelForTokenClassification app = Flask(__name__) label_list=["n", "B-T", "T"] -tokenizer = AutoTokenizer.from_pretrained('./model/term_extractor/') -model = AutoModelForTokenClassification.from_pretrained('./model/term_extractor/', num_labels=len(label_list)).to(device) +tokenizer = AutoTokenizer.from_pretrained('/app/model/term_extractor/') +model = AutoModelForTokenClassification.from_pretrained('/app/model/term_extractor/', num_labels=len(label_list)).to(device) @app.route('/predict',methods=['POST']) def predict(): frame = read_conll(request.files['file']) + # print(frame) sequences = [' '.join(x) for x in frame.word] lemma, pos, msd = frame.lemma, frame.pos, frame.msd preds = [] diff --git a/ATEapi/requirements.txt b/ATEapi/requirements.txt index c0302fe..f1cc4be 100644 --- a/ATEapi/requirements.txt +++ b/ATEapi/requirements.txt @@ -3,5 +3,6 @@ Flask==2.2.2 lemmagen3==3.3.2 numpy==1.23.0 pandas==1.3.5 -torch +torch==1.12.0 transformers==4.20.1 +gunicorn==20.1.0