ATE api update 2
This commit is contained in:
Kikimanox
+12
-15
@@ -3,26 +3,23 @@ FROM python:3.8
|
||||
|
||||
EXPOSE 5000
|
||||
|
||||
# Creates a non-root user with an explicit UID and adds permission to access the /app folder
|
||||
# For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers
|
||||
RUN adduser -u 5678 --disabled-password --gecos "" appuser
|
||||
USER appuser
|
||||
|
||||
# Keeps Python from generating .pyc files in the container
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
|
||||
# Turns off buffering for easier container logging
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
|
||||
# RUN apk add --no-cache gcc musl-dev
|
||||
RUN pip install gunicorn
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PATH="/home/appuser/.local/bin:${PATH}"
|
||||
|
||||
# Install pip requirements
|
||||
COPY requirements.txt .
|
||||
RUN python -m pip install -r requirements.txt
|
||||
|
||||
WORKDIR /app
|
||||
RUN --mount=type=cache,target=/home/appuser/.cache/pip,uid=5678 python -m pip install --user -r requirements.txt
|
||||
RUN python -c "import classla; classla.download('sl')"
|
||||
# WORKDIR /app
|
||||
COPY . /app
|
||||
|
||||
# Creates a non-root user with an explicit UID and adds permission to access the /app folder
|
||||
# For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers
|
||||
RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /app
|
||||
USER appuser
|
||||
|
||||
# During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug
|
||||
CMD ["gunicorn", "--bind", "0.0.0.0:5000", "main:app"]
|
||||
CMD ["gunicorn", "--bind", "0.0.0.0:5000", "--timeout", "128", "app.main:app"]
|
||||
|
||||
+129
-79
@@ -1,108 +1,158 @@
|
||||
import argparse
|
||||
import csv
|
||||
import os
|
||||
import classla
|
||||
classla.download('sl', logging_level='WARNING')
|
||||
import string
|
||||
|
||||
import classla
|
||||
from lemmagen3 import Lemmatizer
|
||||
|
||||
def lem_adj(gender, wrd):
|
||||
lem = Lemmatizer()
|
||||
if gender == 'm':
|
||||
lem.load_model(os.path.join('./model/lemmagen_models/kanon-adj-male.bin'))
|
||||
elif gender == 'f':
|
||||
lem.load_model(os.path.join('./model/lemmagen_models/kanon-adj-female.bin'))
|
||||
elif gender == 'n':
|
||||
lem.load_model(os.path.join('./model/lemmagen_models/kanon-adj-neutral.bin'))
|
||||
# classla.download("sl", logging_level="WARNING")
|
||||
classla_nlp_pipeline = classla.Pipeline(
|
||||
lang="sl",
|
||||
processors="tokenize,pos,lemma,depparse",
|
||||
tokenize_pretokenized=True,
|
||||
logging_level="WARNING",
|
||||
download_method=None
|
||||
)
|
||||
|
||||
form = lem.lemmatize(wrd)
|
||||
return form
|
||||
def _resolve_lemmagen_model_loc(model_name):
|
||||
basedir = os.path.dirname(__file__)
|
||||
return os.path.join(basedir, "/app/model/lemmagen_models", model_name)
|
||||
|
||||
_canon_lemmatizer = Lemmatizer()
|
||||
_canon_lemmatizer.load_model(_resolve_lemmagen_model_loc("kanon.bin"))
|
||||
canon_lemma = _canon_lemmatizer.lemmatize
|
||||
|
||||
def process_nlp_pipeline(lang, text):
|
||||
nlp = classla.Pipeline(lang=lang, processors='tokenize,pos,lemma', tokenize_pretokenized=True, logging_level='WARNING')
|
||||
doc = nlp(text)
|
||||
return doc
|
||||
ADJ_LEMMATIZER_LOC_MAP = {
|
||||
("m", "s"): _resolve_lemmagen_model_loc("kanon-adj-male.bin"),
|
||||
("m", "p"): _resolve_lemmagen_model_loc("kanon-adj-male-plural.bin"),
|
||||
("f", "s"): _resolve_lemmagen_model_loc("kanon-adj-female.bin"),
|
||||
("f", "p"): _resolve_lemmagen_model_loc("kanon-adj-female-plural.bin"),
|
||||
("n", "s"): _resolve_lemmagen_model_loc("kanon-adj-neutral.bin"),
|
||||
("n", "p"): _resolve_lemmagen_model_loc("kanon-adj-neutral-plural.bin"),
|
||||
}
|
||||
|
||||
_ADJ_LEMMATIZER_CACHE = {}
|
||||
|
||||
def lem_adj(gender, number, wrd):
|
||||
lem_key = (gender, number)
|
||||
if lem_key not in _ADJ_LEMMATIZER_CACHE:
|
||||
assert lem_key in ADJ_LEMMATIZER_LOC_MAP
|
||||
lemmatizer_model_loc = ADJ_LEMMATIZER_LOC_MAP[lem_key]
|
||||
lemmatizer = Lemmatizer()
|
||||
lemmatizer.load_model(lemmatizer_model_loc)
|
||||
_ADJ_LEMMATIZER_CACHE[lem_key] = lemmatizer
|
||||
lemmatizer = _ADJ_LEMMATIZER_CACHE[lem_key]
|
||||
return lemmatizer.lemmatize(wrd)
|
||||
|
||||
def get_adj_msd(head, word):
|
||||
feats = head.feats
|
||||
feats_dict = {}
|
||||
feats = feats.strip().split('|')
|
||||
feats = feats.strip().split("|")
|
||||
for f in feats:
|
||||
f = f.strip().split('=')
|
||||
f = f.strip().split("=")
|
||||
feats_dict[f[0]] = f[1]
|
||||
gender = feats_dict['Gender']
|
||||
#print(gender)
|
||||
#gender = gender.strip().split('=')[1]
|
||||
if gender == 'Masc' and len(word.xpos) == 6:
|
||||
msd = word.xpos[:-1]+'ny'
|
||||
elif gender == 'Masc' and len(word.xpos) == 7:
|
||||
msd = word.xpos[:-1]+'y'
|
||||
elif gender == 'Fem':
|
||||
msd = word.xpos[:-1]+'n'
|
||||
elif gender == 'Neut':
|
||||
msd = word.xpos[:-1]+'n'
|
||||
gender = feats_dict["Gender"]
|
||||
if gender == "Masc" and len(word.xpos) == 6:
|
||||
msd = word.xpos[:-1] + "ny"
|
||||
elif gender == "Masc" and len(word.xpos) == 7:
|
||||
msd = word.xpos[:-1] + "y"
|
||||
elif gender == "Fem":
|
||||
msd = word.xpos[:-1] + "n"
|
||||
elif gender == "Neut":
|
||||
msd = word.xpos[:-1] + "n"
|
||||
else:
|
||||
msd = None
|
||||
# msd = None
|
||||
msd = "qqqqqq" # hacky but it means that adverbs are just copied over to the canonical form
|
||||
return msd
|
||||
|
||||
|
||||
def subfinder(mylist, pattern):
|
||||
matches = []
|
||||
for i in range(len(mylist)):
|
||||
if mylist[i].text.lower() == pattern[0] and [t.text.lower() for t in mylist[i:i+len(pattern)]] == pattern:
|
||||
matches.append(mylist[i:i+len(pattern)])
|
||||
return matches
|
||||
|
||||
|
||||
def find_canon(term):
|
||||
head = None
|
||||
pre = []
|
||||
post = []
|
||||
for word in term.words:
|
||||
if word.upos == 'NOUN' or word.upos == 'PROPN':
|
||||
head = word
|
||||
break
|
||||
if head is None:
|
||||
def _is_single_acronym(term):
|
||||
# (single word, all uppercase and length less than 5 characters)
|
||||
if len(term.words) == 1:
|
||||
head2 = term.words[0]
|
||||
lem = Lemmatizer()
|
||||
lem.load_model(os.path.join('./model/lemmagen_models/kanon.bin'))
|
||||
head_form = lem.lemmatize(head2.text.lower())
|
||||
return head_form
|
||||
else:
|
||||
return ' '.join([w.text for w in term.words]) # just return the input because we do not cover such case
|
||||
else:
|
||||
for word in term.words:
|
||||
if word.id < head.id:
|
||||
pre.append(word)
|
||||
elif word.id > head.id:
|
||||
post.append(word)
|
||||
word = term.words[0].text
|
||||
return len(word) < 5 and word.isupper()
|
||||
return False
|
||||
|
||||
def _join_term_words(term):
|
||||
return " ".join([w.text for w in term.words])
|
||||
|
||||
def _process_pre(pre, head, gender, number):
|
||||
canon = []
|
||||
for el in pre:
|
||||
msd = get_adj_msd(head, el)
|
||||
if msd is None:
|
||||
canon.append(el.lemma.lower())
|
||||
if msd[0] == "A":
|
||||
form = lem_adj(gender, number, el.text.lower())
|
||||
canon.append(form)
|
||||
else:
|
||||
if msd[0] == 'A' and msd[3] == 'm':
|
||||
form = lem_adj('m', el.text.lower())
|
||||
canon.append(form)
|
||||
elif msd[0] == 'A' and msd[3] == 'f':
|
||||
form = lem_adj('f', el.text.lower())
|
||||
canon.append(form)
|
||||
elif msd[0] == 'A' and msd[3] == 'n':
|
||||
form = lem_adj('n', el.text.lower())
|
||||
canon.append(form)
|
||||
canon.append(el.lemma.lower())
|
||||
return canon
|
||||
|
||||
lem = Lemmatizer()
|
||||
lem.load_model(os.path.join('./model/lemmagen_models/kanon.bin'))
|
||||
head_form = lem.lemmatize(head.text.lower())
|
||||
def find_canon(term):
|
||||
if _is_single_acronym(term):
|
||||
return term.words[0].text
|
||||
|
||||
head = None
|
||||
pre = []
|
||||
post = []
|
||||
|
||||
for word in term.words:
|
||||
if word.head == 0:
|
||||
head = word
|
||||
elif head is None:
|
||||
pre.append(word)
|
||||
else:
|
||||
post.append(word)
|
||||
## special case where all words are proper nouns and each word is canonized independently
|
||||
if all(w.upos == "PROPN" for w in term.words):
|
||||
canon_name = [canon_lemma(w.text) for w in term.words]
|
||||
return " ".join(canon_name)
|
||||
|
||||
if head is None:
|
||||
if len(term.words) == 1:
|
||||
head2 = term.words[0]
|
||||
return canon_lemma(head2.text.lower())
|
||||
else:
|
||||
# just return the input because we do not cover such case
|
||||
return _join_term_words(term)
|
||||
if head.upos == "VERB": # if the term is not a noun phrase
|
||||
# just return the input because we do not cover such case
|
||||
return _join_term_words(term)
|
||||
if head.upos == "ADJ":
|
||||
if len(term.words) == 1: # for single word adjectives, return male form
|
||||
return lem_adj("m", "s", term.words[0].text.lower())
|
||||
else:
|
||||
# just return the input because we do not cover such case
|
||||
return _join_term_words(term)
|
||||
|
||||
gender = head.xpos[2]
|
||||
number = head.xpos[3]
|
||||
ending = head.lemma[-1]
|
||||
if gender == "f" and number == "p" and ending in "ie": # sani, hlače
|
||||
canon = _process_pre(pre, head, gender, number)
|
||||
canon.append(head.lemma)
|
||||
elif gender == "m" and number == "p" and ending == "i": # možgani
|
||||
canon = _process_pre(pre, head, gender, number)
|
||||
canon.append(head.lemma)
|
||||
elif gender == "n" and number == "p" and ending == "a": # vrata
|
||||
canon = _process_pre(pre, head, gender, number)
|
||||
canon.append(head.lemma)
|
||||
else:
|
||||
canon = _process_pre(pre, head, gender, "s")
|
||||
head_form = canon_lemma(head.text.lower())
|
||||
canon.append(head_form)
|
||||
|
||||
for el in post:
|
||||
canon.append(el.text)
|
||||
return ' '.join(canon)
|
||||
return " ".join(canon)
|
||||
|
||||
def process(forms):
|
||||
text = '\n'.join(forms)
|
||||
doc = process_nlp_pipeline('sl', text)
|
||||
return [find_canon(sent) for sent in doc.sentences]
|
||||
text = "\n".join(forms)
|
||||
doc = classla_nlp_pipeline(text)
|
||||
canonical_forms = []
|
||||
for term in doc.sentences:
|
||||
try:
|
||||
canonical_form = find_canon(term)
|
||||
except Exception:
|
||||
canonical_form = _join_term_words(term)
|
||||
canonical_forms.append(canonical_form)
|
||||
return canonical_forms
|
||||
|
||||
+6
-4
@@ -3,8 +3,9 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
|
||||
|
||||
from utils import *
|
||||
from canonical_utils import *
|
||||
from .utils import *
|
||||
from .canonical_utils import *
|
||||
|
||||
from flask import Flask, request, jsonify
|
||||
from transformers import AutoTokenizer, AutoModelForTokenClassification
|
||||
|
||||
@@ -12,12 +13,13 @@ from transformers import AutoTokenizer, AutoModelForTokenClassification
|
||||
app = Flask(__name__)
|
||||
|
||||
label_list=["n", "B-T", "T"]
|
||||
tokenizer = AutoTokenizer.from_pretrained('./model/term_extractor/')
|
||||
model = AutoModelForTokenClassification.from_pretrained('./model/term_extractor/', num_labels=len(label_list)).to(device)
|
||||
tokenizer = AutoTokenizer.from_pretrained('/app/model/term_extractor/')
|
||||
model = AutoModelForTokenClassification.from_pretrained('/app/model/term_extractor/', num_labels=len(label_list)).to(device)
|
||||
|
||||
@app.route('/predict',methods=['POST'])
|
||||
def predict():
|
||||
frame = read_conll(request.files['file'])
|
||||
# print(frame)
|
||||
sequences = [' '.join(x) for x in frame.word]
|
||||
lemma, pos, msd = frame.lemma, frame.pos, frame.msd
|
||||
preds = []
|
||||
|
||||
@@ -3,5 +3,6 @@ Flask==2.2.2
|
||||
lemmagen3==3.3.2
|
||||
numpy==1.23.0
|
||||
pandas==1.3.5
|
||||
torch
|
||||
torch==1.12.0
|
||||
transformers==4.20.1
|
||||
gunicorn==20.1.0
|
||||
|
||||
Reference in New Issue
Block a user