Main brez modelov
This commit is contained in:
Marko Ferme
@@ -0,0 +1,25 @@
|
||||
# pull official base image
|
||||
FROM python:3.8-slim-buster
|
||||
|
||||
# set work directory
|
||||
WORKDIR /usr/src/app
|
||||
|
||||
# set environment variables
|
||||
ENV PYTHONDONTWRITEBYTECODE 1
|
||||
ENV PYTHONUNBUFFERED 1
|
||||
|
||||
# install system dependencies
|
||||
RUN apt-get update && apt-get install -y netcat
|
||||
#git
|
||||
|
||||
# install dependencies
|
||||
RUN pip install --upgrade pip
|
||||
COPY ./requirements.txt /usr/src/app/requirements.txt
|
||||
RUN pip install -r requirements.txt
|
||||
#RUN --mount=type=cache,target=/root/.cache/pip pip install -r requirements.txt
|
||||
|
||||
# copy project
|
||||
COPY . /usr/src/app/
|
||||
|
||||
# run entrypoint.sh
|
||||
ENTRYPOINT ["/usr/src/app/entrypoint.sh"]
|
||||
@@ -0,0 +1,74 @@
|
||||
###########
|
||||
# BUILDER #
|
||||
###########
|
||||
|
||||
# pull official base image
|
||||
FROM python:3.8-slim-buster as builder
|
||||
|
||||
# set work directory
|
||||
WORKDIR /usr/src/app
|
||||
|
||||
# set environment variables
|
||||
ENV PYTHONDONTWRITEBYTECODE 1
|
||||
ENV PYTHONUNBUFFERED 1
|
||||
|
||||
# install system dependencies
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends gcc
|
||||
|
||||
# lint
|
||||
#RUN pip install --upgrade pip
|
||||
#RUN pip install flake8
|
||||
#COPY . /usr/src/app/
|
||||
#RUN flake8 --ignore=E501,F401 .
|
||||
|
||||
# install python dependencies
|
||||
COPY ./requirements.txt .
|
||||
RUN pip wheel --no-cache-dir --no-deps --wheel-dir /usr/src/app/wheels -r requirements.txt
|
||||
|
||||
|
||||
#########
|
||||
# FINAL #
|
||||
#########
|
||||
|
||||
# pull official base image
|
||||
FROM python:3.8-slim-buster
|
||||
|
||||
# create directory for the app user
|
||||
RUN mkdir -p /home/app
|
||||
|
||||
# create the app user
|
||||
RUN addgroup --system app && adduser --system --group app
|
||||
|
||||
|
||||
# create the appropriate directories
|
||||
ENV HOME=/home/app
|
||||
ENV APP_HOME=/home/app/web
|
||||
RUN mkdir $APP_HOME
|
||||
WORKDIR $APP_HOME
|
||||
|
||||
# install dependencies
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends netcat
|
||||
#git
|
||||
COPY --from=builder /usr/src/app/wheels /wheels
|
||||
COPY --from=builder /usr/src/app/requirements.txt .
|
||||
RUN pip install --upgrade pip
|
||||
RUN pip install --no-cache /wheels/*
|
||||
|
||||
# copy entrypoint-prod.sh
|
||||
COPY ./entrypoint.prod.sh $APP_HOME
|
||||
|
||||
# copy project
|
||||
COPY . $APP_HOME
|
||||
|
||||
# chown all the files to the app user
|
||||
RUN chown -R app:app $APP_HOME
|
||||
|
||||
# a workaround for permission errors for the classla_volume
|
||||
RUN mkdir $HOME/classla_resources && chown app:app $HOME/classla_resources
|
||||
|
||||
# change to the app user
|
||||
USER app
|
||||
|
||||
# run entrypoint.prod.sh
|
||||
ENTRYPOINT ["/home/app/web/entrypoint.prod.sh"]
|
||||
@@ -0,0 +1,53 @@
|
||||
import os
|
||||
import json
|
||||
|
||||
from flask import (
|
||||
Flask,
|
||||
jsonify,
|
||||
send_from_directory,
|
||||
request,
|
||||
redirect,
|
||||
url_for
|
||||
)
|
||||
import werkzeug
|
||||
werkzeug.cached_property = werkzeug.utils.cached_property
|
||||
from werkzeug.utils import secure_filename
|
||||
from werkzeug.middleware.proxy_fix import ProxyFix
|
||||
|
||||
#import flask.scaffold
|
||||
#flask.helpers._endpoint_from_view_func = flask.scaffold._endpoint_from_view_func # fix
|
||||
from flask_restx import Api, Resource, fields, abort, reqparse
|
||||
|
||||
from flask_socketio import SocketIO
|
||||
|
||||
import canonizer
|
||||
|
||||
|
||||
app = Flask(__name__)
|
||||
app.wsgi_app = ProxyFix(app.wsgi_app)
|
||||
api = Api(app, version='1.0',
|
||||
title='API services',
|
||||
description='REST API for canonizer')
|
||||
ns = api.namespace('rest_api', description='REST services API')
|
||||
socketio = SocketIO(app)
|
||||
|
||||
|
||||
canonizer_input = api.model('CanonizerInput', {
|
||||
'forms': fields.List(fields.String, required=True, description='list of input forms')
|
||||
})
|
||||
canonizer_output = api.model('CanonizerOutput', {
|
||||
'canonical_forms': fields.List(fields.String, description='list of canonical forms')
|
||||
})
|
||||
|
||||
|
||||
@ns.route('/canonize')
|
||||
class Canonizer(Resource):
|
||||
@ns.doc('compute canonical forms')
|
||||
@ns.expect(canonizer_input, validate=True)
|
||||
@ns.marshal_with(canonizer_output)
|
||||
def post(self):
|
||||
return {'canonical_forms': canonizer.process(api.payload['forms'])}
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
socketio.run(app, host='0.0.0.0', debug=True, use_reloader=True)
|
||||
@@ -0,0 +1,199 @@
|
||||
import argparse
|
||||
import csv
|
||||
import os
|
||||
import string
|
||||
|
||||
import classla
|
||||
from lemmagen3 import Lemmatizer
|
||||
|
||||
|
||||
classla.download("sl", logging_level="WARNING")
|
||||
classla_nlp_pipeline = classla.Pipeline(
|
||||
lang="sl",
|
||||
processors="tokenize,pos,lemma,depparse",
|
||||
tokenize_pretokenized=True,
|
||||
logging_level="WARNING",
|
||||
)
|
||||
|
||||
|
||||
def _resolve_lemmagen_model_loc(model_name):
|
||||
basedir = os.path.dirname(__file__)
|
||||
return os.path.join(basedir, "lemmagen_models", model_name)
|
||||
|
||||
|
||||
_canon_lemmatizer = Lemmatizer()
|
||||
_canon_lemmatizer.load_model(_resolve_lemmagen_model_loc("kanon.bin"))
|
||||
canon_lemma = _canon_lemmatizer.lemmatize
|
||||
|
||||
ADJ_LEMMATIZER_LOC_MAP = {
|
||||
("m", "s"): _resolve_lemmagen_model_loc("kanon-adj-male.bin"),
|
||||
("m", "p"): _resolve_lemmagen_model_loc("kanon-adj-male-plural.bin"),
|
||||
("f", "s"): _resolve_lemmagen_model_loc("kanon-adj-female.bin"),
|
||||
("f", "p"): _resolve_lemmagen_model_loc("kanon-adj-female-plural.bin"),
|
||||
("n", "s"): _resolve_lemmagen_model_loc("kanon-adj-neutral.bin"),
|
||||
("n", "p"): _resolve_lemmagen_model_loc("kanon-adj-neutral-plural.bin"),
|
||||
}
|
||||
|
||||
_ADJ_LEMMATIZER_CACHE = {}
|
||||
|
||||
|
||||
def lem_adj(gender, number, wrd):
|
||||
lem_key = (gender, number)
|
||||
if lem_key not in _ADJ_LEMMATIZER_CACHE:
|
||||
assert lem_key in ADJ_LEMMATIZER_LOC_MAP
|
||||
lemmatizer_model_loc = ADJ_LEMMATIZER_LOC_MAP[lem_key]
|
||||
lemmatizer = Lemmatizer()
|
||||
lemmatizer.load_model(lemmatizer_model_loc)
|
||||
_ADJ_LEMMATIZER_CACHE[lem_key] = lemmatizer
|
||||
lemmatizer = _ADJ_LEMMATIZER_CACHE[lem_key]
|
||||
return lemmatizer.lemmatize(wrd)
|
||||
|
||||
|
||||
def get_adj_msd(head, word):
|
||||
feats = head.feats
|
||||
feats_dict = {}
|
||||
feats = feats.strip().split("|")
|
||||
for f in feats:
|
||||
f = f.strip().split("=")
|
||||
feats_dict[f[0]] = f[1]
|
||||
gender = feats_dict["Gender"]
|
||||
if gender == "Masc" and len(word.xpos) == 6:
|
||||
msd = word.xpos[:-1] + "ny"
|
||||
elif gender == "Masc" and len(word.xpos) == 7:
|
||||
msd = word.xpos[:-1] + "y"
|
||||
elif gender == "Fem":
|
||||
msd = word.xpos[:-1] + "n"
|
||||
elif gender == "Neut":
|
||||
msd = word.xpos[:-1] + "n"
|
||||
else:
|
||||
# msd = None
|
||||
msd = "qqqqqq" # hacky but it means that adverbs are just copied over to the canonical form
|
||||
return msd
|
||||
|
||||
|
||||
def _is_single_acronym(term):
|
||||
# (single word, all uppercase and length less than 5 characters)
|
||||
if len(term.words) == 1:
|
||||
word = term.words[0].text
|
||||
return len(word) < 5 and word.isupper()
|
||||
return False
|
||||
|
||||
|
||||
def _join_term_words(term):
|
||||
return " ".join([w.text for w in term.words])
|
||||
|
||||
|
||||
def _process_pre(pre, head, gender, number):
|
||||
canon = []
|
||||
for el in pre:
|
||||
msd = get_adj_msd(head, el)
|
||||
if msd[0] == "A":
|
||||
form = lem_adj(gender, number, el.text.lower())
|
||||
canon.append(form)
|
||||
else:
|
||||
canon.append(el.lemma.lower())
|
||||
return canon
|
||||
|
||||
|
||||
def find_canon(term):
|
||||
if _is_single_acronym(term):
|
||||
return term.words[0].text
|
||||
|
||||
head = None
|
||||
pre = []
|
||||
post = []
|
||||
|
||||
for word in term.words:
|
||||
if word.head == 0:
|
||||
head = word
|
||||
elif head is None:
|
||||
pre.append(word)
|
||||
else:
|
||||
post.append(word)
|
||||
## special case where all words are proper nouns and each word is canonized independently
|
||||
if all(w.upos == "PROPN" for w in term.words):
|
||||
canon_name = [canon_lemma(w.text) for w in term.words]
|
||||
return " ".join(canon_name)
|
||||
|
||||
if head is None:
|
||||
if len(term.words) == 1:
|
||||
head2 = term.words[0]
|
||||
return canon_lemma(head2.text.lower())
|
||||
else:
|
||||
# just return the input because we do not cover such case
|
||||
return _join_term_words(term)
|
||||
if head.upos == "VERB": # if the term is not a noun phrase
|
||||
# just return the input because we do not cover such case
|
||||
return _join_term_words(term)
|
||||
if head.upos == "ADJ":
|
||||
if len(term.words) == 1: # for single word adjectives, return male form
|
||||
return lem_adj("m", "s", term.words[0].text.lower())
|
||||
else:
|
||||
# just return the input because we do not cover such case
|
||||
return _join_term_words(term)
|
||||
|
||||
gender = head.xpos[2]
|
||||
number = head.xpos[3]
|
||||
ending = head.lemma[-1]
|
||||
if gender == "f" and number == "p" and ending in "ie": # sani, hlače
|
||||
canon = _process_pre(pre, head, gender, number)
|
||||
canon.append(head.lemma)
|
||||
elif gender == "m" and number == "p" and ending == "i": # možgani
|
||||
canon = _process_pre(pre, head, gender, number)
|
||||
canon.append(head.lemma)
|
||||
elif gender == "n" and number == "p" and ending == "a": # vrata
|
||||
canon = _process_pre(pre, head, gender, number)
|
||||
canon.append(head.lemma)
|
||||
else:
|
||||
canon = _process_pre(pre, head, gender, "s")
|
||||
head_form = canon_lemma(head.text.lower())
|
||||
canon.append(head_form)
|
||||
|
||||
for el in post:
|
||||
canon.append(el.text)
|
||||
return " ".join(canon)
|
||||
|
||||
|
||||
def process(forms):
|
||||
text = "\n".join(forms)
|
||||
doc = classla_nlp_pipeline(text)
|
||||
canonical_forms = []
|
||||
for term in doc.sentences:
|
||||
try:
|
||||
canonical_form = find_canon(term)
|
||||
except Exception:
|
||||
canonical_form = _join_term_words(term)
|
||||
canonical_forms.append(canonical_form)
|
||||
return canonical_forms
|
||||
|
||||
|
||||
def read_csv(fname, columnID=0):
|
||||
data = []
|
||||
with open(fname) as csvfile:
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(csvfile.read(2048))
|
||||
except csv.Error:
|
||||
print("Warning: cannot determine delimiter, assuming Excel CSV dialect.")
|
||||
dialect = "excel"
|
||||
csvfile.seek(0)
|
||||
reader = csv.reader(csvfile, dialect)
|
||||
for i, row in enumerate(reader):
|
||||
try:
|
||||
data.append(row[columnID].strip(string.punctuation))
|
||||
except:
|
||||
print("Error, line {}".format(i))
|
||||
return data
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Converter to canonical form in Slovene language"
|
||||
)
|
||||
parser.add_argument("csv_file", type=argparse.FileType("r"), help="Input csv file")
|
||||
parser.add_argument("column_id", type=int, help="CSV column number (zero indexed)")
|
||||
args = parser.parse_args()
|
||||
|
||||
data = read_csv(args.csv_file.name, columnID=args.column_id)
|
||||
results = process(data)
|
||||
for canon in results:
|
||||
print("{}".format(canon))
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
|
||||
# patching reldi-tokeniser for multi-threaded environment
|
||||
sed -i 's/signal(/pass;#signal(/g' /usr/local/lib/python3.8/site-packages/classla/submodules/reldi_tokeniser/tokeniser.py
|
||||
|
||||
# pre-download models
|
||||
python -c "import classla;classla.download('sl', logging_level='INFO')"
|
||||
|
||||
exec "$@"
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
|
||||
# patching reldi-tokeniser for multi-threaded environment
|
||||
echo "Patching tokeniser.py..."
|
||||
sed -i 's/signal(/pass;#signal(/g' /usr/local/lib/python3.8/site-packages/classla/submodules/reldi_tokeniser/tokeniser.py
|
||||
|
||||
# pre-download models
|
||||
python -c "import classla;classla.download('sl', logging_level='INFO')"
|
||||
|
||||
exec "$@"
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,8 @@
|
||||
Flask==2.1.*
|
||||
flask-restx
|
||||
gunicorn==20.1.0
|
||||
classla==1.0.1
|
||||
lemmagen3==3.3.1
|
||||
flask-socketio==5.1.0
|
||||
simple-websocket==0.2.0
|
||||
markupsafe==2.0.1 # to resolve issue with missing soft_unicode function
|
||||
Reference in New Issue
Block a user