diff --git a/ATEapi/Dockerfile b/ATEapi/Dockerfile index 3d32290..d151b59 100644 --- a/ATEapi/Dockerfile +++ b/ATEapi/Dockerfile @@ -19,6 +19,9 @@ RUN python -m pip install -r requirements.txt WORKDIR /app COPY . /app + +ADD https://kt-cloud.ijs.si/index.php/s/T4qtSKxbxgqr6c5/download/pytorch_model.bin ./model/term_extractor/ + # Creates a non-root user with an explicit UID and adds permission to access the /app folder # For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /app diff --git a/Dockerfile b/Dockerfile index f038eb8..f781814 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,11 @@ FROM python:3.9 -RUN mkdir -p /usr/src/app -WORKDIR /usr/src/app RUN apt-get update && apt-get install python3-pil tesseract-ocr libtesseract-dev tesseract-ocr-eng tesseract-ocr-slv tesseract-ocr-script-latn ffmpeg libsm6 libxext6 libgl1 libmagic1 -y +RUN mkdir -p /usr/src/app +WORKDIR /usr/src/app COPY requirements.txt /usr/src/app/ - # RUN pip install --upgrade pip RUN pip3 install --no-cache-dir -r requirements.txt diff --git a/docker-compose.yml b/docker-compose.yml index 3381022..7b64e0e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,7 @@ services: - "8080:8080" volumes: # when pushing to server, remove the first dot here in front of /mnt - /mnt/ssd/ds_ftp/classla_OS2022:/usr/src/app/classla_OS2022 - - ./swagger_server/classla/classla_resources:/root/classla_resources + - ../classla/classla_resources:/root/classla_resources environment: - PYTHONUNBUFFERED=1 tika2: @@ -20,7 +20,7 @@ services: ate-api: build: ./ATEapi restart: always - ports: - - "5000:5000" + # ports: + # - "5000:5000" volumes: - - ./swagger_server/classla/classla_resources:/home/appuser/classla_resources \ No newline at end of file + - ../classla/classla_resources:/home/appuser/classla_resources \ No newline at end of file diff --git a/swagger_server/controllers/extract_controller.py b/swagger_server/controllers/extract_controller.py index 0051f62..b53df7d 100644 --- a/swagger_server/controllers/extract_controller.py +++ b/swagger_server/controllers/extract_controller.py @@ -2,7 +2,7 @@ import connexion import json from pathlib import Path from swagger_server.models.izlusci_body import IzlusciBody # noqa: E501 -from swagger_server.classla import cl_utils +from swagger_server.utils import cl_utils import requests # ATEapi_endpoint = "http://localhost:5000/predict" diff --git a/swagger_server/controllers/jobs_controller.py b/swagger_server/controllers/jobs_controller.py index bbd0595..6c5518b 100644 --- a/swagger_server/controllers/jobs_controller.py +++ b/swagger_server/controllers/jobs_controller.py @@ -7,7 +7,7 @@ import concurrent.futures as cf from swagger_server.models.job_response import JobResponse # noqa: E501 from swagger_server.requets_db.models.vrsta import (Job) from threading import Thread -from swagger_server.classla import cl_utils +from swagger_server.utils import cl_utils from swagger_server.utils import txt_utils from werkzeug.datastructures import FileStorage import threading diff --git a/swagger_server/controllers/marktext_sync_controller.py b/swagger_server/controllers/marktext_sync_controller.py index bd3531a..051a2ad 100644 --- a/swagger_server/controllers/marktext_sync_controller.py +++ b/swagger_server/controllers/marktext_sync_controller.py @@ -1,4 +1,4 @@ -from swagger_server.classla import cl_utils +from swagger_server.utils import cl_utils from swagger_server.utils import txt_utils @@ -38,6 +38,7 @@ def datoteka_v_besedilo_sync_post(file=None): # noqa: E501 try: return txt_utils.extract_text_prepResp(file) except Exception as e: + print(e) return str(e), 500 diff --git a/swagger_server/utils/cl_utils.py b/swagger_server/utils/cl_utils.py new file mode 100644 index 0000000..d1507f9 --- /dev/null +++ b/swagger_server/utils/cl_utils.py @@ -0,0 +1,34 @@ +import classla +import time +from swagger_server import util +from pathlib import Path +import re + +nlp_loaded = False +nlpSlo = classla.Pipeline('sl', processors='tokenize,ner,pos,lemma,depparse') +nlp_loaded = True + +sent_extractor = re.compile(r"# sent_id = \d+\.\d+(.*?)\n\n", re.MULTILINE | re.DOTALL) + +def raw_text_to_conllu(text): + try: + docall = nlpSlo(text) + docallconllu = docall.to_conll() + + return docallconllu, 200 + except Exception as e: + return e, 400 + + +def multipla_conllus_to_one_from_file_ids(list_file_ids): + sent_cnt = 1 + ret = "# newpar id = 1\n" + files = [f'{util.get_conllu_file_path_by_id(i)}' for i in list_file_ids] + for file in files: + txt = Path(file).read_text('utf-8') + matches = sent_extractor.finditer(txt) + for match in matches: + ret += f'# sent_id = 1.{sent_cnt}{match.group(1)}\n\n' + sent_cnt += 1 + + return ret diff --git a/swagger_server/utils/txt_utils.py b/swagger_server/utils/txt_utils.py index f3b83fe..cca18df 100644 --- a/swagger_server/utils/txt_utils.py +++ b/swagger_server/utils/txt_utils.py @@ -5,7 +5,7 @@ import requests import docx import xml.etree.ElementTree as ET from PyPDF2 import PdfReader -from swagger_server.classla import cl_utils +from swagger_server.utils import cl_utils import cv2 import numpy as np import magic @@ -22,7 +22,7 @@ def extract_text_prepResp(file, content_type=""): content_type = magic.from_file(file.stream.name, mime=True) if tika_responding(): - response = requests.put(tika_server, data=file) + response = requests.put(tika_server, data=file, headers={"Accept": "text/plain; charset=UTF-8"}) return response.text, 200 if "openxmlformats-officedocument.wordprocessingml.document" in content_type: content = '\n'.join([p.text for p in docx.Document(file).paragraphs]) @@ -46,7 +46,7 @@ def extract_text_prepResp(file, content_type=""): def ocr_text_prepResp(file): if tika_responding(): response = requests.put(tika_server, data=file, - headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng"}) + headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng","Accept": "text/plain; charset=UTF-8"}) return response.text, 200 win_p = "C:/Program Files/Tesseract-OCR/tesseract.exe"