tika update

ATEapi/Dockerfile

@@ -19,6 +19,9 @@ RUN python -m pip install -r requirements.txt
 WORKDIR /app
 COPY . /app
 
+
+ADD https://kt-cloud.ijs.si/index.php/s/T4qtSKxbxgqr6c5/download/pytorch_model.bin ./model/term_extractor/
+
 # Creates a non-root user with an explicit UID and adds permission to access the /app folder
 # For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers
 RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /app

Dockerfile

@@ -1,12 +1,11 @@
 FROM python:3.9
 
-RUN mkdir -p /usr/src/app
-WORKDIR /usr/src/app
 
 RUN apt-get update && apt-get install python3-pil tesseract-ocr libtesseract-dev tesseract-ocr-eng tesseract-ocr-slv tesseract-ocr-script-latn ffmpeg libsm6 libxext6 libgl1 libmagic1 -y
 
+RUN mkdir -p /usr/src/app
+WORKDIR /usr/src/app
 COPY requirements.txt /usr/src/app/
-
 # RUN pip install --upgrade pip
 RUN pip3 install --no-cache-dir -r requirements.txt
 

docker-compose.yml

@@ -8,7 +8,7 @@ services:
       - "8080:8080"
     volumes: # when pushing to server, remove the first dot here in front of /mnt
       - /mnt/ssd/ds_ftp/classla_OS2022:/usr/src/app/classla_OS2022
-      - ./swagger_server/classla/classla_resources:/root/classla_resources
+      - ../classla/classla_resources:/root/classla_resources
     environment:
       - PYTHONUNBUFFERED=1
   tika2:
@@ -20,7 +20,7 @@ services:
   ate-api:
     build: ./ATEapi
     restart: always
-    ports:
-      - "5000:5000"
+ #   ports:
+    #  - "5000:5000"
     volumes:
-      - ./swagger_server/classla/classla_resources:/home/appuser/classla_resources
+      - ../classla/classla_resources:/home/appuser/classla_resources

swagger_server/controllers/extract_controller.py

@@ -2,7 +2,7 @@ import connexion
 import json
 from pathlib import Path
 from swagger_server.models.izlusci_body import IzlusciBody  # noqa: E501
-from swagger_server.classla import cl_utils
+from swagger_server.utils import cl_utils
 import requests
 
 # ATEapi_endpoint = "http://localhost:5000/predict"

swagger_server/controllers/jobs_controller.py

@@ -7,7 +7,7 @@ import concurrent.futures as cf
 from swagger_server.models.job_response import JobResponse  # noqa: E501
 from swagger_server.requets_db.models.vrsta import (Job)
 from threading import Thread
-from swagger_server.classla import cl_utils
+from swagger_server.utils import cl_utils
 from swagger_server.utils import txt_utils
 from werkzeug.datastructures import FileStorage
 import threading

swagger_server/controllers/marktext_sync_controller.py

@@ -1,4 +1,4 @@
-from swagger_server.classla import cl_utils
+from swagger_server.utils import cl_utils
 from swagger_server.utils import txt_utils
 
 
@@ -38,6 +38,7 @@ def datoteka_v_besedilo_sync_post(file=None):  # noqa: E501
     try:
         return txt_utils.extract_text_prepResp(file)
     except Exception as e:
+        print(e)
         return str(e), 500
 
 

swagger_server/utils/cl_utils.py

@@ -0,0 +1,34 @@
+import classla
+import time
+from swagger_server import util
+from pathlib import Path
+import re
+
+nlp_loaded = False
+nlpSlo = classla.Pipeline('sl', processors='tokenize,ner,pos,lemma,depparse')
+nlp_loaded = True
+
+sent_extractor = re.compile(r"# sent_id = \d+\.\d+(.*?)\n\n", re.MULTILINE | re.DOTALL)
+
+def raw_text_to_conllu(text):
+    try:
+        docall = nlpSlo(text)
+        docallconllu = docall.to_conll()
+
+        return docallconllu, 200
+    except Exception as e:
+        return e, 400
+
+
+def multipla_conllus_to_one_from_file_ids(list_file_ids):
+    sent_cnt = 1
+    ret = "# newpar id = 1\n"
+    files = [f'{util.get_conllu_file_path_by_id(i)}' for i in list_file_ids]
+    for file in files:
+        txt = Path(file).read_text('utf-8')
+        matches = sent_extractor.finditer(txt)
+        for match in matches:
+            ret += f'# sent_id = 1.{sent_cnt}{match.group(1)}\n\n'
+            sent_cnt += 1
+
+    return ret

swagger_server/utils/txt_utils.py

@@ -5,7 +5,7 @@ import requests
 import docx
 import xml.etree.ElementTree as ET
 from PyPDF2 import PdfReader
-from swagger_server.classla import cl_utils
+from swagger_server.utils import cl_utils
 import cv2
 import numpy as np
 import magic
@@ -22,7 +22,7 @@ def extract_text_prepResp(file, content_type=""):
         content_type = magic.from_file(file.stream.name, mime=True)
 
     if tika_responding():
-        response = requests.put(tika_server, data=file)
+        response = requests.put(tika_server, data=file, headers={"Accept": "text/plain; charset=UTF-8"})
         return response.text, 200
     if "openxmlformats-officedocument.wordprocessingml.document" in content_type:
         content = '\n'.join([p.text for p in docx.Document(file).paragraphs])
@@ -46,7 +46,7 @@ def extract_text_prepResp(file, content_type=""):
 def ocr_text_prepResp(file):
     if tika_responding():
         response = requests.put(tika_server, data=file,
-                                headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng"})
+                                headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng","Accept": "text/plain; charset=UTF-8"})
         return response.text, 200
 
     win_p = "C:/Program Files/Tesseract-OCR/tesseract.exe"