tika update

swagger_server/controllers/extract_controller.py

@@ -2,7 +2,7 @@ import connexion
 import json
 from pathlib import Path
 from swagger_server.models.izlusci_body import IzlusciBody  # noqa: E501
-from swagger_server.classla import cl_utils
+from swagger_server.utils import cl_utils
 import requests
 
 # ATEapi_endpoint = "http://localhost:5000/predict"

swagger_server/controllers/jobs_controller.py

@@ -7,7 +7,7 @@ import concurrent.futures as cf
 from swagger_server.models.job_response import JobResponse  # noqa: E501
 from swagger_server.requets_db.models.vrsta import (Job)
 from threading import Thread
-from swagger_server.classla import cl_utils
+from swagger_server.utils import cl_utils
 from swagger_server.utils import txt_utils
 from werkzeug.datastructures import FileStorage
 import threading

swagger_server/controllers/marktext_sync_controller.py

@@ -1,4 +1,4 @@
-from swagger_server.classla import cl_utils
+from swagger_server.utils import cl_utils
 from swagger_server.utils import txt_utils
 
 
@@ -38,6 +38,7 @@ def datoteka_v_besedilo_sync_post(file=None):  # noqa: E501
     try:
         return txt_utils.extract_text_prepResp(file)
     except Exception as e:
+        print(e)
         return str(e), 500
 
 

swagger_server/utils/cl_utils.py

@@ -0,0 +1,34 @@
+import classla
+import time
+from swagger_server import util
+from pathlib import Path
+import re
+
+nlp_loaded = False
+nlpSlo = classla.Pipeline('sl', processors='tokenize,ner,pos,lemma,depparse')
+nlp_loaded = True
+
+sent_extractor = re.compile(r"# sent_id = \d+\.\d+(.*?)\n\n", re.MULTILINE | re.DOTALL)
+
+def raw_text_to_conllu(text):
+    try:
+        docall = nlpSlo(text)
+        docallconllu = docall.to_conll()
+
+        return docallconllu, 200
+    except Exception as e:
+        return e, 400
+
+
+def multipla_conllus_to_one_from_file_ids(list_file_ids):
+    sent_cnt = 1
+    ret = "# newpar id = 1\n"
+    files = [f'{util.get_conllu_file_path_by_id(i)}' for i in list_file_ids]
+    for file in files:
+        txt = Path(file).read_text('utf-8')
+        matches = sent_extractor.finditer(txt)
+        for match in matches:
+            ret += f'# sent_id = 1.{sent_cnt}{match.group(1)}\n\n'
+            sent_cnt += 1
+
+    return ret

swagger_server/utils/txt_utils.py

@@ -5,7 +5,7 @@ import requests
 import docx
 import xml.etree.ElementTree as ET
 from PyPDF2 import PdfReader
-from swagger_server.classla import cl_utils
+from swagger_server.utils import cl_utils
 import cv2
 import numpy as np
 import magic
@@ -22,7 +22,7 @@ def extract_text_prepResp(file, content_type=""):
         content_type = magic.from_file(file.stream.name, mime=True)
 
     if tika_responding():
-        response = requests.put(tika_server, data=file)
+        response = requests.put(tika_server, data=file, headers={"Accept": "text/plain; charset=UTF-8"})
         return response.text, 200
     if "openxmlformats-officedocument.wordprocessingml.document" in content_type:
         content = '\n'.join([p.text for p in docx.Document(file).paragraphs])
@@ -46,7 +46,7 @@ def extract_text_prepResp(file, content_type=""):
 def ocr_text_prepResp(file):
     if tika_responding():
         response = requests.put(tika_server, data=file,
-                                headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng"})
+                                headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng","Accept": "text/plain; charset=UTF-8"})
         return response.text, 200
 
     win_p = "C:/Program Files/Tesseract-OCR/tesseract.exe"