tika update

swagger_server/utils/cl_utils.py

@@ -0,0 +1,34 @@
+import classla
+import time
+from swagger_server import util
+from pathlib import Path
+import re
+
+nlp_loaded = False
+nlpSlo = classla.Pipeline('sl', processors='tokenize,ner,pos,lemma,depparse')
+nlp_loaded = True
+
+sent_extractor = re.compile(r"# sent_id = \d+\.\d+(.*?)\n\n", re.MULTILINE | re.DOTALL)
+
+def raw_text_to_conllu(text):
+    try:
+        docall = nlpSlo(text)
+        docallconllu = docall.to_conll()
+
+        return docallconllu, 200
+    except Exception as e:
+        return e, 400
+
+
+def multipla_conllus_to_one_from_file_ids(list_file_ids):
+    sent_cnt = 1
+    ret = "# newpar id = 1\n"
+    files = [f'{util.get_conllu_file_path_by_id(i)}' for i in list_file_ids]
+    for file in files:
+        txt = Path(file).read_text('utf-8')
+        matches = sent_extractor.finditer(txt)
+        for match in matches:
+            ret += f'# sent_id = 1.{sent_cnt}{match.group(1)}\n\n'
+            sent_cnt += 1
+
+    return ret

swagger_server/utils/txt_utils.py

@@ -5,7 +5,7 @@ import requests
 import docx
 import xml.etree.ElementTree as ET
 from PyPDF2 import PdfReader
-from swagger_server.classla import cl_utils
+from swagger_server.utils import cl_utils
 import cv2
 import numpy as np
 import magic
@@ -22,7 +22,7 @@ def extract_text_prepResp(file, content_type=""):
         content_type = magic.from_file(file.stream.name, mime=True)
 
     if tika_responding():
-        response = requests.put(tika_server, data=file)
+        response = requests.put(tika_server, data=file, headers={"Accept": "text/plain; charset=UTF-8"})
         return response.text, 200
     if "openxmlformats-officedocument.wordprocessingml.document" in content_type:
         content = '\n'.join([p.text for p in docx.Document(file).paragraphs])
@@ -46,7 +46,7 @@ def extract_text_prepResp(file, content_type=""):
 def ocr_text_prepResp(file):
     if tika_responding():
         response = requests.put(tika_server, data=file,
-                                headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng"})
+                                headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng","Accept": "text/plain; charset=UTF-8"})
         return response.text, 200
 
     win_p = "C:/Program Files/Tesseract-OCR/tesseract.exe"