webapps/term_extractor
18
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updated endpoint definitions/names and reformatted some files and code

Browse Source
This commit is contained in:
Kikimanox
2022-09-14 16:40:29 +02:00
parent 0b90b6555b
commit 9e82f3b93a
32 changed files with 988 additions and 742 deletions
Download Patch File Download Diff File Expand all files Collapse all files
swagger_server/controllers/doc2text_controller.py
-157
View File
@@ -1,157 +0,0 @@
import os.path
from tempfile import TemporaryFile
import connexion
import pytesseract
import six
import requests
import json
from swagger_server import util
import pandas as pd
import docx
import codecs
import xml.etree.ElementTree as ET
from PyPDF2 import PdfReader
from swagger_server.classla import cl_utils
import traceback
import cv2
import numpy as np
tika_server = "http://tika2:9999/tika"
# endpoint below to be used only for development purposes (don't need to run docker)
# tika_server = "http://rsdo.lhrs.feri.um.si:9998/tika"
def extract_text_prepResp(file):
if tika_responding():
response = requests.put(tika_server, data=file)
return response.text, 200
if "openxmlformats-officedocument.wordprocessingml.document" in file.content_type:
content = [p.text for p in docx.Document(file).paragraphs]
elif "application/pdf" in file.content_type:
reader = PdfReader(file)
content = '\n'.join([p.extract_text() for p in reader.pages])
content = "ZACASNO UPORABLJEN DRUGI BRALEC KOT TIKA, TA BO SE DODANA KASNEJE...\n\n" + content
elif "text/xml" in file.content_type:
root = ET.parse(file).getroot()
plainText = root.findall('PlainText')
if len(plainText) == 0:
return "Didn't find anything in PlainText", 400
content = '\n'.join([pt.text for pt in plainText])
# elif "text/plain" in file.content_type:
else:
content = file.read().decode('utf-8')
return content, 200
def ocr_text_prepResp(file):
if tika_responding():
response = requests.put(tika_server, data=file,
headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng"})
return response.text, 200
win_p = "C:/Program Files/Tesseract-OCR/tesseract.exe"
if os.path.exists(win_p):
pytesseract.pytesseract.tesseract_cmd = win_p
# convert string data to numpy array
file_bytes = np.fromstring(file.read(), np.uint8)
# convert numpy array to image
img = cv2.imdecode(file_bytes, cv2.IMREAD_COLOR)
conf = '-l eng+slv'
return pytesseract.image_to_string(img, config=conf), 200
def tika_responding():
try:
ret = requests.get(tika_server)
return ret.status_code == 200
except:
return False
def datoteka_v_besedilo_post(file=None): # noqa: E501
"""Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vraca besedilo
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
if file is None:
return "No file provided", 400
try:
return extract_text_prepResp(file)
except Exception as e:
return str(e), 500
def get_text_ocr(file=None): # noqa: E501
"""Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo s pomočjo ocr razpoznavanja
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
if file is None:
return "No file provided", 400
try:
return ocr_text_prepResp(file)
except Exception as e:
return str(e), 500
def datoteka_v_besedilo_in_classla(file=None): # noqa: E501
"""Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vraca conllu
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
if not cl_utils.nlp_loaded:
return "NLP Models still loading up since server restart, please retry later.", 500
if file is None:
return "No file provided", 400
try:
txt, _ = extract_text_prepResp(file)
return cl_utils.raw_text_to_conllu(txt)
except Exception as e:
return str(e), 500
def get_conllu_ocr(file=None): # noqa: E501
"""Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v conllu s pomočjo ocr razpoznavanja
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
if not cl_utils.nlp_loaded:
return "NLP Models still loading up since server restart, please retry later.", 500
if file is None:
return "No file provided", 400
try:
if tika_responding():
response = requests.put(tika_server, data=file,
headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng"})
return cl_utils.raw_text_to_conllu(response.text)
else:
txt, _ = ocr_text_prepResp(file)
return cl_utils.raw_text_to_conllu(txt)
except Exception as e:
return str(e), 500
swagger_server/controllers/extract_controller.py
-3
View File
@@ -1,9 +1,6 @@
import connexion
import six
from swagger_server.models.izlusci_body import IzlusciBody # noqa: E501
from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat # noqa: E501
from swagger_server import util
def get_candidates(body): # noqa: E501
swagger_server/controllers/jobs_controller.py
+29 -17
View File
@@ -1,41 +1,51 @@
import datetime
import random
import connexion
import peewee
import six
import asyncio
from swagger_server.models.job_response import JobResponse # noqa: E501
from swagger_server import util
from swagger_server.requets_db.models.vrsta import (Job, JobManager)
from threading import Semaphore, Thread
from swagger_server.requets_db.models.vrsta import (Job)
from threading import Thread
from swagger_server.classla import cl_utils
CLASSLA_CONCURANCE_LIMIT = 4
classla_sem = asyncio.Semaphore(CLASSLA_CONCURANCE_LIMIT)
def get_job_status(job_id, show_estimated_completion=None): # noqa: E501
def delete_job(job_id): # noqa: E501
"""Izbriše job
# noqa: E501
:param job_id:
:type job_id: int
:rtype: str
"""
return 'Endpoint currently disabled'
def get_job_status(job_id): # noqa: E501
"""Vrne status
# noqa: E501
:param job_id:
:type job_id: int
:param show_estimated_completion: Calculate estimate time remaining based on various factors (could be inaccurate)
:type show_estimated_completion: bool
:rtype: JobResponse
"""
try:
job = Job.get_by_id(job_id)
if not job.finished_on:
est_com = None
# todo: if estimate completion: calculate it and set it to est_com
return JobResponse(finished_job=False, estimated_completion=est_com), 200
return JobResponse(finished_job=True, completed_at=job.finished_on, job_result=job.job_output), 200
except peewee.DoesNotExist as e:
if job.started_on is None:
return JobResponse(job_status="waiting in que", created_on=job.created_on), 200
if job.started_on is not None and job.finished_on is None:
return JobResponse(job_status="currently processing", created_on=job.created_on,
started_on=job.started_on), 200
if job.started_on is not None and job.finished_on is not None:
return JobResponse(job_status="finished processing", created_on=job.created_on, started_on=job.started_on,
finished_on=job.finished_on, job_result=job.job_output), 200
except peewee.DoesNotExist:
return "Job with this ID does not exist", 404
@@ -43,6 +53,7 @@ def clear_up_unfinished_jobs():
"""
In case server crashed while jobs were in queue...
"""
# tu more but !=, ne deluje ce je "is not"
Job.update(started_on=None).where(Job.started_on != None, Job.finished_on == None).execute()
@@ -56,7 +67,7 @@ async def try_do_jobs():
if classla_sem._value > 0:
# classla
unfinished_jobs = Job.select() \
.where(Job.finished_on == None, Job.started_on == None, Job.job_type == 1) \
.where(Job.finished_on == None, Job.started_on == None, Job.job_type == 2) \
.limit(classla_sem._value)
tasks = [
asyncio.ensure_future(execute_classla_job(job))
@@ -91,6 +102,8 @@ async def execute_classla_job(job: Job):
clear_up_unfinished_jobs()
loop = asyncio.get_event_loop()
def loop_in_thread(loop):
asyncio.set_event_loop(loop)
loop.run_until_complete(try_do_jobs())
@@ -99,7 +112,6 @@ def loop_in_thread(loop):
t = Thread(target=loop_in_thread, args=(loop,))
t.start()
# clear_up_unfinished_jobs()
# loop = asyncio.get_event_loop()
# loop.run_until_complete(try_do_jobs()) this version seems more at home, but it blocks the thread, fix that?
swagger_server/controllers/marktext_async_controller.py
+79
View File
@@ -0,0 +1,79 @@
import connexion
from swagger_server.models.oznaci_besedilo_async_body import OznaciBesediloAsyncBody # noqa: E501
from swagger_server.requets_db.models.vrsta import (JobManager)
from swagger_server.utils import txt_utils
def get_text(body): # noqa: E501
"""Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
# noqa: E501
:param body:
:type body: dict | bytes
:rtype: str
"""
if connexion.request.is_json:
body = OznaciBesediloAsyncBody.from_dict(connexion.request.get_json()) # noqa: E501
# conllu = cl_utils.raw_text_to_conllu(body.besedilo)
# return conllu
job, is_old_job = JobManager.create_job(2, body.besedilo)
if job is None:
return "Something went wrong", 500
ret = {'check_job_url': f'{connexion.request.url_root}/job/{job.id}'}
return ret, 200 # Todo: Update swagger to the newest response template later
def get_conllu_from_file_async(file=None): # noqa: E501
"""Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
return 'do some magic!'
def get_conllu_from_file_ocr_async(file=None): # noqa: E501
"""Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
return 'do some magic!'
def get_text_from_doc_async(file=None): # noqa: E501
"""Pretvori datoteko v besedilo, vrača tekst
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
return 'do some magic!'
def get_text_from_file_ocr_async(file=None): # noqa: E501
"""Pretvori datoteko v besedilo s pomočjo ocr razpoznavanja, vrača tekst
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
return 'do some magic!'
swagger_server/controllers/marktext_controller.py
-82
View File
@@ -1,82 +0,0 @@
import connexion
import six
from swagger_server.models.oznaci_besedilo_async_body import OznaciBesediloAsyncBody # noqa: E501
from swagger_server import util
from swagger_server.classla import cl_utils
from swagger_server.requets_db.models.vrsta import (Job, JobManager)
import swagger_server.controllers.doc2text_controller as d2t
def get_text(body): # noqa: E501
"""Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
# noqa: E501
:param body:
:type body: dict | bytes
:rtype: str
"""
if connexion.request.is_json:
body = OznaciBesediloAsyncBody.from_dict(connexion.request.get_json()) # noqa: E501
# conllu = cl_utils.raw_text_to_conllu(body.besedilo)
# return conllu
job, is_old_job = JobManager.create_job(1, body.besedilo)
if job is None:
return "Something went wrong", 500
ret = {'check_job_url': f'{connexion.request.url_root}/job/{job.id}'}
return ret, 200 # Todo: Update swagger to the newest response template later
def get_text_from_file(file=None): # noqa: E501
"""Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
if file is None:
return "No file provided", 400
txt, status = d2t.extract_text_prepResp(file)
# Todo: instead of parsing text here, instead save the file into the tb or locally, and then parsing
# Todo: when the job actually executes (this version of the implementation is temporary)
if status == 200:
job, is_old_job = JobManager.create_job(1, txt)
if job is None:
return "Something went wrong", 500
ret = {'check_job_url': f'{connexion.request.url_root}/job/{job.id}'}
return ret, 200
else:
return "Something went wrong", 500
def get_text_from_file_ocr(file=None): # noqa: E501
"""Pretvori datoteko v besedilo s pomočjo ocr razpoznavanja in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
if file is None:
return "No file provided", 400
txt, status = d2t.ocr_text_prepResp(file)
# Todo: instead of ocr-ing text here, instead save the file into the tb or locally, and then ocr
# Todo: when the job actually executes (this version of the implementation is temporary)
if status == 200:
job, is_old_job = JobManager.create_job(1, txt)
if job is None:
return "Something went wrong", 500
ret = {'check_job_url': f'{connexion.request.url_root}/job/{job.id}'}
return ret, 200
else:
return "Something went wrong", 500
swagger_server/controllers/marktext_sync_controller.py
+80
View File
@@ -0,0 +1,80 @@
from swagger_server.classla import cl_utils
from swagger_server.utils import txt_utils
def datoteka_v_besedilo_in_classla(file=None): # noqa: E501
"""Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vrača conllu
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
if not cl_utils.nlp_loaded:
return "NLP Models still loading up since server restart, please try again later.", 500
if file is None:
return "No file provided", 400
try:
txt, _ = txt_utils.extract_text_prepResp(file)
return cl_utils.raw_text_to_conllu(txt)
except Exception as e:
return str(e), 500
def datoteka_v_besedilo_sync_post(file=None): # noqa: E501
"""Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vrača besedilo
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
if file is None:
return "No file provided", 400
try:
return txt_utils.extract_text_prepResp(file)
except Exception as e:
return str(e), 500
def get_conllu_ocr(file=None): # noqa: E501
"""Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v conllu s pomočjo ocr razpoznavanja
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
if not cl_utils.nlp_loaded:
return "NLP Models still loading up since server restart, please try again later.", 500
if file is None:
return "No file provided", 400
try:
txt, _ = txt_utils.ocr_text_prepResp(file)
return cl_utils.raw_text_to_conllu(txt)
except Exception as e:
return str(e), 500
def get_text_ocr(file=None): # noqa: E501
"""Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo s pomočjo ocr razpoznavanja
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
if file is None:
return "No file provided", 400
try:
return txt_utils.ocr_text_prepResp(file)
except Exception as e:
return str(e), 500
swagger_server/controllers/oss_controller.py
-6
View File
@@ -1,11 +1,5 @@
import connexion
import six
import os
from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat # noqa: E501
from swagger_server import util
from flask import send_file
from swagger_server.db_utils import Ngrams_Manager
def get_conllus(leta, vrste, kljucnebesede, cerifpodrocja): # noqa: E501