diff --git a/rsdo5.json b/rsdo5.json index e352680..a260847 100644 --- a/rsdo5.json +++ b/rsdo5.json @@ -27,14 +27,6 @@ "type": "integer", "format": "int64" } - }, - { - "name": "show_estimated_completion", - "in": "query", - "schema": { - "type": "boolean" - }, - "description": "Calculate estimate time remaining based on various factors (could be inaccurate)" } ], "responses": { @@ -49,14 +41,44 @@ } } } + }, + "delete": { + "tags": [ + "jobs" + ], + "summary": "Izbriše job", + "operationId": "deleteJob", + "parameters": [ + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "string" + } + } + } + } + } } }, "/oznaciBesediloAsync": { "post": { "tags": [ - "marktext" + "marktext-async" ], - "summary": "Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format", + "summary": "Označi surovo (angl. raw) besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format", "operationId": "getText", "requestBody": { "content": { @@ -87,13 +109,13 @@ } } }, - "/pretvoriDatotekoInOznaciAsync": { + "/datotekaVBesediloAsync": { "post": { "tags": [ - "marktext" + "marktext-async" ], - "summary": "Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format", - "operationId": "getTextFromFile", + "summary": "Pretvori datoteko v besedilo, vrača tekst", + "operationId": "getTextFromDocAsync", "requestBody": { "content": { "multipart/form-data": { @@ -126,13 +148,91 @@ } } }, - "/pretvoriDatotekoInOznaciAsync/ocr": { + "/datotekaVConlluAsync": { "post": { "tags": [ - "marktext" + "marktext-async" ], - "summary": "Pretvori datoteko v besedilo s pomočjo ocr razpoznavanja in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format", - "operationId": "getTextFromFileOcr", + "summary": "Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format", + "operationId": "getConlluFromFileAsync", + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "required": [ + "file" + ], + "type": "object", + "properties": { + "file": { + "type": "string", + "format": "binary" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/datotekaVConlluAsync/ocr": { + "post": { + "tags": [ + "marktext-async" + ], + "summary": "Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format", + "operationId": "getConlluFromFileOcrAsync", + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "required": [ + "file" + ], + "type": "object", + "properties": { + "file": { + "type": "string", + "format": "binary" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/datotekaVBesediloAsync/ocr": { + "post": { + "tags": [ + "marktext-async" + ], + "summary": "Pretvori datoteko v besedilo s pomočjo ocr razpoznavanja, vrača tekst", + "operationId": "getTextFromFileOcrAsync", "requestBody": { "content": { "multipart/form-data": { @@ -213,12 +313,12 @@ } } }, - "/datotekaVBesedilo": { + "/datotekaVBesediloSync": { "post": { "tags": [ - "doc-2text" + "marktext-sync" ], - "summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vraca besedilo", + "summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vrača besedilo", "requestBody": { "content": { "multipart/form-data": { @@ -251,10 +351,10 @@ } } }, - "/datotekaVBesedilo/ocr": { + "/datotekaVBesediloSync/ocr": { "post": { "tags": [ - "doc-2text" + "marktext-sync" ], "summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo s pomočjo ocr razpoznavanja", "operationId": "getTextOcr", @@ -293,9 +393,9 @@ "/datotekaVConlluSync": { "post": { "tags": [ - "doc-2text" + "marktext-sync" ], - "summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vraca conllu", + "summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vrača conllu", "operationId": "datotekaVBesediloInClassla", "requestBody": { "content": { @@ -332,7 +432,7 @@ "/datotekaVConlluSync/ocr": { "post": { "tags": [ - "doc-2text" + "marktext-sync" ], "summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v conllu s pomočjo ocr razpoznavanja", "operationId": "getConlluOcr", @@ -867,14 +967,19 @@ "JobResponse": { "type": "object", "properties": { - "finished_job": { - "type": "boolean" + "job_status": { + "type": "string", + "enum": ["waiting in que", "currently processing", "finished processing"] }, - "completed_at": { + "finished_on": { "type": "string", "format": "date-time" }, - "estimated_completion": { + "started_on": { + "type": "string", + "format": "date-time" + }, + "created_on": { "type": "string", "format": "date-time" }, diff --git a/swagger_server/controllers/doc2text_controller.py b/swagger_server/controllers/doc2text_controller.py deleted file mode 100644 index 797279a..0000000 --- a/swagger_server/controllers/doc2text_controller.py +++ /dev/null @@ -1,157 +0,0 @@ -import os.path -from tempfile import TemporaryFile - -import connexion -import pytesseract -import six -import requests -import json -from swagger_server import util -import pandas as pd -import docx -import codecs -import xml.etree.ElementTree as ET -from PyPDF2 import PdfReader -from swagger_server.classla import cl_utils -import traceback -import cv2 -import numpy as np - -tika_server = "http://tika2:9999/tika" - - -# endpoint below to be used only for development purposes (don't need to run docker) -# tika_server = "http://rsdo.lhrs.feri.um.si:9998/tika" - - -def extract_text_prepResp(file): - if tika_responding(): - response = requests.put(tika_server, data=file) - return response.text, 200 - if "openxmlformats-officedocument.wordprocessingml.document" in file.content_type: - content = [p.text for p in docx.Document(file).paragraphs] - elif "application/pdf" in file.content_type: - reader = PdfReader(file) - content = '\n'.join([p.extract_text() for p in reader.pages]) - content = "ZACASNO UPORABLJEN DRUGI BRALEC KOT TIKA, TA BO SE DODANA KASNEJE...\n\n" + content - elif "text/xml" in file.content_type: - root = ET.parse(file).getroot() - plainText = root.findall('PlainText') - if len(plainText) == 0: - return "Didn't find anything in PlainText", 400 - content = '\n'.join([pt.text for pt in plainText]) - # elif "text/plain" in file.content_type: - else: - content = file.read().decode('utf-8') - - return content, 200 - - -def ocr_text_prepResp(file): - if tika_responding(): - response = requests.put(tika_server, data=file, - headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng"}) - return response.text, 200 - - win_p = "C:/Program Files/Tesseract-OCR/tesseract.exe" - if os.path.exists(win_p): - pytesseract.pytesseract.tesseract_cmd = win_p - - # convert string data to numpy array - file_bytes = np.fromstring(file.read(), np.uint8) - # convert numpy array to image - img = cv2.imdecode(file_bytes, cv2.IMREAD_COLOR) - - conf = '-l eng+slv' - return pytesseract.image_to_string(img, config=conf), 200 - - -def tika_responding(): - try: - ret = requests.get(tika_server) - return ret.status_code == 200 - except: - return False - - -def datoteka_v_besedilo_post(file=None): # noqa: E501 - """Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vraca besedilo - - # noqa: E501 - - :param file: - :type file: strstr - - :rtype: str - """ - if file is None: - return "No file provided", 400 - try: - return extract_text_prepResp(file) - except Exception as e: - return str(e), 500 - - -def get_text_ocr(file=None): # noqa: E501 - """Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo s pomočjo ocr razpoznavanja - - # noqa: E501 - - :param file: - :type file: strstr - - :rtype: str - """ - if file is None: - return "No file provided", 400 - try: - return ocr_text_prepResp(file) - except Exception as e: - return str(e), 500 - - -def datoteka_v_besedilo_in_classla(file=None): # noqa: E501 - """Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vraca conllu - - # noqa: E501 - - :param file: - :type file: strstr - - :rtype: str - """ - if not cl_utils.nlp_loaded: - return "NLP Models still loading up since server restart, please retry later.", 500 - if file is None: - return "No file provided", 400 - try: - txt, _ = extract_text_prepResp(file) - return cl_utils.raw_text_to_conllu(txt) - except Exception as e: - return str(e), 500 - - -def get_conllu_ocr(file=None): # noqa: E501 - """Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v conllu s pomočjo ocr razpoznavanja - - # noqa: E501 - - :param file: - :type file: strstr - - :rtype: str - """ - if not cl_utils.nlp_loaded: - return "NLP Models still loading up since server restart, please retry later.", 500 - if file is None: - return "No file provided", 400 - try: - if tika_responding(): - response = requests.put(tika_server, data=file, - headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng"}) - return cl_utils.raw_text_to_conllu(response.text) - else: - txt, _ = ocr_text_prepResp(file) - return cl_utils.raw_text_to_conllu(txt) - except Exception as e: - return str(e), 500 diff --git a/swagger_server/controllers/extract_controller.py b/swagger_server/controllers/extract_controller.py index fbe9849..006c385 100644 --- a/swagger_server/controllers/extract_controller.py +++ b/swagger_server/controllers/extract_controller.py @@ -1,9 +1,6 @@ import connexion -import six from swagger_server.models.izlusci_body import IzlusciBody # noqa: E501 -from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat # noqa: E501 -from swagger_server import util def get_candidates(body): # noqa: E501 diff --git a/swagger_server/controllers/jobs_controller.py b/swagger_server/controllers/jobs_controller.py index 833a8a2..1afc869 100644 --- a/swagger_server/controllers/jobs_controller.py +++ b/swagger_server/controllers/jobs_controller.py @@ -1,41 +1,51 @@ import datetime -import random -import connexion import peewee -import six import asyncio from swagger_server.models.job_response import JobResponse # noqa: E501 -from swagger_server import util -from swagger_server.requets_db.models.vrsta import (Job, JobManager) -from threading import Semaphore, Thread +from swagger_server.requets_db.models.vrsta import (Job) +from threading import Thread from swagger_server.classla import cl_utils CLASSLA_CONCURANCE_LIMIT = 4 classla_sem = asyncio.Semaphore(CLASSLA_CONCURANCE_LIMIT) -def get_job_status(job_id, show_estimated_completion=None): # noqa: E501 +def delete_job(job_id): # noqa: E501 + """Izbriše job + + # noqa: E501 + + :param job_id: + :type job_id: int + + :rtype: str + """ + return 'Endpoint currently disabled' + + +def get_job_status(job_id): # noqa: E501 """Vrne status # noqa: E501 :param job_id: :type job_id: int - :param show_estimated_completion: Calculate estimate time remaining based on various factors (could be inaccurate) - :type show_estimated_completion: bool :rtype: JobResponse """ try: job = Job.get_by_id(job_id) - if not job.finished_on: - est_com = None - # todo: if estimate completion: calculate it and set it to est_com - return JobResponse(finished_job=False, estimated_completion=est_com), 200 - return JobResponse(finished_job=True, completed_at=job.finished_on, job_result=job.job_output), 200 - except peewee.DoesNotExist as e: + if job.started_on is None: + return JobResponse(job_status="waiting in que", created_on=job.created_on), 200 + if job.started_on is not None and job.finished_on is None: + return JobResponse(job_status="currently processing", created_on=job.created_on, + started_on=job.started_on), 200 + if job.started_on is not None and job.finished_on is not None: + return JobResponse(job_status="finished processing", created_on=job.created_on, started_on=job.started_on, + finished_on=job.finished_on, job_result=job.job_output), 200 + except peewee.DoesNotExist: return "Job with this ID does not exist", 404 @@ -43,6 +53,7 @@ def clear_up_unfinished_jobs(): """ In case server crashed while jobs were in queue... """ + # tu more but !=, ne deluje ce je "is not" Job.update(started_on=None).where(Job.started_on != None, Job.finished_on == None).execute() @@ -56,7 +67,7 @@ async def try_do_jobs(): if classla_sem._value > 0: # classla unfinished_jobs = Job.select() \ - .where(Job.finished_on == None, Job.started_on == None, Job.job_type == 1) \ + .where(Job.finished_on == None, Job.started_on == None, Job.job_type == 2) \ .limit(classla_sem._value) tasks = [ asyncio.ensure_future(execute_classla_job(job)) @@ -91,6 +102,8 @@ async def execute_classla_job(job: Job): clear_up_unfinished_jobs() loop = asyncio.get_event_loop() + + def loop_in_thread(loop): asyncio.set_event_loop(loop) loop.run_until_complete(try_do_jobs()) @@ -99,7 +112,6 @@ def loop_in_thread(loop): t = Thread(target=loop_in_thread, args=(loop,)) t.start() - # clear_up_unfinished_jobs() # loop = asyncio.get_event_loop() # loop.run_until_complete(try_do_jobs()) this version seems more at home, but it blocks the thread, fix that? diff --git a/swagger_server/controllers/marktext_async_controller.py b/swagger_server/controllers/marktext_async_controller.py new file mode 100644 index 0000000..1cd4efb --- /dev/null +++ b/swagger_server/controllers/marktext_async_controller.py @@ -0,0 +1,79 @@ +import connexion + +from swagger_server.models.oznaci_besedilo_async_body import OznaciBesediloAsyncBody # noqa: E501 +from swagger_server.requets_db.models.vrsta import (JobManager) +from swagger_server.utils import txt_utils + + +def get_text(body): # noqa: E501 + """Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format + + # noqa: E501 + + :param body: + :type body: dict | bytes + + :rtype: str + """ + if connexion.request.is_json: + body = OznaciBesediloAsyncBody.from_dict(connexion.request.get_json()) # noqa: E501 + # conllu = cl_utils.raw_text_to_conllu(body.besedilo) + # return conllu + job, is_old_job = JobManager.create_job(2, body.besedilo) + if job is None: + return "Something went wrong", 500 + ret = {'check_job_url': f'{connexion.request.url_root}/job/{job.id}'} + + return ret, 200 # Todo: Update swagger to the newest response template later + + +def get_conllu_from_file_async(file=None): # noqa: E501 + """Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format + + # noqa: E501 + + :param file: + :type file: strstr + + :rtype: str + """ + return 'do some magic!' + + +def get_conllu_from_file_ocr_async(file=None): # noqa: E501 + """Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format + + # noqa: E501 + + :param file: + :type file: strstr + + :rtype: str + """ + return 'do some magic!' + + +def get_text_from_doc_async(file=None): # noqa: E501 + """Pretvori datoteko v besedilo, vrača tekst + + # noqa: E501 + + :param file: + :type file: strstr + + :rtype: str + """ + return 'do some magic!' + + +def get_text_from_file_ocr_async(file=None): # noqa: E501 + """Pretvori datoteko v besedilo s pomočjo ocr razpoznavanja, vrača tekst + + # noqa: E501 + + :param file: + :type file: strstr + + :rtype: str + """ + return 'do some magic!' diff --git a/swagger_server/controllers/marktext_controller.py b/swagger_server/controllers/marktext_controller.py deleted file mode 100644 index 11d7d23..0000000 --- a/swagger_server/controllers/marktext_controller.py +++ /dev/null @@ -1,82 +0,0 @@ -import connexion -import six - -from swagger_server.models.oznaci_besedilo_async_body import OznaciBesediloAsyncBody # noqa: E501 -from swagger_server import util -from swagger_server.classla import cl_utils -from swagger_server.requets_db.models.vrsta import (Job, JobManager) -import swagger_server.controllers.doc2text_controller as d2t - - -def get_text(body): # noqa: E501 - """Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format - - # noqa: E501 - - :param body: - :type body: dict | bytes - - :rtype: str - """ - if connexion.request.is_json: - body = OznaciBesediloAsyncBody.from_dict(connexion.request.get_json()) # noqa: E501 - # conllu = cl_utils.raw_text_to_conllu(body.besedilo) - # return conllu - job, is_old_job = JobManager.create_job(1, body.besedilo) - if job is None: - return "Something went wrong", 500 - ret = {'check_job_url': f'{connexion.request.url_root}/job/{job.id}'} - - return ret, 200 # Todo: Update swagger to the newest response template later - - -def get_text_from_file(file=None): # noqa: E501 - """Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format - - # noqa: E501 - - :param file: - :type file: strstr - - :rtype: str - """ - if file is None: - return "No file provided", 400 - txt, status = d2t.extract_text_prepResp(file) - # Todo: instead of parsing text here, instead save the file into the tb or locally, and then parsing - # Todo: when the job actually executes (this version of the implementation is temporary) - - if status == 200: - job, is_old_job = JobManager.create_job(1, txt) - if job is None: - return "Something went wrong", 500 - ret = {'check_job_url': f'{connexion.request.url_root}/job/{job.id}'} - return ret, 200 - else: - return "Something went wrong", 500 - - -def get_text_from_file_ocr(file=None): # noqa: E501 - """Pretvori datoteko v besedilo s pomočjo ocr razpoznavanja in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format - - # noqa: E501 - - :param file: - :type file: strstr - - :rtype: str - """ - if file is None: - return "No file provided", 400 - txt, status = d2t.ocr_text_prepResp(file) - # Todo: instead of ocr-ing text here, instead save the file into the tb or locally, and then ocr - # Todo: when the job actually executes (this version of the implementation is temporary) - - if status == 200: - job, is_old_job = JobManager.create_job(1, txt) - if job is None: - return "Something went wrong", 500 - ret = {'check_job_url': f'{connexion.request.url_root}/job/{job.id}'} - return ret, 200 - else: - return "Something went wrong", 500 diff --git a/swagger_server/controllers/marktext_sync_controller.py b/swagger_server/controllers/marktext_sync_controller.py new file mode 100644 index 0000000..bd3531a --- /dev/null +++ b/swagger_server/controllers/marktext_sync_controller.py @@ -0,0 +1,80 @@ +from swagger_server.classla import cl_utils +from swagger_server.utils import txt_utils + + +def datoteka_v_besedilo_in_classla(file=None): # noqa: E501 + """Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vrača conllu + + # noqa: E501 + + :param file: + :type file: strstr + + :rtype: str + """ + if not cl_utils.nlp_loaded: + return "NLP Models still loading up since server restart, please try again later.", 500 + if file is None: + return "No file provided", 400 + try: + txt, _ = txt_utils.extract_text_prepResp(file) + return cl_utils.raw_text_to_conllu(txt) + except Exception as e: + return str(e), 500 + + +def datoteka_v_besedilo_sync_post(file=None): # noqa: E501 + """Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vrača besedilo + + # noqa: E501 + + :param file: + :type file: strstr + + :rtype: str + """ + if file is None: + return "No file provided", 400 + try: + return txt_utils.extract_text_prepResp(file) + except Exception as e: + return str(e), 500 + + +def get_conllu_ocr(file=None): # noqa: E501 + """Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v conllu s pomočjo ocr razpoznavanja + + # noqa: E501 + + :param file: + :type file: strstr + + :rtype: str + """ + if not cl_utils.nlp_loaded: + return "NLP Models still loading up since server restart, please try again later.", 500 + if file is None: + return "No file provided", 400 + try: + txt, _ = txt_utils.ocr_text_prepResp(file) + return cl_utils.raw_text_to_conllu(txt) + except Exception as e: + return str(e), 500 + + +def get_text_ocr(file=None): # noqa: E501 + """Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo s pomočjo ocr razpoznavanja + + # noqa: E501 + + :param file: + :type file: strstr + + :rtype: str + """ + if file is None: + return "No file provided", 400 + try: + return txt_utils.ocr_text_prepResp(file) + except Exception as e: + return str(e), 500 diff --git a/swagger_server/controllers/oss_controller.py b/swagger_server/controllers/oss_controller.py index c141c5d..192a8f2 100644 --- a/swagger_server/controllers/oss_controller.py +++ b/swagger_server/controllers/oss_controller.py @@ -1,11 +1,5 @@ -import connexion -import six -import os - -from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat # noqa: E501 from swagger_server import util from flask import send_file -from swagger_server.db_utils import Ngrams_Manager def get_conllus(leta, vrste, kljucnebesede, cerifpodrocja): # noqa: E501 diff --git a/swagger_server/models/__init__.py b/swagger_server/models/__init__.py index 4adc092..c7bfe56 100644 --- a/swagger_server/models/__init__.py +++ b/swagger_server/models/__init__.py @@ -3,13 +3,15 @@ # flake8: noqa from __future__ import absolute_import # import models into model package -from swagger_server.models.izlusci_body import IzlusciBody -from swagger_server.models.oznaci_besedilo_async_body import OznaciBesediloAsyncBody -from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat -from swagger_server.models.job_response import JobResponse -from swagger_server.models.datoteka_v_besedilo_body import DatotekaVBesediloBody -from swagger_server.models.datoteka_v_besedilo_ocr_body import DatotekaVBesediloOcrBody +from swagger_server.models.datoteka_v_besedilo_async_body import DatotekaVBesediloAsyncBody +from swagger_server.models.datoteka_v_besedilo_async_ocr_body import DatotekaVBesediloAsyncOcrBody +from swagger_server.models.datoteka_v_besedilo_sync_body import DatotekaVBesediloSyncBody +from swagger_server.models.datoteka_v_besedilo_sync_ocr_body import DatotekaVBesediloSyncOcrBody +from swagger_server.models.datoteka_v_conllu_async_body import DatotekaVConlluAsyncBody +from swagger_server.models.datoteka_v_conllu_async_ocr_body import DatotekaVConlluAsyncOcrBody from swagger_server.models.datoteka_v_conllu_sync_body import DatotekaVConlluSyncBody from swagger_server.models.datoteka_v_conllu_sync_ocr_body import DatotekaVConlluSyncOcrBody -from swagger_server.models.pretvori_datoteko_in_oznaci_async_body import PretvoriDatotekoInOznaciAsyncBody -from swagger_server.models.pretvori_datoteko_in_oznaci_async_ocr_body import PretvoriDatotekoInOznaciAsyncOcrBody +from swagger_server.models.izlusci_body import IzlusciBody +from swagger_server.models.job_response import JobResponse +from swagger_server.models.oznaci_besedilo_async_body import OznaciBesediloAsyncBody +from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat diff --git a/swagger_server/models/pretvori_datoteko_in_oznaci_async_ocr_body.py b/swagger_server/models/datoteka_v_besedilo_async_body.py similarity index 59% rename from swagger_server/models/pretvori_datoteko_in_oznaci_async_ocr_body.py rename to swagger_server/models/datoteka_v_besedilo_async_body.py index efb4e5f..bcf9469 100644 --- a/swagger_server/models/pretvori_datoteko_in_oznaci_async_ocr_body.py +++ b/swagger_server/models/datoteka_v_besedilo_async_body.py @@ -9,15 +9,15 @@ from swagger_server.models.base_model_ import Model from swagger_server import util -class PretvoriDatotekoInOznaciAsyncOcrBody(Model): +class DatotekaVBesediloAsyncBody(Model): """NOTE: This class is auto generated by the swagger code generator program. Do not edit the class manually. """ def __init__(self, file: str=None): # noqa: E501 - """PretvoriDatotekoInOznaciAsyncOcrBody - a model defined in Swagger + """DatotekaVBesediloAsyncBody - a model defined in Swagger - :param file: The file of this PretvoriDatotekoInOznaciAsyncOcrBody. # noqa: E501 + :param file: The file of this DatotekaVBesediloAsyncBody. # noqa: E501 :type file: str """ self.swagger_types = { @@ -30,32 +30,32 @@ class PretvoriDatotekoInOznaciAsyncOcrBody(Model): self._file = file @classmethod - def from_dict(cls, dikt) -> 'PretvoriDatotekoInOznaciAsyncOcrBody': + def from_dict(cls, dikt) -> 'DatotekaVBesediloAsyncBody': """Returns the dict as a model :param dikt: A dict. :type: dict - :return: The pretvoriDatotekoInOznaciAsync_ocr_body of this PretvoriDatotekoInOznaciAsyncOcrBody. # noqa: E501 - :rtype: PretvoriDatotekoInOznaciAsyncOcrBody + :return: The datotekaVBesediloAsync_body of this DatotekaVBesediloAsyncBody. # noqa: E501 + :rtype: DatotekaVBesediloAsyncBody """ return util.deserialize_model(dikt, cls) @property def file(self) -> str: - """Gets the file of this PretvoriDatotekoInOznaciAsyncOcrBody. + """Gets the file of this DatotekaVBesediloAsyncBody. - :return: The file of this PretvoriDatotekoInOznaciAsyncOcrBody. + :return: The file of this DatotekaVBesediloAsyncBody. :rtype: str """ return self._file @file.setter def file(self, file: str): - """Sets the file of this PretvoriDatotekoInOznaciAsyncOcrBody. + """Sets the file of this DatotekaVBesediloAsyncBody. - :param file: The file of this PretvoriDatotekoInOznaciAsyncOcrBody. + :param file: The file of this DatotekaVBesediloAsyncBody. :type file: str """ if file is None: diff --git a/swagger_server/models/pretvori_datoteko_in_oznaci_async_body.py b/swagger_server/models/datoteka_v_besedilo_async_ocr_body.py similarity index 61% rename from swagger_server/models/pretvori_datoteko_in_oznaci_async_body.py rename to swagger_server/models/datoteka_v_besedilo_async_ocr_body.py index 93872bf..76e028e 100644 --- a/swagger_server/models/pretvori_datoteko_in_oznaci_async_body.py +++ b/swagger_server/models/datoteka_v_besedilo_async_ocr_body.py @@ -9,15 +9,15 @@ from swagger_server.models.base_model_ import Model from swagger_server import util -class PretvoriDatotekoInOznaciAsyncBody(Model): +class DatotekaVBesediloAsyncOcrBody(Model): """NOTE: This class is auto generated by the swagger code generator program. Do not edit the class manually. """ def __init__(self, file: str=None): # noqa: E501 - """PretvoriDatotekoInOznaciAsyncBody - a model defined in Swagger + """DatotekaVBesediloAsyncOcrBody - a model defined in Swagger - :param file: The file of this PretvoriDatotekoInOznaciAsyncBody. # noqa: E501 + :param file: The file of this DatotekaVBesediloAsyncOcrBody. # noqa: E501 :type file: str """ self.swagger_types = { @@ -30,32 +30,32 @@ class PretvoriDatotekoInOznaciAsyncBody(Model): self._file = file @classmethod - def from_dict(cls, dikt) -> 'PretvoriDatotekoInOznaciAsyncBody': + def from_dict(cls, dikt) -> 'DatotekaVBesediloAsyncOcrBody': """Returns the dict as a model :param dikt: A dict. :type: dict - :return: The pretvoriDatotekoInOznaciAsync_body of this PretvoriDatotekoInOznaciAsyncBody. # noqa: E501 - :rtype: PretvoriDatotekoInOznaciAsyncBody + :return: The datotekaVBesediloAsync_ocr_body of this DatotekaVBesediloAsyncOcrBody. # noqa: E501 + :rtype: DatotekaVBesediloAsyncOcrBody """ return util.deserialize_model(dikt, cls) @property def file(self) -> str: - """Gets the file of this PretvoriDatotekoInOznaciAsyncBody. + """Gets the file of this DatotekaVBesediloAsyncOcrBody. - :return: The file of this PretvoriDatotekoInOznaciAsyncBody. + :return: The file of this DatotekaVBesediloAsyncOcrBody. :rtype: str """ return self._file @file.setter def file(self, file: str): - """Sets the file of this PretvoriDatotekoInOznaciAsyncBody. + """Sets the file of this DatotekaVBesediloAsyncOcrBody. - :param file: The file of this PretvoriDatotekoInOznaciAsyncBody. + :param file: The file of this DatotekaVBesediloAsyncOcrBody. :type file: str """ if file is None: diff --git a/swagger_server/models/datoteka_v_besedilo_body.py b/swagger_server/models/datoteka_v_besedilo_sync_body.py similarity index 63% rename from swagger_server/models/datoteka_v_besedilo_body.py rename to swagger_server/models/datoteka_v_besedilo_sync_body.py index 954f905..6df60ff 100644 --- a/swagger_server/models/datoteka_v_besedilo_body.py +++ b/swagger_server/models/datoteka_v_besedilo_sync_body.py @@ -9,15 +9,15 @@ from swagger_server.models.base_model_ import Model from swagger_server import util -class DatotekaVBesediloBody(Model): +class DatotekaVBesediloSyncBody(Model): """NOTE: This class is auto generated by the swagger code generator program. Do not edit the class manually. """ def __init__(self, file: str=None): # noqa: E501 - """DatotekaVBesediloBody - a model defined in Swagger + """DatotekaVBesediloSyncBody - a model defined in Swagger - :param file: The file of this DatotekaVBesediloBody. # noqa: E501 + :param file: The file of this DatotekaVBesediloSyncBody. # noqa: E501 :type file: str """ self.swagger_types = { @@ -30,32 +30,32 @@ class DatotekaVBesediloBody(Model): self._file = file @classmethod - def from_dict(cls, dikt) -> 'DatotekaVBesediloBody': + def from_dict(cls, dikt) -> 'DatotekaVBesediloSyncBody': """Returns the dict as a model :param dikt: A dict. :type: dict - :return: The datotekaVBesedilo_body of this DatotekaVBesediloBody. # noqa: E501 - :rtype: DatotekaVBesediloBody + :return: The datotekaVBesediloSync_body of this DatotekaVBesediloSyncBody. # noqa: E501 + :rtype: DatotekaVBesediloSyncBody """ return util.deserialize_model(dikt, cls) @property def file(self) -> str: - """Gets the file of this DatotekaVBesediloBody. + """Gets the file of this DatotekaVBesediloSyncBody. - :return: The file of this DatotekaVBesediloBody. + :return: The file of this DatotekaVBesediloSyncBody. :rtype: str """ return self._file @file.setter def file(self, file: str): - """Sets the file of this DatotekaVBesediloBody. + """Sets the file of this DatotekaVBesediloSyncBody. - :param file: The file of this DatotekaVBesediloBody. + :param file: The file of this DatotekaVBesediloSyncBody. :type file: str """ if file is None: diff --git a/swagger_server/models/datoteka_v_besedilo_sync_ocr_body.py b/swagger_server/models/datoteka_v_besedilo_sync_ocr_body.py new file mode 100644 index 0000000..059baa9 --- /dev/null +++ b/swagger_server/models/datoteka_v_besedilo_sync_ocr_body.py @@ -0,0 +1,64 @@ +# coding: utf-8 + +from __future__ import absolute_import +from datetime import date, datetime # noqa: F401 + +from typing import List, Dict # noqa: F401 + +from swagger_server.models.base_model_ import Model +from swagger_server import util + + +class DatotekaVBesediloSyncOcrBody(Model): + """NOTE: This class is auto generated by the swagger code generator program. + + Do not edit the class manually. + """ + def __init__(self, file: str=None): # noqa: E501 + """DatotekaVBesediloSyncOcrBody - a model defined in Swagger + + :param file: The file of this DatotekaVBesediloSyncOcrBody. # noqa: E501 + :type file: str + """ + self.swagger_types = { + 'file': str + } + + self.attribute_map = { + 'file': 'file' + } + self._file = file + + @classmethod + def from_dict(cls, dikt) -> 'DatotekaVBesediloSyncOcrBody': + """Returns the dict as a model + + :param dikt: A dict. + :type: dict + :return: The datotekaVBesediloSync_ocr_body of this DatotekaVBesediloSyncOcrBody. # noqa: E501 + :rtype: DatotekaVBesediloSyncOcrBody + """ + return util.deserialize_model(dikt, cls) + + @property + def file(self) -> str: + """Gets the file of this DatotekaVBesediloSyncOcrBody. + + + :return: The file of this DatotekaVBesediloSyncOcrBody. + :rtype: str + """ + return self._file + + @file.setter + def file(self, file: str): + """Sets the file of this DatotekaVBesediloSyncOcrBody. + + + :param file: The file of this DatotekaVBesediloSyncOcrBody. + :type file: str + """ + if file is None: + raise ValueError("Invalid value for `file`, must not be `None`") # noqa: E501 + + self._file = file diff --git a/swagger_server/models/datoteka_v_besedilo_ocr_body.py b/swagger_server/models/datoteka_v_conllu_async_body.py similarity index 65% rename from swagger_server/models/datoteka_v_besedilo_ocr_body.py rename to swagger_server/models/datoteka_v_conllu_async_body.py index cb1d7e6..2ad9058 100644 --- a/swagger_server/models/datoteka_v_besedilo_ocr_body.py +++ b/swagger_server/models/datoteka_v_conllu_async_body.py @@ -9,15 +9,15 @@ from swagger_server.models.base_model_ import Model from swagger_server import util -class DatotekaVBesediloOcrBody(Model): +class DatotekaVConlluAsyncBody(Model): """NOTE: This class is auto generated by the swagger code generator program. Do not edit the class manually. """ def __init__(self, file: str=None): # noqa: E501 - """DatotekaVBesediloOcrBody - a model defined in Swagger + """DatotekaVConlluAsyncBody - a model defined in Swagger - :param file: The file of this DatotekaVBesediloOcrBody. # noqa: E501 + :param file: The file of this DatotekaVConlluAsyncBody. # noqa: E501 :type file: str """ self.swagger_types = { @@ -30,32 +30,32 @@ class DatotekaVBesediloOcrBody(Model): self._file = file @classmethod - def from_dict(cls, dikt) -> 'DatotekaVBesediloOcrBody': + def from_dict(cls, dikt) -> 'DatotekaVConlluAsyncBody': """Returns the dict as a model :param dikt: A dict. :type: dict - :return: The datotekaVBesedilo_ocr_body of this DatotekaVBesediloOcrBody. # noqa: E501 - :rtype: DatotekaVBesediloOcrBody + :return: The datotekaVConlluAsync_body of this DatotekaVConlluAsyncBody. # noqa: E501 + :rtype: DatotekaVConlluAsyncBody """ return util.deserialize_model(dikt, cls) @property def file(self) -> str: - """Gets the file of this DatotekaVBesediloOcrBody. + """Gets the file of this DatotekaVConlluAsyncBody. - :return: The file of this DatotekaVBesediloOcrBody. + :return: The file of this DatotekaVConlluAsyncBody. :rtype: str """ return self._file @file.setter def file(self, file: str): - """Sets the file of this DatotekaVBesediloOcrBody. + """Sets the file of this DatotekaVConlluAsyncBody. - :param file: The file of this DatotekaVBesediloOcrBody. + :param file: The file of this DatotekaVConlluAsyncBody. :type file: str """ if file is None: diff --git a/swagger_server/models/datoteka_v_conllu_async_ocr_body.py b/swagger_server/models/datoteka_v_conllu_async_ocr_body.py new file mode 100644 index 0000000..9b33a73 --- /dev/null +++ b/swagger_server/models/datoteka_v_conllu_async_ocr_body.py @@ -0,0 +1,64 @@ +# coding: utf-8 + +from __future__ import absolute_import +from datetime import date, datetime # noqa: F401 + +from typing import List, Dict # noqa: F401 + +from swagger_server.models.base_model_ import Model +from swagger_server import util + + +class DatotekaVConlluAsyncOcrBody(Model): + """NOTE: This class is auto generated by the swagger code generator program. + + Do not edit the class manually. + """ + def __init__(self, file: str=None): # noqa: E501 + """DatotekaVConlluAsyncOcrBody - a model defined in Swagger + + :param file: The file of this DatotekaVConlluAsyncOcrBody. # noqa: E501 + :type file: str + """ + self.swagger_types = { + 'file': str + } + + self.attribute_map = { + 'file': 'file' + } + self._file = file + + @classmethod + def from_dict(cls, dikt) -> 'DatotekaVConlluAsyncOcrBody': + """Returns the dict as a model + + :param dikt: A dict. + :type: dict + :return: The datotekaVConlluAsync_ocr_body of this DatotekaVConlluAsyncOcrBody. # noqa: E501 + :rtype: DatotekaVConlluAsyncOcrBody + """ + return util.deserialize_model(dikt, cls) + + @property + def file(self) -> str: + """Gets the file of this DatotekaVConlluAsyncOcrBody. + + + :return: The file of this DatotekaVConlluAsyncOcrBody. + :rtype: str + """ + return self._file + + @file.setter + def file(self, file: str): + """Sets the file of this DatotekaVConlluAsyncOcrBody. + + + :param file: The file of this DatotekaVConlluAsyncOcrBody. + :type file: str + """ + if file is None: + raise ValueError("Invalid value for `file`, must not be `None`") # noqa: E501 + + self._file = file diff --git a/swagger_server/models/job_response.py b/swagger_server/models/job_response.py index c4507a3..2a0bfc0 100644 --- a/swagger_server/models/job_response.py +++ b/swagger_server/models/job_response.py @@ -14,34 +14,39 @@ class JobResponse(Model): Do not edit the class manually. """ - def __init__(self, finished_job: bool=None, completed_at: datetime=None, estimated_completion: datetime=None, job_result: str=None): # noqa: E501 + def __init__(self, job_status: str=None, finished_on: datetime=None, started_on: datetime=None, created_on: datetime=None, job_result: str=None): # noqa: E501 """JobResponse - a model defined in Swagger - :param finished_job: The finished_job of this JobResponse. # noqa: E501 - :type finished_job: bool - :param completed_at: The completed_at of this JobResponse. # noqa: E501 - :type completed_at: datetime - :param estimated_completion: The estimated_completion of this JobResponse. # noqa: E501 - :type estimated_completion: datetime + :param job_status: The job_status of this JobResponse. # noqa: E501 + :type job_status: str + :param finished_on: The finished_on of this JobResponse. # noqa: E501 + :type finished_on: datetime + :param started_on: The started_on of this JobResponse. # noqa: E501 + :type started_on: datetime + :param created_on: The created_on of this JobResponse. # noqa: E501 + :type created_on: datetime :param job_result: The job_result of this JobResponse. # noqa: E501 :type job_result: str """ self.swagger_types = { - 'finished_job': bool, - 'completed_at': datetime, - 'estimated_completion': datetime, + 'job_status': str, + 'finished_on': datetime, + 'started_on': datetime, + 'created_on': datetime, 'job_result': str } self.attribute_map = { - 'finished_job': 'finished_job', - 'completed_at': 'completed_at', - 'estimated_completion': 'estimated_completion', + 'job_status': 'job_status', + 'finished_on': 'finished_on', + 'started_on': 'started_on', + 'created_on': 'created_on', 'job_result': 'job_result' } - self._finished_job = finished_job - self._completed_at = completed_at - self._estimated_completion = estimated_completion + self._job_status = job_status + self._finished_on = finished_on + self._started_on = started_on + self._created_on = created_on self._job_result = job_result @classmethod @@ -56,69 +61,94 @@ class JobResponse(Model): return util.deserialize_model(dikt, cls) @property - def finished_job(self) -> bool: - """Gets the finished_job of this JobResponse. + def job_status(self) -> str: + """Gets the job_status of this JobResponse. - :return: The finished_job of this JobResponse. - :rtype: bool + :return: The job_status of this JobResponse. + :rtype: str """ - return self._finished_job + return self._job_status - @finished_job.setter - def finished_job(self, finished_job: bool): - """Sets the finished_job of this JobResponse. + @job_status.setter + def job_status(self, job_status: str): + """Sets the job_status of this JobResponse. - :param finished_job: The finished_job of this JobResponse. - :type finished_job: bool + :param job_status: The job_status of this JobResponse. + :type job_status: str """ - if finished_job is None: - raise ValueError("Invalid value for `finished_job`, must not be `None`") # noqa: E501 + allowed_values = ["waiting in que", "currently processing", "finished processing"] # noqa: E501 + if job_status not in allowed_values: + raise ValueError( + "Invalid value for `job_status` ({0}), must be one of {1}" + .format(job_status, allowed_values) + ) - self._finished_job = finished_job + self._job_status = job_status @property - def completed_at(self) -> datetime: - """Gets the completed_at of this JobResponse. + def finished_on(self) -> datetime: + """Gets the finished_on of this JobResponse. - :return: The completed_at of this JobResponse. + :return: The finished_on of this JobResponse. :rtype: datetime """ - return self._completed_at + return self._finished_on - @completed_at.setter - def completed_at(self, completed_at: datetime): - """Sets the completed_at of this JobResponse. + @finished_on.setter + def finished_on(self, finished_on: datetime): + """Sets the finished_on of this JobResponse. - :param completed_at: The completed_at of this JobResponse. - :type completed_at: datetime + :param finished_on: The finished_on of this JobResponse. + :type finished_on: datetime """ - self._completed_at = completed_at + self._finished_on = finished_on @property - def estimated_completion(self) -> datetime: - """Gets the estimated_completion of this JobResponse. + def started_on(self) -> datetime: + """Gets the started_on of this JobResponse. - :return: The estimated_completion of this JobResponse. + :return: The started_on of this JobResponse. :rtype: datetime """ - return self._estimated_completion + return self._started_on - @estimated_completion.setter - def estimated_completion(self, estimated_completion: datetime): - """Sets the estimated_completion of this JobResponse. + @started_on.setter + def started_on(self, started_on: datetime): + """Sets the started_on of this JobResponse. - :param estimated_completion: The estimated_completion of this JobResponse. - :type estimated_completion: datetime + :param started_on: The started_on of this JobResponse. + :type started_on: datetime """ - self._estimated_completion = estimated_completion + self._started_on = started_on + + @property + def created_on(self) -> datetime: + """Gets the created_on of this JobResponse. + + + :return: The created_on of this JobResponse. + :rtype: datetime + """ + return self._created_on + + @created_on.setter + def created_on(self, created_on: datetime): + """Sets the created_on of this JobResponse. + + + :param created_on: The created_on of this JobResponse. + :type created_on: datetime + """ + + self._created_on = created_on @property def job_result(self) -> str: diff --git a/swagger_server/requets_db/models/vrsta.py b/swagger_server/requets_db/models/vrsta.py index f637905..73f8a96 100644 --- a/swagger_server/requets_db/models/vrsta.py +++ b/swagger_server/requets_db/models/vrsta.py @@ -28,13 +28,13 @@ class BaseModel(Model): class Job(BaseModel): id = AutoField() - job_type = IntegerField() # 1 = oznaci besedilo + job_type = IntegerField() # 1 = pretvori datoteko v besedilo, 2 = oznaci besedilo, 21 = oboje job_input = TextField(index=True) job_output = TextField(null=True) created_on = DateTimeField(default=datetime.utcnow) finished_on = DateTimeField(null=True) - input_size = IntegerField() started_on = DateTimeField(null=True) + input_size = IntegerField() # db.drop_tables([Job]) diff --git a/swagger_server/swagger/swagger.yaml b/swagger_server/swagger/swagger.yaml index 0138170..63445a5 100644 --- a/swagger_server/swagger/swagger.yaml +++ b/swagger_server/swagger/swagger.yaml @@ -21,15 +21,6 @@ paths: schema: type: integer format: int64 - - name: show_estimated_completion - in: query - description: Calculate estimate time remaining based on various factors (could - be inaccurate) - required: false - style: form - explode: true - schema: - type: boolean responses: "200": description: OK @@ -38,12 +29,35 @@ paths: schema: $ref: '#/components/schemas/JobResponse' x-openapi-router-controller: swagger_server.controllers.jobs_controller + delete: + tags: + - jobs + summary: Izbriše job + operationId: delete_job + parameters: + - name: job_id + in: path + required: true + style: simple + explode: false + schema: + type: integer + format: int64 + responses: + "200": + description: OK + content: + '*/*': + schema: + type: string + x-content-type: '*/*' + x-openapi-router-controller: swagger_server.controllers.jobs_controller /oznaciBesediloAsync: post: tags: - - marktext - summary: Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne - conll-u format + - marktext-async + summary: Označi surovo (angl. raw) besedilo s classlo/stanzo z uporabo slovenskih + modelov ter vrne conll-u format operationId: get_text requestBody: content: @@ -59,19 +73,39 @@ paths: schema: type: string x-content-type: '*/*' - x-openapi-router-controller: swagger_server.controllers.marktext_controller - /pretvoriDatotekoInOznaciAsync: + x-openapi-router-controller: swagger_server.controllers.marktext_async_controller + /datotekaVBesediloAsync: post: tags: - - marktext + - marktext-async + summary: "Pretvori datoteko v besedilo, vrača tekst" + operationId: get_text_from_doc_async + requestBody: + content: + multipart/form-data: + schema: + $ref: '#/components/schemas/datotekaVBesediloAsync_body' + responses: + "200": + description: OK + content: + '*/*': + schema: + type: string + x-content-type: '*/*' + x-openapi-router-controller: swagger_server.controllers.marktext_async_controller + /datotekaVConlluAsync: + post: + tags: + - marktext-async summary: Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format - operationId: get_text_from_file + operationId: get_conllu_from_file_async requestBody: content: multipart/form-data: schema: - $ref: '#/components/schemas/pretvoriDatotekoInOznaciAsync_body' + $ref: '#/components/schemas/datotekaVConlluAsync_body' responses: "200": description: OK @@ -80,19 +114,19 @@ paths: schema: type: string x-content-type: '*/*' - x-openapi-router-controller: swagger_server.controllers.marktext_controller - /pretvoriDatotekoInOznaciAsync/ocr: + x-openapi-router-controller: swagger_server.controllers.marktext_async_controller + /datotekaVConlluAsync/ocr: post: tags: - - marktext - summary: Pretvori datoteko v besedilo s pomočjo ocr razpoznavanja in označi - s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format - operationId: get_text_from_file_ocr + - marktext-async + summary: Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih + modelov ter vrne conll-u format + operationId: get_conllu_from_file_ocr_async requestBody: content: multipart/form-data: schema: - $ref: '#/components/schemas/pretvoriDatotekoInOznaciAsync_ocr_body' + $ref: '#/components/schemas/datotekaVConlluAsync_ocr_body' responses: "200": description: OK @@ -101,7 +135,27 @@ paths: schema: type: string x-content-type: '*/*' - x-openapi-router-controller: swagger_server.controllers.marktext_controller + x-openapi-router-controller: swagger_server.controllers.marktext_async_controller + /datotekaVBesediloAsync/ocr: + post: + tags: + - marktext-async + summary: "Pretvori datoteko v besedilo s pomočjo ocr razpoznavanja, vrača tekst" + operationId: get_text_from_file_ocr_async + requestBody: + content: + multipart/form-data: + schema: + $ref: '#/components/schemas/datotekaVBesediloAsync_ocr_body' + responses: + "200": + description: OK + content: + '*/*': + schema: + type: string + x-content-type: '*/*' + x-openapi-router-controller: swagger_server.controllers.marktext_async_controller /izlusci: post: tags: @@ -125,17 +179,17 @@ paths: $ref: '#/components/schemas/TerminoloskiKandidat' x-content-type: '*/*' x-openapi-router-controller: swagger_server.controllers.extract_controller - /datotekaVBesedilo: + /datotekaVBesediloSync: post: tags: - - doc-2text - summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vraca besedilo" - operationId: datoteka_v_besedilo_post + - marktext-sync + summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vrača besedilo" + operationId: datoteka_v_besedilo_sync_post requestBody: content: multipart/form-data: schema: - $ref: '#/components/schemas/datotekaVBesedilo_body' + $ref: '#/components/schemas/datotekaVBesediloSync_body' responses: "200": description: OK @@ -144,11 +198,11 @@ paths: schema: type: string x-content-type: '*/*' - x-openapi-router-controller: swagger_server.controllers.doc2text_controller - /datotekaVBesedilo/ocr: + x-openapi-router-controller: swagger_server.controllers.marktext_sync_controller + /datotekaVBesediloSync/ocr: post: tags: - - doc-2text + - marktext-sync summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo\ \ s pomočjo ocr razpoznavanja" operationId: get_text_ocr @@ -156,7 +210,7 @@ paths: content: multipart/form-data: schema: - $ref: '#/components/schemas/datotekaVBesedilo_ocr_body' + $ref: '#/components/schemas/datotekaVBesediloSync_ocr_body' responses: "200": description: OK @@ -165,12 +219,12 @@ paths: schema: type: string x-content-type: '*/*' - x-openapi-router-controller: swagger_server.controllers.doc2text_controller + x-openapi-router-controller: swagger_server.controllers.marktext_sync_controller /datotekaVConlluSync: post: tags: - - doc-2text - summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vraca conllu" + - marktext-sync + summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vrača conllu" operationId: datoteka_v_besedilo_in_classla requestBody: content: @@ -185,11 +239,11 @@ paths: schema: type: string x-content-type: '*/*' - x-openapi-router-controller: swagger_server.controllers.doc2text_controller + x-openapi-router-controller: swagger_server.controllers.marktext_sync_controller /datotekaVConlluSync/ocr: post: tags: - - doc-2text + - marktext-sync summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v conllu s\ \ pomočjo ocr razpoznavanja" operationId: get_conllu_ocr @@ -206,7 +260,7 @@ paths: schema: type: string x-content-type: '*/*' - x-openapi-router-controller: swagger_server.controllers.doc2text_controller + x-openapi-router-controller: swagger_server.controllers.marktext_sync_controller /oss/steviloBesedilPoIskanju: get: tags: @@ -605,27 +659,35 @@ components: - finished_job type: object properties: - finished_job: - type: boolean - completed_at: + job_status: + type: string + enum: + - waiting in que + - currently processing + - finished processing + finished_on: type: string format: date-time - estimated_completion: + started_on: + type: string + format: date-time + created_on: type: string format: date-time job_result: type: string example: - completed_at: 2000-01-23T04:56:07.000+00:00 - estimated_completion: 2000-01-23T04:56:07.000+00:00 + job_status: finished processing + started_on: 2000-01-23T04:56:07.000+00:00 + created_on: 2000-01-23T04:56:07.000+00:00 + finished_on: 2000-01-23T04:56:07.000+00:00 job_result: job_result - finished_job: true oznaciBesediloAsync_body: type: object properties: besedilo: type: string - pretvoriDatotekoInOznaciAsync_body: + datotekaVBesediloAsync_body: required: - file type: object @@ -633,7 +695,23 @@ components: file: type: string format: binary - pretvoriDatotekoInOznaciAsync_ocr_body: + datotekaVConlluAsync_body: + required: + - file + type: object + properties: + file: + type: string + format: binary + datotekaVConlluAsync_ocr_body: + required: + - file + type: object + properties: + file: + type: string + format: binary + datotekaVBesediloAsync_ocr_body: required: - file type: object @@ -652,7 +730,7 @@ components: type: array items: type: string - datotekaVBesedilo_body: + datotekaVBesediloSync_body: required: - file type: object @@ -660,7 +738,7 @@ components: file: type: string format: binary - datotekaVBesedilo_ocr_body: + datotekaVBesediloSync_ocr_body: required: - file type: object diff --git a/swagger_server/test/__init__.py b/swagger_server/test/__init__.py index 6445063..e69de29 100644 --- a/swagger_server/test/__init__.py +++ b/swagger_server/test/__init__.py @@ -1,16 +0,0 @@ -import logging - -import connexion -from flask_testing import TestCase - -from swagger_server.encoder import JSONEncoder - - -class BaseTestCase(TestCase): - - def create_app(self): - logging.getLogger('connexion.operation').setLevel('ERROR') - app = connexion.App(__name__, specification_dir='../swagger/') - app.app.json_encoder = JSONEncoder - app.add_api('swagger.yaml') - return app.app diff --git a/swagger_server/test/__main__.py b/swagger_server/test/__main__.py new file mode 100644 index 0000000..44ac5d1 --- /dev/null +++ b/swagger_server/test/__main__.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 + +import connexion + +from swagger_server import encoder + + +def main(): + app = connexion.App(__name__, specification_dir='./swagger/') + app.app.json_encoder = encoder.JSONEncoder + app.add_api('swagger.yaml', arguments={'title': 'OpenAPI definition'}, pythonic_params=True) + app.run(port=8080) + + +if __name__ == '__main__': + main() diff --git a/swagger_server/test/encoder.py b/swagger_server/test/encoder.py new file mode 100644 index 0000000..61ba472 --- /dev/null +++ b/swagger_server/test/encoder.py @@ -0,0 +1,20 @@ +from connexion.apps.flask_app import FlaskJSONEncoder +import six + +from swagger_server.models.base_model_ import Model + + +class JSONEncoder(FlaskJSONEncoder): + include_nulls = False + + def default(self, o): + if isinstance(o, Model): + dikt = {} + for attr, _ in six.iteritems(o.swagger_types): + value = getattr(o, attr) + if value is None and not self.include_nulls: + continue + attr = o.attribute_map[attr] + dikt[attr] = value + return dikt + return FlaskJSONEncoder.default(self, o) diff --git a/swagger_server/test/test_doc2text_controller.py b/swagger_server/test/test_doc2text_controller.py deleted file mode 100644 index 43d8aa3..0000000 --- a/swagger_server/test/test_doc2text_controller.py +++ /dev/null @@ -1,45 +0,0 @@ -# coding: utf-8 - -from __future__ import absolute_import - -from flask import json -from six import BytesIO - -from swagger_server.test import BaseTestCase - - -class TestDoc2textController(BaseTestCase): - """Doc2textController integration test stubs""" - - def test_datoteka_v_besedilo_post(self): - """Test case for datoteka_v_besedilo_post - - Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo - """ - data = dict(file='file_example') - response = self.client.open( - '/datotekaVBesedilo/', - method='POST', - data=data, - content_type='multipart/form-data') - self.assert200(response, - 'Response body is : ' + response.data.decode('utf-8')) - - def test_get_text_ocr(self): - """Test case for get_text_ocr - - Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo s pomočjo ocr razpoznavanja - """ - data = dict(file='file_example') - response = self.client.open( - '/datotekaVBesedilo/ocr', - method='POST', - data=data, - content_type='multipart/form-data') - self.assert200(response, - 'Response body is : ' + response.data.decode('utf-8')) - - -if __name__ == '__main__': - import unittest - unittest.main() diff --git a/swagger_server/test/test_extract_controller.py b/swagger_server/test/test_extract_controller.py deleted file mode 100644 index 63da4ac..0000000 --- a/swagger_server/test/test_extract_controller.py +++ /dev/null @@ -1,33 +0,0 @@ -# coding: utf-8 - -from __future__ import absolute_import - -from flask import json -from six import BytesIO - -from swagger_server.models.izlusci_body import IzlusciBody # noqa: E501 -from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat # noqa: E501 -from swagger_server.test import BaseTestCase - - -class TestExtractController(BaseTestCase): - """ExtractController integration test stubs""" - - def test_get_candidates(self): - """Test case for get_candidates - - Izlusci terminološke kandidate iz seznama besedil v conllu obliki - """ - body = IzlusciBody() - response = self.client.open( - '/izlusci', - method='POST', - data=json.dumps(body), - content_type='application/json') - self.assert200(response, - 'Response body is : ' + response.data.decode('utf-8')) - - -if __name__ == '__main__': - import unittest - unittest.main() diff --git a/swagger_server/test/test_jobs_controller.py b/swagger_server/test/test_jobs_controller.py deleted file mode 100644 index 942f5ce..0000000 --- a/swagger_server/test/test_jobs_controller.py +++ /dev/null @@ -1,32 +0,0 @@ -# coding: utf-8 - -from __future__ import absolute_import - -from flask import json -from six import BytesIO - -from swagger_server.models.job_response import JobResponse # noqa: E501 -from swagger_server.test import BaseTestCase - - -class TestJobsController(BaseTestCase): - """JobsController integration test stubs""" - - def test_get_job_status(self): - """Test case for get_job_status - - Vrne status - """ - query_string = [('job_id', 789), - ('show_estimated_completion', True)] - response = self.client.open( - '/job', - method='GET', - query_string=query_string) - self.assert200(response, - 'Response body is : ' + response.data.decode('utf-8')) - - -if __name__ == '__main__': - import unittest - unittest.main() diff --git a/swagger_server/test/test_marktext_controller.py b/swagger_server/test/test_marktext_controller.py deleted file mode 100644 index 4f051c5..0000000 --- a/swagger_server/test/test_marktext_controller.py +++ /dev/null @@ -1,32 +0,0 @@ -# coding: utf-8 - -from __future__ import absolute_import - -from flask import json -from six import BytesIO - -from swagger_server.models.oznaci_besedilo_async_body import OznaciBesediloAsyncBody # noqa: E501 -from swagger_server.test import BaseTestCase - - -class TestMarktextController(BaseTestCase): - """MarktextController integration test stubs""" - - def test_get_text(self): - """Test case for get_text - - Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format - """ - body = OznaciBesediloAsyncBody() - response = self.client.open( - '/oznaciBesediloAsync', - method='POST', - data=json.dumps(body), - content_type='application/json') - self.assert200(response, - 'Response body is : ' + response.data.decode('utf-8')) - - -if __name__ == '__main__': - import unittest - unittest.main() diff --git a/swagger_server/test/test_oss_controller.py b/swagger_server/test/test_oss_controller.py deleted file mode 100644 index f9d71b0..0000000 --- a/swagger_server/test/test_oss_controller.py +++ /dev/null @@ -1,137 +0,0 @@ -# coding: utf-8 - -from __future__ import absolute_import - -from flask import json -from six import BytesIO - -from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat # noqa: E501 -from swagger_server.test import BaseTestCase - - -class TestOssController(BaseTestCase): - """OssController integration test stubs""" - - def test_get_conllu(self): - """Test case for get_conllu - - Vrne CoNNL-U po id-ju datoteke - """ - query_string = [('file_id', 789)] - response = self.client.open( - '/oss/conlluPoId', - method='GET', - query_string=query_string) - self.assert200(response, - 'Response body is : ' + response.data.decode('utf-8')) - - def test_get_conllus(self): - """Test case for get_conllus - - Vrne seznam CoNNL-U-jev glede na iskalne pogoje - """ - query_string = [('leta', 56), - ('vrste', 'vrste_example'), - ('kljucnebesede', 'kljucnebesede_example'), - ('cerifpodrocja', 56)] - response = self.client.open( - '/oss/conlluPoIskanju', - method='GET', - query_string=query_string) - self.assert200(response, - 'Response body is : ' + response.data.decode('utf-8')) - - def test_get_extracted_words(self): - """Test case for get_extracted_words - - Vrne terminloške kandidate glede na - """ - query_string = [('leta', 56), - ('vrste', 'vrste_example'), - ('kljucnebesede', 'kljucnebesede_example'), - ('cerifpodrocja', 56)] - response = self.client.open( - '/oss/izlusciPoIskanju', - method='GET', - query_string=query_string) - self.assert200(response, - 'Response body is : ' + response.data.decode('utf-8')) - - def test_get_file(self): - """Test case for get_file - - Vrne binarni zapis v originalnem formatu po id-ju datoteke - """ - query_string = [('file_id', 789)] - response = self.client.open( - '/oss/datotekaPoId', - method='GET', - query_string=query_string) - self.assert200(response, - 'Response body is : ' + response.data.decode('utf-8')) - - def test_get_files(self): - """Test case for get_files - - Vrne seznam binarnih zapisov v originalnem formatu glede na iskalne pogoje - """ - query_string = [('leta', 56), - ('vrste', 'vrste_example'), - ('kljucnebesede', 'kljucnebesede_example'), - ('cerifpodrocja', 56)] - response = self.client.open( - '/oss/datotekePoIskanju', - method='GET', - query_string=query_string) - self.assert200(response, - 'Response body is : ' + response.data.decode('utf-8')) - - def test_get_number_texts(self): - """Test case for get_number_texts - - Vrne število besedil glede na iskalne pogoje - """ - query_string = [('leta', 56), - ('vrste', 'vrste_example'), - ('kljucnebesede', 'kljucnebesede_example'), - ('cerifpodrocja', 56)] - response = self.client.open( - '/oss/steviloBesedilPoIskanju', - method='GET', - query_string=query_string) - self.assert200(response, - 'Response body is : ' + response.data.decode('utf-8')) - - def test_get_texts(self): - """Test case for get_texts - - Vrne seznam besedil glede na iskalne pogoje - """ - query_string = [('leta', 56), - ('vrste', 'vrste_example'), - ('kljucnebesede', 'kljucnebesede_example'), - ('cerifpodrocja', 56)] - response = self.client.open( - '/oss/besedilaPoIskanju', - method='GET', - query_string=query_string) - self.assert200(response, - 'Response body is : ' + response.data.decode('utf-8')) - - def test_oss_besedilo_po_id_get(self): - """Test case for oss_besedilo_po_id_get - - Vrne besedilo po id-ju datoteke - """ - query_string = [('file_id', 789)] - response = self.client.open( - '/oss/besediloPoId', - method='GET', - query_string=query_string) - self.assert200(response, - 'Response body is : ' + response.data.decode('utf-8')) - - -if __name__ == '__main__': - import unittest - unittest.main() diff --git a/swagger_server/test/type_util.py b/swagger_server/test/type_util.py new file mode 100644 index 0000000..0563f81 --- /dev/null +++ b/swagger_server/test/type_util.py @@ -0,0 +1,32 @@ +# coding: utf-8 + +import sys + +if sys.version_info < (3, 7): + import typing + + def is_generic(klass): + """ Determine whether klass is a generic class """ + return type(klass) == typing.GenericMeta + + def is_dict(klass): + """ Determine whether klass is a Dict """ + return klass.__extra__ == dict + + def is_list(klass): + """ Determine whether klass is a List """ + return klass.__extra__ == list + +else: + + def is_generic(klass): + """ Determine whether klass is a generic class """ + return hasattr(klass, '__origin__') + + def is_dict(klass): + """ Determine whether klass is a Dict """ + return klass.__origin__ == dict + + def is_list(klass): + """ Determine whether klass is a List """ + return klass.__origin__ == list diff --git a/swagger_server/test/util.py b/swagger_server/test/util.py new file mode 100644 index 0000000..812b2ad --- /dev/null +++ b/swagger_server/test/util.py @@ -0,0 +1,142 @@ +import datetime + +import six +import typing +from swagger_server import type_util + + +def _deserialize(data, klass): + """Deserializes dict, list, str into an object. + + :param data: dict, list or str. + :param klass: class literal, or string of class name. + + :return: object. + """ + if data is None: + return None + + if klass in six.integer_types or klass in (float, str, bool, bytearray): + return _deserialize_primitive(data, klass) + elif klass == object: + return _deserialize_object(data) + elif klass == datetime.date: + return deserialize_date(data) + elif klass == datetime.datetime: + return deserialize_datetime(data) + elif type_util.is_generic(klass): + if type_util.is_list(klass): + return _deserialize_list(data, klass.__args__[0]) + if type_util.is_dict(klass): + return _deserialize_dict(data, klass.__args__[1]) + else: + return deserialize_model(data, klass) + + +def _deserialize_primitive(data, klass): + """Deserializes to primitive type. + + :param data: data to deserialize. + :param klass: class literal. + + :return: int, long, float, str, bool. + :rtype: int | long | float | str | bool + """ + try: + value = klass(data) + except UnicodeEncodeError: + value = six.u(data) + except TypeError: + value = data + return value + + +def _deserialize_object(value): + """Return an original value. + + :return: object. + """ + return value + + +def deserialize_date(string): + """Deserializes string to date. + + :param string: str. + :type string: str + :return: date. + :rtype: date + """ + try: + from dateutil.parser import parse + return parse(string).date() + except ImportError: + return string + + +def deserialize_datetime(string): + """Deserializes string to datetime. + + The string should be in iso8601 datetime format. + + :param string: str. + :type string: str + :return: datetime. + :rtype: datetime + """ + try: + from dateutil.parser import parse + return parse(string) + except ImportError: + return string + + +def deserialize_model(data, klass): + """Deserializes list or dict to model. + + :param data: dict, list. + :type data: dict | list + :param klass: class literal. + :return: model object. + """ + instance = klass() + + if not instance.swagger_types: + return data + + for attr, attr_type in six.iteritems(instance.swagger_types): + if data is not None \ + and instance.attribute_map[attr] in data \ + and isinstance(data, (list, dict)): + value = data[instance.attribute_map[attr]] + setattr(instance, attr, _deserialize(value, attr_type)) + + return instance + + +def _deserialize_list(data, boxed_type): + """Deserializes a list and its elements. + + :param data: list to deserialize. + :type data: list + :param boxed_type: class literal. + + :return: deserialized list. + :rtype: list + """ + return [_deserialize(sub_data, boxed_type) + for sub_data in data] + + +def _deserialize_dict(data, boxed_type): + """Deserializes a dict and its elements. + + :param data: dict to deserialize. + :type data: dict + :param boxed_type: class literal. + + :return: deserialized dict. + :rtype: dict + """ + return {k: _deserialize(v, boxed_type) + for k, v in six.iteritems(data)} diff --git a/swagger_server/util.py b/swagger_server/util.py index ec8e7cc..8130360 100644 --- a/swagger_server/util.py +++ b/swagger_server/util.py @@ -177,8 +177,8 @@ def get_files_by_keywords(kljucnebesede): ngrams_path = "classla_OS2022/ngrams/" if is_docker() else "../mnt/ssd/ds_ftp/classla_OS2022/ngrams/" print("Looping trough ngrams") for path, dirs, files, in os.walk(ngrams_path): - for i, _file in enumerate(files[:200]): - if i % 200 == 0: print(f"{i}/{len(files)}") + for i, _file in enumerate(files[:100]): + if i % 500 == 0: print(f"{i}/{len(files)}") file = f'{ngrams_path}{_file}' data = pd.read_csv(file, sep='\t') amount = len(data[data['ngram_len'] == 1 & data['gram_text'].str.lower().isin(kljucnebesede)]) diff --git a/swagger_server/utils/__init__.py b/swagger_server/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/swagger_server/db_utils.py b/swagger_server/utils/db_utils.py similarity index 100% rename from swagger_server/db_utils.py rename to swagger_server/utils/db_utils.py diff --git a/swagger_server/utils/txt_utils.py b/swagger_server/utils/txt_utils.py new file mode 100644 index 0000000..d28ee67 --- /dev/null +++ b/swagger_server/utils/txt_utils.py @@ -0,0 +1,65 @@ +import os.path + +import pytesseract +import requests +import docx +import xml.etree.ElementTree as ET +from PyPDF2 import PdfReader +from swagger_server.classla import cl_utils +import cv2 +import numpy as np + +tika_server = "http://tika2:9999/tika" + + +# endpoint below to be used only for development purposes (don't need to run docker) +# tika_server = "http://rsdo.lhrs.feri.um.si:9998/tika" + +def extract_text_prepResp(file): + if tika_responding(): + response = requests.put(tika_server, data=file) + return response.text, 200 + if "openxmlformats-officedocument.wordprocessingml.document" in file.content_type: + content = '\n'.join([p.text for p in docx.Document(file).paragraphs]) + elif "application/pdf" in file.content_type: + reader = PdfReader(file) + content = '\n'.join([p.extract_text() for p in reader.pages]) + content = content + elif "text/xml" in file.content_type: + root = ET.parse(file).getroot() + plainText = root.findall('PlainText') + if len(plainText) == 0: + return "Didn't find anything in PlainText", 400 + content = '\n'.join([pt.text for pt in plainText]) + # elif "text/plain" in file.content_type: + else: + content = file.read().decode('utf-8') + + return content, 200 + + +def ocr_text_prepResp(file): + if tika_responding(): + response = requests.put(tika_server, data=file, + headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng"}) + return response.text, 200 + + win_p = "C:/Program Files/Tesseract-OCR/tesseract.exe" + if os.path.exists(win_p): + pytesseract.pytesseract.tesseract_cmd = win_p + + # convert string data to numpy array + file_bytes = np.fromstring(file.read(), np.uint8) + # convert numpy array to image + img = cv2.imdecode(file_bytes, cv2.IMREAD_COLOR) + + conf = '-l eng+slv' + return pytesseract.image_to_string(img, config=conf), 200 + + +def tika_responding(): + try: + ret = requests.get(tika_server) + return ret.status_code == 200 + except: + return False \ No newline at end of file