webapps/term_extractor
18
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updated endpoint definitions/names and reformatted some files and code

Browse Source
This commit is contained in:
Kikimanox
2022-09-14 16:40:29 +02:00
parent 0b90b6555b
commit 9e82f3b93a
32 changed files with 988 additions and 742 deletions
Download Patch File Download Diff File Expand all files Collapse all files
swagger_server/controllers/doc2text_controller.py
-157
View File
@@ -1,157 +0,0 @@
import os.path
from tempfile import TemporaryFile
import connexion
import pytesseract
import six
import requests
import json
from swagger_server import util
import pandas as pd
import docx
import codecs
import xml.etree.ElementTree as ET
from PyPDF2 import PdfReader
from swagger_server.classla import cl_utils
import traceback
import cv2
import numpy as np
tika_server = "http://tika2:9999/tika"
# endpoint below to be used only for development purposes (don't need to run docker)
# tika_server = "http://rsdo.lhrs.feri.um.si:9998/tika"
def extract_text_prepResp(file):
if tika_responding():
response = requests.put(tika_server, data=file)
return response.text, 200
if "openxmlformats-officedocument.wordprocessingml.document" in file.content_type:
content = [p.text for p in docx.Document(file).paragraphs]
elif "application/pdf" in file.content_type:
reader = PdfReader(file)
content = '\n'.join([p.extract_text() for p in reader.pages])
content = "ZACASNO UPORABLJEN DRUGI BRALEC KOT TIKA, TA BO SE DODANA KASNEJE...\n\n" + content
elif "text/xml" in file.content_type:
root = ET.parse(file).getroot()
plainText = root.findall('PlainText')
if len(plainText) == 0:
return "Didn't find anything in PlainText", 400
content = '\n'.join([pt.text for pt in plainText])
# elif "text/plain" in file.content_type:
else:
content = file.read().decode('utf-8')
return content, 200
def ocr_text_prepResp(file):
if tika_responding():
response = requests.put(tika_server, data=file,
headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng"})
return response.text, 200
win_p = "C:/Program Files/Tesseract-OCR/tesseract.exe"
if os.path.exists(win_p):
pytesseract.pytesseract.tesseract_cmd = win_p
# convert string data to numpy array
file_bytes = np.fromstring(file.read(), np.uint8)
# convert numpy array to image
img = cv2.imdecode(file_bytes, cv2.IMREAD_COLOR)
conf = '-l eng+slv'
return pytesseract.image_to_string(img, config=conf), 200
def tika_responding():
try:
ret = requests.get(tika_server)
return ret.status_code == 200
except:
return False
def datoteka_v_besedilo_post(file=None): # noqa: E501
"""Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vraca besedilo
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
if file is None:
return "No file provided", 400
try:
return extract_text_prepResp(file)
except Exception as e:
return str(e), 500
def get_text_ocr(file=None): # noqa: E501
"""Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo s pomočjo ocr razpoznavanja
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
if file is None:
return "No file provided", 400
try:
return ocr_text_prepResp(file)
except Exception as e:
return str(e), 500
def datoteka_v_besedilo_in_classla(file=None): # noqa: E501
"""Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vraca conllu
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
if not cl_utils.nlp_loaded:
return "NLP Models still loading up since server restart, please retry later.", 500
if file is None:
return "No file provided", 400
try:
txt, _ = extract_text_prepResp(file)
return cl_utils.raw_text_to_conllu(txt)
except Exception as e:
return str(e), 500
def get_conllu_ocr(file=None): # noqa: E501
"""Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v conllu s pomočjo ocr razpoznavanja
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
if not cl_utils.nlp_loaded:
return "NLP Models still loading up since server restart, please retry later.", 500
if file is None:
return "No file provided", 400
try:
if tika_responding():
response = requests.put(tika_server, data=file,
headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng"})
return cl_utils.raw_text_to_conllu(response.text)
else:
txt, _ = ocr_text_prepResp(file)
return cl_utils.raw_text_to_conllu(txt)
except Exception as e:
return str(e), 500
swagger_server/controllers/extract_controller.py
-3
View File
@@ -1,9 +1,6 @@
import connexion
import six
from swagger_server.models.izlusci_body import IzlusciBody # noqa: E501
from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat # noqa: E501
from swagger_server import util
def get_candidates(body): # noqa: E501
swagger_server/controllers/jobs_controller.py
+29 -17
View File
@@ -1,41 +1,51 @@
import datetime
import random
import connexion
import peewee
import six
import asyncio
from swagger_server.models.job_response import JobResponse # noqa: E501
from swagger_server import util
from swagger_server.requets_db.models.vrsta import (Job, JobManager)
from threading import Semaphore, Thread
from swagger_server.requets_db.models.vrsta import (Job)
from threading import Thread
from swagger_server.classla import cl_utils
CLASSLA_CONCURANCE_LIMIT = 4
classla_sem = asyncio.Semaphore(CLASSLA_CONCURANCE_LIMIT)
def get_job_status(job_id, show_estimated_completion=None): # noqa: E501
def delete_job(job_id): # noqa: E501
"""Izbriše job
# noqa: E501
:param job_id:
:type job_id: int
:rtype: str
"""
return 'Endpoint currently disabled'
def get_job_status(job_id): # noqa: E501
"""Vrne status
# noqa: E501
:param job_id:
:type job_id: int
:param show_estimated_completion: Calculate estimate time remaining based on various factors (could be inaccurate)
:type show_estimated_completion: bool
:rtype: JobResponse
"""
try:
job = Job.get_by_id(job_id)
if not job.finished_on:
est_com = None
# todo: if estimate completion: calculate it and set it to est_com
return JobResponse(finished_job=False, estimated_completion=est_com), 200
return JobResponse(finished_job=True, completed_at=job.finished_on, job_result=job.job_output), 200
except peewee.DoesNotExist as e:
if job.started_on is None:
return JobResponse(job_status="waiting in que", created_on=job.created_on), 200
if job.started_on is not None and job.finished_on is None:
return JobResponse(job_status="currently processing", created_on=job.created_on,
started_on=job.started_on), 200
if job.started_on is not None and job.finished_on is not None:
return JobResponse(job_status="finished processing", created_on=job.created_on, started_on=job.started_on,
finished_on=job.finished_on, job_result=job.job_output), 200
except peewee.DoesNotExist:
return "Job with this ID does not exist", 404
@@ -43,6 +53,7 @@ def clear_up_unfinished_jobs():
"""
In case server crashed while jobs were in queue...
"""
# tu more but !=, ne deluje ce je "is not"
Job.update(started_on=None).where(Job.started_on != None, Job.finished_on == None).execute()
@@ -56,7 +67,7 @@ async def try_do_jobs():
if classla_sem._value > 0:
# classla
unfinished_jobs = Job.select() \
.where(Job.finished_on == None, Job.started_on == None, Job.job_type == 1) \
.where(Job.finished_on == None, Job.started_on == None, Job.job_type == 2) \
.limit(classla_sem._value)
tasks = [
asyncio.ensure_future(execute_classla_job(job))
@@ -91,6 +102,8 @@ async def execute_classla_job(job: Job):
clear_up_unfinished_jobs()
loop = asyncio.get_event_loop()
def loop_in_thread(loop):
asyncio.set_event_loop(loop)
loop.run_until_complete(try_do_jobs())
@@ -99,7 +112,6 @@ def loop_in_thread(loop):
t = Thread(target=loop_in_thread, args=(loop,))
t.start()
# clear_up_unfinished_jobs()
# loop = asyncio.get_event_loop()
# loop.run_until_complete(try_do_jobs()) this version seems more at home, but it blocks the thread, fix that?
swagger_server/controllers/marktext_async_controller.py
+79
View File
@@ -0,0 +1,79 @@
import connexion
from swagger_server.models.oznaci_besedilo_async_body import OznaciBesediloAsyncBody # noqa: E501
from swagger_server.requets_db.models.vrsta import (JobManager)
from swagger_server.utils import txt_utils
def get_text(body): # noqa: E501
"""Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
# noqa: E501
:param body:
:type body: dict | bytes
:rtype: str
"""
if connexion.request.is_json:
body = OznaciBesediloAsyncBody.from_dict(connexion.request.get_json()) # noqa: E501
# conllu = cl_utils.raw_text_to_conllu(body.besedilo)
# return conllu
job, is_old_job = JobManager.create_job(2, body.besedilo)
if job is None:
return "Something went wrong", 500
ret = {'check_job_url': f'{connexion.request.url_root}/job/{job.id}'}
return ret, 200 # Todo: Update swagger to the newest response template later
def get_conllu_from_file_async(file=None): # noqa: E501
"""Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
return 'do some magic!'
def get_conllu_from_file_ocr_async(file=None): # noqa: E501
"""Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
return 'do some magic!'
def get_text_from_doc_async(file=None): # noqa: E501
"""Pretvori datoteko v besedilo, vrača tekst
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
return 'do some magic!'
def get_text_from_file_ocr_async(file=None): # noqa: E501
"""Pretvori datoteko v besedilo s pomočjo ocr razpoznavanja, vrača tekst
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
return 'do some magic!'
swagger_server/controllers/marktext_controller.py
-82
View File
@@ -1,82 +0,0 @@
import connexion
import six
from swagger_server.models.oznaci_besedilo_async_body import OznaciBesediloAsyncBody # noqa: E501
from swagger_server import util
from swagger_server.classla import cl_utils
from swagger_server.requets_db.models.vrsta import (Job, JobManager)
import swagger_server.controllers.doc2text_controller as d2t
def get_text(body): # noqa: E501
"""Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
# noqa: E501
:param body:
:type body: dict | bytes
:rtype: str
"""
if connexion.request.is_json:
body = OznaciBesediloAsyncBody.from_dict(connexion.request.get_json()) # noqa: E501
# conllu = cl_utils.raw_text_to_conllu(body.besedilo)
# return conllu
job, is_old_job = JobManager.create_job(1, body.besedilo)
if job is None:
return "Something went wrong", 500
ret = {'check_job_url': f'{connexion.request.url_root}/job/{job.id}'}
return ret, 200 # Todo: Update swagger to the newest response template later
def get_text_from_file(file=None): # noqa: E501
"""Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
if file is None:
return "No file provided", 400
txt, status = d2t.extract_text_prepResp(file)
# Todo: instead of parsing text here, instead save the file into the tb or locally, and then parsing
# Todo: when the job actually executes (this version of the implementation is temporary)
if status == 200:
job, is_old_job = JobManager.create_job(1, txt)
if job is None:
return "Something went wrong", 500
ret = {'check_job_url': f'{connexion.request.url_root}/job/{job.id}'}
return ret, 200
else:
return "Something went wrong", 500
def get_text_from_file_ocr(file=None): # noqa: E501
"""Pretvori datoteko v besedilo s pomočjo ocr razpoznavanja in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
if file is None:
return "No file provided", 400
txt, status = d2t.ocr_text_prepResp(file)
# Todo: instead of ocr-ing text here, instead save the file into the tb or locally, and then ocr
# Todo: when the job actually executes (this version of the implementation is temporary)
if status == 200:
job, is_old_job = JobManager.create_job(1, txt)
if job is None:
return "Something went wrong", 500
ret = {'check_job_url': f'{connexion.request.url_root}/job/{job.id}'}
return ret, 200
else:
return "Something went wrong", 500
swagger_server/controllers/marktext_sync_controller.py
+80
View File
@@ -0,0 +1,80 @@
from swagger_server.classla import cl_utils
from swagger_server.utils import txt_utils
def datoteka_v_besedilo_in_classla(file=None): # noqa: E501
"""Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vrača conllu
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
if not cl_utils.nlp_loaded:
return "NLP Models still loading up since server restart, please try again later.", 500
if file is None:
return "No file provided", 400
try:
txt, _ = txt_utils.extract_text_prepResp(file)
return cl_utils.raw_text_to_conllu(txt)
except Exception as e:
return str(e), 500
def datoteka_v_besedilo_sync_post(file=None): # noqa: E501
"""Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vrača besedilo
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
if file is None:
return "No file provided", 400
try:
return txt_utils.extract_text_prepResp(file)
except Exception as e:
return str(e), 500
def get_conllu_ocr(file=None): # noqa: E501
"""Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v conllu s pomočjo ocr razpoznavanja
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
if not cl_utils.nlp_loaded:
return "NLP Models still loading up since server restart, please try again later.", 500
if file is None:
return "No file provided", 400
try:
txt, _ = txt_utils.ocr_text_prepResp(file)
return cl_utils.raw_text_to_conllu(txt)
except Exception as e:
return str(e), 500
def get_text_ocr(file=None): # noqa: E501
"""Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo s pomočjo ocr razpoznavanja
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
if file is None:
return "No file provided", 400
try:
return txt_utils.ocr_text_prepResp(file)
except Exception as e:
return str(e), 500
swagger_server/controllers/oss_controller.py
-6
View File
@@ -1,11 +1,5 @@
import connexion
import six
import os
from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat # noqa: E501
from swagger_server import util
from flask import send_file
from swagger_server.db_utils import Ngrams_Manager
def get_conllus(leta, vrste, kljucnebesede, cerifpodrocja): # noqa: E501
swagger_server/models/__init__.py
+10 -8
View File
@@ -3,13 +3,15 @@
# flake8: noqa
from __future__ import absolute_import
# import models into model package
from swagger_server.models.izlusci_body import IzlusciBody
from swagger_server.models.oznaci_besedilo_async_body import OznaciBesediloAsyncBody
from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat
from swagger_server.models.job_response import JobResponse
from swagger_server.models.datoteka_v_besedilo_body import DatotekaVBesediloBody
from swagger_server.models.datoteka_v_besedilo_ocr_body import DatotekaVBesediloOcrBody
from swagger_server.models.datoteka_v_besedilo_async_body import DatotekaVBesediloAsyncBody
from swagger_server.models.datoteka_v_besedilo_async_ocr_body import DatotekaVBesediloAsyncOcrBody
from swagger_server.models.datoteka_v_besedilo_sync_body import DatotekaVBesediloSyncBody
from swagger_server.models.datoteka_v_besedilo_sync_ocr_body import DatotekaVBesediloSyncOcrBody
from swagger_server.models.datoteka_v_conllu_async_body import DatotekaVConlluAsyncBody
from swagger_server.models.datoteka_v_conllu_async_ocr_body import DatotekaVConlluAsyncOcrBody
from swagger_server.models.datoteka_v_conllu_sync_body import DatotekaVConlluSyncBody
from swagger_server.models.datoteka_v_conllu_sync_ocr_body import DatotekaVConlluSyncOcrBody
from swagger_server.models.pretvori_datoteko_in_oznaci_async_body import PretvoriDatotekoInOznaciAsyncBody
from swagger_server.models.pretvori_datoteko_in_oznaci_async_ocr_body import PretvoriDatotekoInOznaciAsyncOcrBody
from swagger_server.models.izlusci_body import IzlusciBody
from swagger_server.models.job_response import JobResponse
from swagger_server.models.oznaci_besedilo_async_body import OznaciBesediloAsyncBody
from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat
swagger_server/models/pretvori_datoteko_in_oznaci_async_ocr_body.py → swagger_server/models/datoteka_v_besedilo_async_body.py
+10 -10
View File
@@ -9,15 +9,15 @@ from swagger_server.models.base_model_ import Model
from swagger_server import util
class PretvoriDatotekoInOznaciAsyncOcrBody(Model):
class DatotekaVBesediloAsyncBody(Model):
"""NOTE: This class is auto generated by the swagger code generator program.
Do not edit the class manually.
"""
def __init__(self, file: str=None): # noqa: E501
"""PretvoriDatotekoInOznaciAsyncOcrBody - a model defined in Swagger
"""DatotekaVBesediloAsyncBody - a model defined in Swagger
:param file: The file of this PretvoriDatotekoInOznaciAsyncOcrBody. # noqa: E501
:param file: The file of this DatotekaVBesediloAsyncBody. # noqa: E501
:type file: str
"""
self.swagger_types = {
@@ -30,32 +30,32 @@ class PretvoriDatotekoInOznaciAsyncOcrBody(Model):
self._file = file
@classmethod
def from_dict(cls, dikt) -> 'PretvoriDatotekoInOznaciAsyncOcrBody':
def from_dict(cls, dikt) -> 'DatotekaVBesediloAsyncBody':
"""Returns the dict as a model
:param dikt: A dict.
:type: dict
:return: The pretvoriDatotekoInOznaciAsync_ocr_body of this PretvoriDatotekoInOznaciAsyncOcrBody. # noqa: E501
:rtype: PretvoriDatotekoInOznaciAsyncOcrBody
:return: The datotekaVBesediloAsync_body of this DatotekaVBesediloAsyncBody. # noqa: E501
:rtype: DatotekaVBesediloAsyncBody
"""
return util.deserialize_model(dikt, cls)
@property
def file(self) -> str:
"""Gets the file of this PretvoriDatotekoInOznaciAsyncOcrBody.
"""Gets the file of this DatotekaVBesediloAsyncBody.
:return: The file of this PretvoriDatotekoInOznaciAsyncOcrBody.
:return: The file of this DatotekaVBesediloAsyncBody.
:rtype: str
"""
return self._file
@file.setter
def file(self, file: str):
"""Sets the file of this PretvoriDatotekoInOznaciAsyncOcrBody.
"""Sets the file of this DatotekaVBesediloAsyncBody.
:param file: The file of this PretvoriDatotekoInOznaciAsyncOcrBody.
:param file: The file of this DatotekaVBesediloAsyncBody.
:type file: str
"""
if file is None:
swagger_server/models/pretvori_datoteko_in_oznaci_async_body.py → swagger_server/models/datoteka_v_besedilo_async_ocr_body.py
+10 -10
View File
@@ -9,15 +9,15 @@ from swagger_server.models.base_model_ import Model
from swagger_server import util
class PretvoriDatotekoInOznaciAsyncBody(Model):
class DatotekaVBesediloAsyncOcrBody(Model):
"""NOTE: This class is auto generated by the swagger code generator program.
Do not edit the class manually.
"""
def __init__(self, file: str=None): # noqa: E501
"""PretvoriDatotekoInOznaciAsyncBody - a model defined in Swagger
"""DatotekaVBesediloAsyncOcrBody - a model defined in Swagger
:param file: The file of this PretvoriDatotekoInOznaciAsyncBody. # noqa: E501
:param file: The file of this DatotekaVBesediloAsyncOcrBody. # noqa: E501
:type file: str
"""
self.swagger_types = {
@@ -30,32 +30,32 @@ class PretvoriDatotekoInOznaciAsyncBody(Model):
self._file = file
@classmethod
def from_dict(cls, dikt) -> 'PretvoriDatotekoInOznaciAsyncBody':
def from_dict(cls, dikt) -> 'DatotekaVBesediloAsyncOcrBody':
"""Returns the dict as a model
:param dikt: A dict.
:type: dict
:return: The pretvoriDatotekoInOznaciAsync_body of this PretvoriDatotekoInOznaciAsyncBody. # noqa: E501
:rtype: PretvoriDatotekoInOznaciAsyncBody
:return: The datotekaVBesediloAsync_ocr_body of this DatotekaVBesediloAsyncOcrBody. # noqa: E501
:rtype: DatotekaVBesediloAsyncOcrBody
"""
return util.deserialize_model(dikt, cls)
@property
def file(self) -> str:
"""Gets the file of this PretvoriDatotekoInOznaciAsyncBody.
"""Gets the file of this DatotekaVBesediloAsyncOcrBody.
:return: The file of this PretvoriDatotekoInOznaciAsyncBody.
:return: The file of this DatotekaVBesediloAsyncOcrBody.
:rtype: str
"""
return self._file
@file.setter
def file(self, file: str):
"""Sets the file of this PretvoriDatotekoInOznaciAsyncBody.
"""Sets the file of this DatotekaVBesediloAsyncOcrBody.
:param file: The file of this PretvoriDatotekoInOznaciAsyncBody.
:param file: The file of this DatotekaVBesediloAsyncOcrBody.
:type file: str
"""
if file is None:
swagger_server/models/datoteka_v_besedilo_body.py → swagger_server/models/datoteka_v_besedilo_sync_body.py
+10 -10
View File
@@ -9,15 +9,15 @@ from swagger_server.models.base_model_ import Model
from swagger_server import util
class DatotekaVBesediloBody(Model):
class DatotekaVBesediloSyncBody(Model):
"""NOTE: This class is auto generated by the swagger code generator program.
Do not edit the class manually.
"""
def __init__(self, file: str=None): # noqa: E501
"""DatotekaVBesediloBody - a model defined in Swagger
"""DatotekaVBesediloSyncBody - a model defined in Swagger
:param file: The file of this DatotekaVBesediloBody. # noqa: E501
:param file: The file of this DatotekaVBesediloSyncBody. # noqa: E501
:type file: str
"""
self.swagger_types = {
@@ -30,32 +30,32 @@ class DatotekaVBesediloBody(Model):
self._file = file
@classmethod
def from_dict(cls, dikt) -> 'DatotekaVBesediloBody':
def from_dict(cls, dikt) -> 'DatotekaVBesediloSyncBody':
"""Returns the dict as a model
:param dikt: A dict.
:type: dict
:return: The datotekaVBesedilo_body of this DatotekaVBesediloBody. # noqa: E501
:rtype: DatotekaVBesediloBody
:return: The datotekaVBesediloSync_body of this DatotekaVBesediloSyncBody. # noqa: E501
:rtype: DatotekaVBesediloSyncBody
"""
return util.deserialize_model(dikt, cls)
@property
def file(self) -> str:
"""Gets the file of this DatotekaVBesediloBody.
"""Gets the file of this DatotekaVBesediloSyncBody.
:return: The file of this DatotekaVBesediloBody.
:return: The file of this DatotekaVBesediloSyncBody.
:rtype: str
"""
return self._file
@file.setter
def file(self, file: str):
"""Sets the file of this DatotekaVBesediloBody.
"""Sets the file of this DatotekaVBesediloSyncBody.
:param file: The file of this DatotekaVBesediloBody.
:param file: The file of this DatotekaVBesediloSyncBody.
:type file: str
"""
if file is None:
swagger_server/models/datoteka_v_besedilo_sync_ocr_body.py
+64
View File
@@ -0,0 +1,64 @@
# coding: utf-8
from __future__ import absolute_import
from datetime import date, datetime # noqa: F401
from typing import List, Dict # noqa: F401
from swagger_server.models.base_model_ import Model
from swagger_server import util
class DatotekaVBesediloSyncOcrBody(Model):
"""NOTE: This class is auto generated by the swagger code generator program.
Do not edit the class manually.
"""
def __init__(self, file: str=None): # noqa: E501
"""DatotekaVBesediloSyncOcrBody - a model defined in Swagger
:param file: The file of this DatotekaVBesediloSyncOcrBody. # noqa: E501
:type file: str
"""
self.swagger_types = {
'file': str
}
self.attribute_map = {
'file': 'file'
}
self._file = file
@classmethod
def from_dict(cls, dikt) -> 'DatotekaVBesediloSyncOcrBody':
"""Returns the dict as a model
:param dikt: A dict.
:type: dict
:return: The datotekaVBesediloSync_ocr_body of this DatotekaVBesediloSyncOcrBody. # noqa: E501
:rtype: DatotekaVBesediloSyncOcrBody
"""
return util.deserialize_model(dikt, cls)
@property
def file(self) -> str:
"""Gets the file of this DatotekaVBesediloSyncOcrBody.
:return: The file of this DatotekaVBesediloSyncOcrBody.
:rtype: str
"""
return self._file
@file.setter
def file(self, file: str):
"""Sets the file of this DatotekaVBesediloSyncOcrBody.
:param file: The file of this DatotekaVBesediloSyncOcrBody.
:type file: str
"""
if file is None:
raise ValueError("Invalid value for `file`, must not be `None`") # noqa: E501
self._file = file
swagger_server/models/datoteka_v_besedilo_ocr_body.py → swagger_server/models/datoteka_v_conllu_async_body.py
+10 -10
View File
@@ -9,15 +9,15 @@ from swagger_server.models.base_model_ import Model
from swagger_server import util
class DatotekaVBesediloOcrBody(Model):
class DatotekaVConlluAsyncBody(Model):
"""NOTE: This class is auto generated by the swagger code generator program.
Do not edit the class manually.
"""
def __init__(self, file: str=None): # noqa: E501
"""DatotekaVBesediloOcrBody - a model defined in Swagger
"""DatotekaVConlluAsyncBody - a model defined in Swagger
:param file: The file of this DatotekaVBesediloOcrBody. # noqa: E501
:param file: The file of this DatotekaVConlluAsyncBody. # noqa: E501
:type file: str
"""
self.swagger_types = {
@@ -30,32 +30,32 @@ class DatotekaVBesediloOcrBody(Model):
self._file = file
@classmethod
def from_dict(cls, dikt) -> 'DatotekaVBesediloOcrBody':
def from_dict(cls, dikt) -> 'DatotekaVConlluAsyncBody':
"""Returns the dict as a model
:param dikt: A dict.
:type: dict
:return: The datotekaVBesedilo_ocr_body of this DatotekaVBesediloOcrBody. # noqa: E501
:rtype: DatotekaVBesediloOcrBody
:return: The datotekaVConlluAsync_body of this DatotekaVConlluAsyncBody. # noqa: E501
:rtype: DatotekaVConlluAsyncBody
"""
return util.deserialize_model(dikt, cls)
@property
def file(self) -> str:
"""Gets the file of this DatotekaVBesediloOcrBody.
"""Gets the file of this DatotekaVConlluAsyncBody.
:return: The file of this DatotekaVBesediloOcrBody.
:return: The file of this DatotekaVConlluAsyncBody.
:rtype: str
"""
return self._file
@file.setter
def file(self, file: str):
"""Sets the file of this DatotekaVBesediloOcrBody.
"""Sets the file of this DatotekaVConlluAsyncBody.
:param file: The file of this DatotekaVBesediloOcrBody.
:param file: The file of this DatotekaVConlluAsyncBody.
:type file: str
"""
if file is None:
swagger_server/models/datoteka_v_conllu_async_ocr_body.py
+64
View File
@@ -0,0 +1,64 @@
# coding: utf-8
from __future__ import absolute_import
from datetime import date, datetime # noqa: F401
from typing import List, Dict # noqa: F401
from swagger_server.models.base_model_ import Model
from swagger_server import util
class DatotekaVConlluAsyncOcrBody(Model):
"""NOTE: This class is auto generated by the swagger code generator program.
Do not edit the class manually.
"""
def __init__(self, file: str=None): # noqa: E501
"""DatotekaVConlluAsyncOcrBody - a model defined in Swagger
:param file: The file of this DatotekaVConlluAsyncOcrBody. # noqa: E501
:type file: str
"""
self.swagger_types = {
'file': str
}
self.attribute_map = {
'file': 'file'
}
self._file = file
@classmethod
def from_dict(cls, dikt) -> 'DatotekaVConlluAsyncOcrBody':
"""Returns the dict as a model
:param dikt: A dict.
:type: dict
:return: The datotekaVConlluAsync_ocr_body of this DatotekaVConlluAsyncOcrBody. # noqa: E501
:rtype: DatotekaVConlluAsyncOcrBody
"""
return util.deserialize_model(dikt, cls)
@property
def file(self) -> str:
"""Gets the file of this DatotekaVConlluAsyncOcrBody.
:return: The file of this DatotekaVConlluAsyncOcrBody.
:rtype: str
"""
return self._file
@file.setter
def file(self, file: str):
"""Sets the file of this DatotekaVConlluAsyncOcrBody.
:param file: The file of this DatotekaVConlluAsyncOcrBody.
:type file: str
"""
if file is None:
raise ValueError("Invalid value for `file`, must not be `None`") # noqa: E501
self._file = file
swagger_server/models/job_response.py
+79 -49
View File
@@ -14,34 +14,39 @@ class JobResponse(Model):
Do not edit the class manually.
"""
def __init__(self, finished_job: bool=None, completed_at: datetime=None, estimated_completion: datetime=None, job_result: str=None): # noqa: E501
def __init__(self, job_status: str=None, finished_on: datetime=None, started_on: datetime=None, created_on: datetime=None, job_result: str=None): # noqa: E501
"""JobResponse - a model defined in Swagger
:param finished_job: The finished_job of this JobResponse. # noqa: E501
:type finished_job: bool
:param completed_at: The completed_at of this JobResponse. # noqa: E501
:type completed_at: datetime
:param estimated_completion: The estimated_completion of this JobResponse. # noqa: E501
:type estimated_completion: datetime
:param job_status: The job_status of this JobResponse. # noqa: E501
:type job_status: str
:param finished_on: The finished_on of this JobResponse. # noqa: E501
:type finished_on: datetime
:param started_on: The started_on of this JobResponse. # noqa: E501
:type started_on: datetime
:param created_on: The created_on of this JobResponse. # noqa: E501
:type created_on: datetime
:param job_result: The job_result of this JobResponse. # noqa: E501
:type job_result: str
"""
self.swagger_types = {
'finished_job': bool,
'completed_at': datetime,
'estimated_completion': datetime,
'job_status': str,
'finished_on': datetime,
'started_on': datetime,
'created_on': datetime,
'job_result': str
}
self.attribute_map = {
'finished_job': 'finished_job',
'completed_at': 'completed_at',
'estimated_completion': 'estimated_completion',
'job_status': 'job_status',
'finished_on': 'finished_on',
'started_on': 'started_on',
'created_on': 'created_on',
'job_result': 'job_result'
}
self._finished_job = finished_job
self._completed_at = completed_at
self._estimated_completion = estimated_completion
self._job_status = job_status
self._finished_on = finished_on
self._started_on = started_on
self._created_on = created_on
self._job_result = job_result
@classmethod
@@ -56,69 +61,94 @@ class JobResponse(Model):
return util.deserialize_model(dikt, cls)
@property
def finished_job(self) -> bool:
"""Gets the finished_job of this JobResponse.
def job_status(self) -> str:
"""Gets the job_status of this JobResponse.
:return: The finished_job of this JobResponse.
:rtype: bool
:return: The job_status of this JobResponse.
:rtype: str
"""
return self._finished_job
return self._job_status
@finished_job.setter
def finished_job(self, finished_job: bool):
"""Sets the finished_job of this JobResponse.
@job_status.setter
def job_status(self, job_status: str):
"""Sets the job_status of this JobResponse.
:param finished_job: The finished_job of this JobResponse.
:type finished_job: bool
:param job_status: The job_status of this JobResponse.
:type job_status: str
"""
if finished_job is None:
raise ValueError("Invalid value for `finished_job`, must not be `None`") # noqa: E501
allowed_values = ["waiting in que", "currently processing", "finished processing"] # noqa: E501
if job_status not in allowed_values:
raise ValueError(
"Invalid value for `job_status` ({0}), must be one of {1}"
.format(job_status, allowed_values)
)
self._finished_job = finished_job
self._job_status = job_status
@property
def completed_at(self) -> datetime:
"""Gets the completed_at of this JobResponse.
def finished_on(self) -> datetime:
"""Gets the finished_on of this JobResponse.
:return: The completed_at of this JobResponse.
:return: The finished_on of this JobResponse.
:rtype: datetime
"""
return self._completed_at
return self._finished_on
@completed_at.setter
def completed_at(self, completed_at: datetime):
"""Sets the completed_at of this JobResponse.
@finished_on.setter
def finished_on(self, finished_on: datetime):
"""Sets the finished_on of this JobResponse.
:param completed_at: The completed_at of this JobResponse.
:type completed_at: datetime
:param finished_on: The finished_on of this JobResponse.
:type finished_on: datetime
"""
self._completed_at = completed_at
self._finished_on = finished_on
@property
def estimated_completion(self) -> datetime:
"""Gets the estimated_completion of this JobResponse.
def started_on(self) -> datetime:
"""Gets the started_on of this JobResponse.
:return: The estimated_completion of this JobResponse.
:return: The started_on of this JobResponse.
:rtype: datetime
"""
return self._estimated_completion
return self._started_on
@estimated_completion.setter
def estimated_completion(self, estimated_completion: datetime):
"""Sets the estimated_completion of this JobResponse.
@started_on.setter
def started_on(self, started_on: datetime):
"""Sets the started_on of this JobResponse.
:param estimated_completion: The estimated_completion of this JobResponse.
:type estimated_completion: datetime
:param started_on: The started_on of this JobResponse.
:type started_on: datetime
"""
self._estimated_completion = estimated_completion
self._started_on = started_on
@property
def created_on(self) -> datetime:
"""Gets the created_on of this JobResponse.
:return: The created_on of this JobResponse.
:rtype: datetime
"""
return self._created_on
@created_on.setter
def created_on(self, created_on: datetime):
"""Sets the created_on of this JobResponse.
:param created_on: The created_on of this JobResponse.
:type created_on: datetime
"""
self._created_on = created_on
@property
def job_result(self) -> str:
swagger_server/requets_db/models/vrsta.py
+2 -2
View File
@@ -28,13 +28,13 @@ class BaseModel(Model):
class Job(BaseModel):
id = AutoField()
job_type = IntegerField() # 1 = oznaci besedilo
job_type = IntegerField() # 1 = pretvori datoteko v besedilo, 2 = oznaci besedilo, 21 = oboje
job_input = TextField(index=True)
job_output = TextField(null=True)
created_on = DateTimeField(default=datetime.utcnow)
finished_on = DateTimeField(null=True)
input_size = IntegerField()
started_on = DateTimeField(null=True)
input_size = IntegerField()
# db.drop_tables([Job])
swagger_server/swagger/swagger.yaml
+129 -51
View File
@@ -21,15 +21,6 @@ paths:
schema:
type: integer
format: int64
- name: show_estimated_completion
in: query
description: Calculate estimate time remaining based on various factors (could
be inaccurate)
required: false
style: form
explode: true
schema:
type: boolean
responses:
"200":
description: OK
@@ -38,12 +29,35 @@ paths:
schema:
$ref: '#/components/schemas/JobResponse'
x-openapi-router-controller: swagger_server.controllers.jobs_controller
delete:
tags:
- jobs
summary: Izbriše job
operationId: delete_job
parameters:
- name: job_id
in: path
required: true
style: simple
explode: false
schema:
type: integer
format: int64
responses:
"200":
description: OK
content:
'*/*':
schema:
type: string
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.jobs_controller
/oznaciBesediloAsync:
post:
tags:
- marktext
summary: Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne
conll-u format
- marktext-async
summary: Označi surovo (angl. raw) besedilo s classlo/stanzo z uporabo slovenskih
modelov ter vrne conll-u format
operationId: get_text
requestBody:
content:
@@ -59,19 +73,39 @@ paths:
schema:
type: string
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.marktext_controller
/pretvoriDatotekoInOznaciAsync:
x-openapi-router-controller: swagger_server.controllers.marktext_async_controller
/datotekaVBesediloAsync:
post:
tags:
- marktext
- marktext-async
summary: "Pretvori datoteko v besedilo, vrača tekst"
operationId: get_text_from_doc_async
requestBody:
content:
multipart/form-data:
schema:
$ref: '#/components/schemas/datotekaVBesediloAsync_body'
responses:
"200":
description: OK
content:
'*/*':
schema:
type: string
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.marktext_async_controller
/datotekaVConlluAsync:
post:
tags:
- marktext-async
summary: Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih
modelov ter vrne conll-u format
operationId: get_text_from_file
operationId: get_conllu_from_file_async
requestBody:
content:
multipart/form-data:
schema:
$ref: '#/components/schemas/pretvoriDatotekoInOznaciAsync_body'
$ref: '#/components/schemas/datotekaVConlluAsync_body'
responses:
"200":
description: OK
@@ -80,19 +114,19 @@ paths:
schema:
type: string
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.marktext_controller
/pretvoriDatotekoInOznaciAsync/ocr:
x-openapi-router-controller: swagger_server.controllers.marktext_async_controller
/datotekaVConlluAsync/ocr:
post:
tags:
- marktext
summary: Pretvori datoteko v besedilo s pomočjo ocr razpoznavanja in označi
s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
operationId: get_text_from_file_ocr
- marktext-async
summary: Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih
modelov ter vrne conll-u format
operationId: get_conllu_from_file_ocr_async
requestBody:
content:
multipart/form-data:
schema:
$ref: '#/components/schemas/pretvoriDatotekoInOznaciAsync_ocr_body'
$ref: '#/components/schemas/datotekaVConlluAsync_ocr_body'
responses:
"200":
description: OK
@@ -101,7 +135,27 @@ paths:
schema:
type: string
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.marktext_controller
x-openapi-router-controller: swagger_server.controllers.marktext_async_controller
/datotekaVBesediloAsync/ocr:
post:
tags:
- marktext-async
summary: "Pretvori datoteko v besedilo s pomočjo ocr razpoznavanja, vrača tekst"
operationId: get_text_from_file_ocr_async
requestBody:
content:
multipart/form-data:
schema:
$ref: '#/components/schemas/datotekaVBesediloAsync_ocr_body'
responses:
"200":
description: OK
content:
'*/*':
schema:
type: string
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.marktext_async_controller
/izlusci:
post:
tags:
@@ -125,17 +179,17 @@ paths:
$ref: '#/components/schemas/TerminoloskiKandidat'
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.extract_controller
/datotekaVBesedilo:
/datotekaVBesediloSync:
post:
tags:
- doc-2text
summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vraca besedilo"
operationId: datoteka_v_besedilo_post
- marktext-sync
summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vrača besedilo"
operationId: datoteka_v_besedilo_sync_post
requestBody:
content:
multipart/form-data:
schema:
$ref: '#/components/schemas/datotekaVBesedilo_body'
$ref: '#/components/schemas/datotekaVBesediloSync_body'
responses:
"200":
description: OK
@@ -144,11 +198,11 @@ paths:
schema:
type: string
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.doc2text_controller
/datotekaVBesedilo/ocr:
x-openapi-router-controller: swagger_server.controllers.marktext_sync_controller
/datotekaVBesediloSync/ocr:
post:
tags:
- doc-2text
- marktext-sync
summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo\
\ s pomočjo ocr razpoznavanja"
operationId: get_text_ocr
@@ -156,7 +210,7 @@ paths:
content:
multipart/form-data:
schema:
$ref: '#/components/schemas/datotekaVBesedilo_ocr_body'
$ref: '#/components/schemas/datotekaVBesediloSync_ocr_body'
responses:
"200":
description: OK
@@ -165,12 +219,12 @@ paths:
schema:
type: string
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.doc2text_controller
x-openapi-router-controller: swagger_server.controllers.marktext_sync_controller
/datotekaVConlluSync:
post:
tags:
- doc-2text
summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vraca conllu"
- marktext-sync
summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vrača conllu"
operationId: datoteka_v_besedilo_in_classla
requestBody:
content:
@@ -185,11 +239,11 @@ paths:
schema:
type: string
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.doc2text_controller
x-openapi-router-controller: swagger_server.controllers.marktext_sync_controller
/datotekaVConlluSync/ocr:
post:
tags:
- doc-2text
- marktext-sync
summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v conllu s\
\ pomočjo ocr razpoznavanja"
operationId: get_conllu_ocr
@@ -206,7 +260,7 @@ paths:
schema:
type: string
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.doc2text_controller
x-openapi-router-controller: swagger_server.controllers.marktext_sync_controller
/oss/steviloBesedilPoIskanju:
get:
tags:
@@ -605,27 +659,35 @@ components:
- finished_job
type: object
properties:
finished_job:
type: boolean
completed_at:
job_status:
type: string
enum:
- waiting in que
- currently processing
- finished processing
finished_on:
type: string
format: date-time
estimated_completion:
started_on:
type: string
format: date-time
created_on:
type: string
format: date-time
job_result:
type: string
example:
completed_at: 2000-01-23T04:56:07.000+00:00
estimated_completion: 2000-01-23T04:56:07.000+00:00
job_status: finished processing
started_on: 2000-01-23T04:56:07.000+00:00
created_on: 2000-01-23T04:56:07.000+00:00
finished_on: 2000-01-23T04:56:07.000+00:00
job_result: job_result
finished_job: true
oznaciBesediloAsync_body:
type: object
properties:
besedilo:
type: string
pretvoriDatotekoInOznaciAsync_body:
datotekaVBesediloAsync_body:
required:
- file
type: object
@@ -633,7 +695,23 @@ components:
file:
type: string
format: binary
pretvoriDatotekoInOznaciAsync_ocr_body:
datotekaVConlluAsync_body:
required:
- file
type: object
properties:
file:
type: string
format: binary
datotekaVConlluAsync_ocr_body:
required:
- file
type: object
properties:
file:
type: string
format: binary
datotekaVBesediloAsync_ocr_body:
required:
- file
type: object
@@ -652,7 +730,7 @@ components:
type: array
items:
type: string
datotekaVBesedilo_body:
datotekaVBesediloSync_body:
required:
- file
type: object
@@ -660,7 +738,7 @@ components:
file:
type: string
format: binary
datotekaVBesedilo_ocr_body:
datotekaVBesediloSync_ocr_body:
required:
- file
type: object
swagger_server/test/__init__.py
-16
View File
@@ -1,16 +0,0 @@
import logging
import connexion
from flask_testing import TestCase
from swagger_server.encoder import JSONEncoder
class BaseTestCase(TestCase):
def create_app(self):
logging.getLogger('connexion.operation').setLevel('ERROR')
app = connexion.App(__name__, specification_dir='../swagger/')
app.app.json_encoder = JSONEncoder
app.add_api('swagger.yaml')
return app.app
swagger_server/test/__main__.py
+16
View File
@@ -0,0 +1,16 @@
#!/usr/bin/env python3
import connexion
from swagger_server import encoder
def main():
app = connexion.App(__name__, specification_dir='./swagger/')
app.app.json_encoder = encoder.JSONEncoder
app.add_api('swagger.yaml', arguments={'title': 'OpenAPI definition'}, pythonic_params=True)
app.run(port=8080)
if __name__ == '__main__':
main()
swagger_server/test/encoder.py
+20
View File
@@ -0,0 +1,20 @@
from connexion.apps.flask_app import FlaskJSONEncoder
import six
from swagger_server.models.base_model_ import Model
class JSONEncoder(FlaskJSONEncoder):
include_nulls = False
def default(self, o):
if isinstance(o, Model):
dikt = {}
for attr, _ in six.iteritems(o.swagger_types):
value = getattr(o, attr)
if value is None and not self.include_nulls:
continue
attr = o.attribute_map[attr]
dikt[attr] = value
return dikt
return FlaskJSONEncoder.default(self, o)
swagger_server/test/test_doc2text_controller.py
-45
View File
@@ -1,45 +0,0 @@
# coding: utf-8
from __future__ import absolute_import
from flask import json
from six import BytesIO
from swagger_server.test import BaseTestCase
class TestDoc2textController(BaseTestCase):
"""Doc2textController integration test stubs"""
def test_datoteka_v_besedilo_post(self):
"""Test case for datoteka_v_besedilo_post
Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo
"""
data = dict(file='file_example')
response = self.client.open(
'/datotekaVBesedilo/',
method='POST',
data=data,
content_type='multipart/form-data')
self.assert200(response,
'Response body is : ' + response.data.decode('utf-8'))
def test_get_text_ocr(self):
"""Test case for get_text_ocr
Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo s pomočjo ocr razpoznavanja
"""
data = dict(file='file_example')
response = self.client.open(
'/datotekaVBesedilo/ocr',
method='POST',
data=data,
content_type='multipart/form-data')
self.assert200(response,
'Response body is : ' + response.data.decode('utf-8'))
if __name__ == '__main__':
import unittest
unittest.main()
swagger_server/test/test_extract_controller.py
-33
View File
@@ -1,33 +0,0 @@
# coding: utf-8
from __future__ import absolute_import
from flask import json
from six import BytesIO
from swagger_server.models.izlusci_body import IzlusciBody # noqa: E501
from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat # noqa: E501
from swagger_server.test import BaseTestCase
class TestExtractController(BaseTestCase):
"""ExtractController integration test stubs"""
def test_get_candidates(self):
"""Test case for get_candidates
Izlusci terminološke kandidate iz seznama besedil v conllu obliki
"""
body = IzlusciBody()
response = self.client.open(
'/izlusci',
method='POST',
data=json.dumps(body),
content_type='application/json')
self.assert200(response,
'Response body is : ' + response.data.decode('utf-8'))
if __name__ == '__main__':
import unittest
unittest.main()
swagger_server/test/test_jobs_controller.py
-32
View File
@@ -1,32 +0,0 @@
# coding: utf-8
from __future__ import absolute_import
from flask import json
from six import BytesIO
from swagger_server.models.job_response import JobResponse # noqa: E501
from swagger_server.test import BaseTestCase
class TestJobsController(BaseTestCase):
"""JobsController integration test stubs"""
def test_get_job_status(self):
"""Test case for get_job_status
Vrne status
"""
query_string = [('job_id', 789),
('show_estimated_completion', True)]
response = self.client.open(
'/job',
method='GET',
query_string=query_string)
self.assert200(response,
'Response body is : ' + response.data.decode('utf-8'))
if __name__ == '__main__':
import unittest
unittest.main()
swagger_server/test/test_marktext_controller.py
-32
View File
@@ -1,32 +0,0 @@
# coding: utf-8
from __future__ import absolute_import
from flask import json
from six import BytesIO
from swagger_server.models.oznaci_besedilo_async_body import OznaciBesediloAsyncBody # noqa: E501
from swagger_server.test import BaseTestCase
class TestMarktextController(BaseTestCase):
"""MarktextController integration test stubs"""
def test_get_text(self):
"""Test case for get_text
Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
"""
body = OznaciBesediloAsyncBody()
response = self.client.open(
'/oznaciBesediloAsync',
method='POST',
data=json.dumps(body),
content_type='application/json')
self.assert200(response,
'Response body is : ' + response.data.decode('utf-8'))
if __name__ == '__main__':
import unittest
unittest.main()
swagger_server/test/test_oss_controller.py
-137
View File
@@ -1,137 +0,0 @@
# coding: utf-8
from __future__ import absolute_import
from flask import json
from six import BytesIO
from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat # noqa: E501
from swagger_server.test import BaseTestCase
class TestOssController(BaseTestCase):
"""OssController integration test stubs"""
def test_get_conllu(self):
"""Test case for get_conllu
Vrne CoNNL-U po id-ju datoteke
"""
query_string = [('file_id', 789)]
response = self.client.open(
'/oss/conlluPoId',
method='GET',
query_string=query_string)
self.assert200(response,
'Response body is : ' + response.data.decode('utf-8'))
def test_get_conllus(self):
"""Test case for get_conllus
Vrne seznam CoNNL-U-jev glede na iskalne pogoje
"""
query_string = [('leta', 56),
('vrste', 'vrste_example'),
('kljucnebesede', 'kljucnebesede_example'),
('cerifpodrocja', 56)]
response = self.client.open(
'/oss/conlluPoIskanju',
method='GET',
query_string=query_string)
self.assert200(response,
'Response body is : ' + response.data.decode('utf-8'))
def test_get_extracted_words(self):
"""Test case for get_extracted_words
Vrne terminloške kandidate glede na
"""
query_string = [('leta', 56),
('vrste', 'vrste_example'),
('kljucnebesede', 'kljucnebesede_example'),
('cerifpodrocja', 56)]
response = self.client.open(
'/oss/izlusciPoIskanju',
method='GET',
query_string=query_string)
self.assert200(response,
'Response body is : ' + response.data.decode('utf-8'))
def test_get_file(self):
"""Test case for get_file
Vrne binarni zapis v originalnem formatu po id-ju datoteke
"""
query_string = [('file_id', 789)]
response = self.client.open(
'/oss/datotekaPoId',
method='GET',
query_string=query_string)
self.assert200(response,
'Response body is : ' + response.data.decode('utf-8'))
def test_get_files(self):
"""Test case for get_files
Vrne seznam binarnih zapisov v originalnem formatu glede na iskalne pogoje
"""
query_string = [('leta', 56),
('vrste', 'vrste_example'),
('kljucnebesede', 'kljucnebesede_example'),
('cerifpodrocja', 56)]
response = self.client.open(
'/oss/datotekePoIskanju',
method='GET',
query_string=query_string)
self.assert200(response,
'Response body is : ' + response.data.decode('utf-8'))
def test_get_number_texts(self):
"""Test case for get_number_texts
Vrne število besedil glede na iskalne pogoje
"""
query_string = [('leta', 56),
('vrste', 'vrste_example'),
('kljucnebesede', 'kljucnebesede_example'),
('cerifpodrocja', 56)]
response = self.client.open(
'/oss/steviloBesedilPoIskanju',
method='GET',
query_string=query_string)
self.assert200(response,
'Response body is : ' + response.data.decode('utf-8'))
def test_get_texts(self):
"""Test case for get_texts
Vrne seznam besedil glede na iskalne pogoje
"""
query_string = [('leta', 56),
('vrste', 'vrste_example'),
('kljucnebesede', 'kljucnebesede_example'),
('cerifpodrocja', 56)]
response = self.client.open(
'/oss/besedilaPoIskanju',
method='GET',
query_string=query_string)
self.assert200(response,
'Response body is : ' + response.data.decode('utf-8'))
def test_oss_besedilo_po_id_get(self):
"""Test case for oss_besedilo_po_id_get
Vrne besedilo po id-ju datoteke
"""
query_string = [('file_id', 789)]
response = self.client.open(
'/oss/besediloPoId',
method='GET',
query_string=query_string)
self.assert200(response,
'Response body is : ' + response.data.decode('utf-8'))
if __name__ == '__main__':
import unittest
unittest.main()
swagger_server/test/type_util.py
+32
View File
@@ -0,0 +1,32 @@
# coding: utf-8
import sys
if sys.version_info < (3, 7):
import typing
def is_generic(klass):
""" Determine whether klass is a generic class """
return type(klass) == typing.GenericMeta
def is_dict(klass):
""" Determine whether klass is a Dict """
return klass.__extra__ == dict
def is_list(klass):
""" Determine whether klass is a List """
return klass.__extra__ == list
else:
def is_generic(klass):
""" Determine whether klass is a generic class """
return hasattr(klass, '__origin__')
def is_dict(klass):
""" Determine whether klass is a Dict """
return klass.__origin__ == dict
def is_list(klass):
""" Determine whether klass is a List """
return klass.__origin__ == list
swagger_server/test/util.py
+142
View File
@@ -0,0 +1,142 @@
import datetime
import six
import typing
from swagger_server import type_util
def _deserialize(data, klass):
"""Deserializes dict, list, str into an object.
:param data: dict, list or str.
:param klass: class literal, or string of class name.
:return: object.
"""
if data is None:
return None
if klass in six.integer_types or klass in (float, str, bool, bytearray):
return _deserialize_primitive(data, klass)
elif klass == object:
return _deserialize_object(data)
elif klass == datetime.date:
return deserialize_date(data)
elif klass == datetime.datetime:
return deserialize_datetime(data)
elif type_util.is_generic(klass):
if type_util.is_list(klass):
return _deserialize_list(data, klass.__args__[0])
if type_util.is_dict(klass):
return _deserialize_dict(data, klass.__args__[1])
else:
return deserialize_model(data, klass)
def _deserialize_primitive(data, klass):
"""Deserializes to primitive type.
:param data: data to deserialize.
:param klass: class literal.
:return: int, long, float, str, bool.
:rtype: int | long | float | str | bool
"""
try:
value = klass(data)
except UnicodeEncodeError:
value = six.u(data)
except TypeError:
value = data
return value
def _deserialize_object(value):
"""Return an original value.
:return: object.
"""
return value
def deserialize_date(string):
"""Deserializes string to date.
:param string: str.
:type string: str
:return: date.
:rtype: date
"""
try:
from dateutil.parser import parse
return parse(string).date()
except ImportError:
return string
def deserialize_datetime(string):
"""Deserializes string to datetime.
The string should be in iso8601 datetime format.
:param string: str.
:type string: str
:return: datetime.
:rtype: datetime
"""
try:
from dateutil.parser import parse
return parse(string)
except ImportError:
return string
def deserialize_model(data, klass):
"""Deserializes list or dict to model.
:param data: dict, list.
:type data: dict | list
:param klass: class literal.
:return: model object.
"""
instance = klass()
if not instance.swagger_types:
return data
for attr, attr_type in six.iteritems(instance.swagger_types):
if data is not None \
and instance.attribute_map[attr] in data \
and isinstance(data, (list, dict)):
value = data[instance.attribute_map[attr]]
setattr(instance, attr, _deserialize(value, attr_type))
return instance
def _deserialize_list(data, boxed_type):
"""Deserializes a list and its elements.
:param data: list to deserialize.
:type data: list
:param boxed_type: class literal.
:return: deserialized list.
:rtype: list
"""
return [_deserialize(sub_data, boxed_type)
for sub_data in data]
def _deserialize_dict(data, boxed_type):
"""Deserializes a dict and its elements.
:param data: dict to deserialize.
:type data: dict
:param boxed_type: class literal.
:return: deserialized dict.
:rtype: dict
"""
return {k: _deserialize(v, boxed_type)
for k, v in six.iteritems(data)}
swagger_server/util.py
+2 -2
View File
@@ -177,8 +177,8 @@ def get_files_by_keywords(kljucnebesede):
ngrams_path = "classla_OS2022/ngrams/" if is_docker() else "../mnt/ssd/ds_ftp/classla_OS2022/ngrams/"
print("Looping trough ngrams")
for path, dirs, files, in os.walk(ngrams_path):
for i, _file in enumerate(files[:200]):
if i % 200 == 0: print(f"{i}/{len(files)}")
for i, _file in enumerate(files[:100]):
if i % 500 == 0: print(f"{i}/{len(files)}")
file = f'{ngrams_path}{_file}'
data = pd.read_csv(file, sep='\t')
amount = len(data[data['ngram_len'] == 1 & data['gram_text'].str.lower().isin(kljucnebesede)])
swagger_server/utils/__init__.py
View File
swagger_server/db_utils.py → swagger_server/utils/db_utils.py
View File
swagger_server/utils/txt_utils.py
+65
View File
@@ -0,0 +1,65 @@
import os.path
import pytesseract
import requests
import docx
import xml.etree.ElementTree as ET
from PyPDF2 import PdfReader
from swagger_server.classla import cl_utils
import cv2
import numpy as np
tika_server = "http://tika2:9999/tika"
# endpoint below to be used only for development purposes (don't need to run docker)
# tika_server = "http://rsdo.lhrs.feri.um.si:9998/tika"
def extract_text_prepResp(file):
if tika_responding():
response = requests.put(tika_server, data=file)
return response.text, 200
if "openxmlformats-officedocument.wordprocessingml.document" in file.content_type:
content = '\n'.join([p.text for p in docx.Document(file).paragraphs])
elif "application/pdf" in file.content_type:
reader = PdfReader(file)
content = '\n'.join([p.extract_text() for p in reader.pages])
content = content
elif "text/xml" in file.content_type:
root = ET.parse(file).getroot()
plainText = root.findall('PlainText')
if len(plainText) == 0:
return "Didn't find anything in PlainText", 400
content = '\n'.join([pt.text for pt in plainText])
# elif "text/plain" in file.content_type:
else:
content = file.read().decode('utf-8')
return content, 200
def ocr_text_prepResp(file):
if tika_responding():
response = requests.put(tika_server, data=file,
headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng"})
return response.text, 200
win_p = "C:/Program Files/Tesseract-OCR/tesseract.exe"
if os.path.exists(win_p):
pytesseract.pytesseract.tesseract_cmd = win_p
# convert string data to numpy array
file_bytes = np.fromstring(file.read(), np.uint8)
# convert numpy array to image
img = cv2.imdecode(file_bytes, cv2.IMREAD_COLOR)
conf = '-l eng+slv'
return pytesseract.image_to_string(img, config=conf), 200
def tika_responding():
try:
ret = requests.get(tika_server)
return ret.status_code == 200
except:
return False