webapps/term_extractor
18
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added two more async endpoints (temporary implementation)

Browse Source
This commit is contained in:
Kikimanox
2022-09-01 18:30:58 +02:00
parent fcf192b00b
commit 729dfb14cf
8 changed files with 589 additions and 264 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rsdo5.json
+78
View File
@@ -87,6 +87,84 @@
}
}
},
"/pretvoriDatotekoInOznaciAsync": {
"post": {
"tags": [
"marktext"
],
"summary": "Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format",
"operationId": "getTextFromFile",
"requestBody": {
"content": {
"multipart/form-data": {
"schema": {
"required": [
"file"
],
"type": "object",
"properties": {
"file": {
"type": "string",
"format": "binary"
}
}
}
}
}
},
"responses": {
"200": {
"description": "OK",
"content": {
"*/*": {
"schema": {
"type": "string"
}
}
}
}
}
}
},
"/pretvoriDatotekoInOznaciAsync/ocr": {
"post": {
"tags": [
"marktext"
],
"summary": "Pretvori datoteko v besedilo s pomočjo ocr razpoznavanja in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format",
"operationId": "getTextFromFileOcr",
"requestBody": {
"content": {
"multipart/form-data": {
"schema": {
"required": [
"file"
],
"type": "object",
"properties": {
"file": {
"type": "string",
"format": "binary"
}
}
}
}
}
},
"responses": {
"200": {
"description": "OK",
"content": {
"*/*": {
"schema": {
"type": "string"
}
}
}
}
}
}
},
"/izlusci": {
"post": {
"tags": [
swagger_server/controllers/doc2text_controller.py
+6 -6
View File
@@ -24,7 +24,7 @@ tika_server = "http://tika2:9999/tika_NAROBENURLZANALASC"
# tika_server = "http://rsdo.lhrs.feri.um.si:9998/tika"
def extract_text(file):
def extract_text_prepResp(file):
if tika_responding():
response = requests.put(tika_server, data=file)
return response.text, 200
@@ -47,7 +47,7 @@ def extract_text(file):
return content, 200
def ocr_text(file):
def ocr_text_prepResp(file):
if tika_responding():
response = requests.put(tika_server, data=file,
headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng"})
@@ -87,7 +87,7 @@ def datoteka_v_besedilo_post(file=None): # noqa: E501
if file is None:
return "No file provided", 400
try:
return extract_text(file)
return extract_text_prepResp(file)
except Exception as e:
return str(e), 500
@@ -105,7 +105,7 @@ def get_text_ocr(file=None): # noqa: E501
if file is None:
return "No file provided", 400
try:
return ocr_text(file)
return ocr_text_prepResp(file)
except Exception as e:
return str(e), 500
@@ -125,7 +125,7 @@ def datoteka_v_besedilo_in_classla(file=None): # noqa: E501
if file is None:
return "No file provided", 400
try:
txt, _ = extract_text(file)
txt, _ = extract_text_prepResp(file)
return cl_utils.raw_text_to_conllu(txt)
except Exception as e:
return str(e), 500
@@ -151,7 +151,7 @@ def get_conllu_ocr(file=None): # noqa: E501
headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng"})
return cl_utils.raw_text_to_conllu(response.text)
else:
txt, _ = ocr_text(file)
txt, _ = ocr_text_prepResp(file)
return cl_utils.raw_text_to_conllu(txt)
except Exception as e:
return str(e), 500
swagger_server/controllers/marktext_controller.py
+54 -1
View File
@@ -5,6 +5,7 @@ from swagger_server.models.oznaci_besedilo_async_body import OznaciBesediloAsync
from swagger_server import util
from swagger_server.classla import cl_utils
from swagger_server.requets_db.models.vrsta import (Job, JobManager)
import swagger_server.controllers.doc2text_controller as d2t
def get_text(body): # noqa: E501
@@ -26,4 +27,56 @@ def get_text(body): # noqa: E501
return "Something went wrong", 500
ret = {'check_job_url': f'{connexion.request.url_root}/job/{job.id}'}
return ret # Todo: Update swagger to the newest response template later
return ret, 200 # Todo: Update swagger to the newest response template later
def get_text_from_file(file=None): # noqa: E501
"""Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
if file is None:
return "No file provided", 400
txt, status = d2t.extract_text_prepResp(file)
# Todo: instead of parsing text here, instead save the file into the tb or locally, and then parsing
# Todo: when the job actually executes (this version of the implementation is temporary)
if status == 200:
job, is_old_job = JobManager.create_job(1, txt)
if job is None:
return "Something went wrong", 500
ret = {'check_job_url': f'{connexion.request.url_root}/job/{job.id}'}
return ret, 200
else:
return "Something went wrong", 500
def get_text_from_file_ocr(file=None): # noqa: E501
"""Pretvori datoteko v besedilo s pomočjo ocr razpoznavanja in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
if file is None:
return "No file provided", 400
txt, status = d2t.ocr_text_prepResp(file)
# Todo: instead of ocr-ing text here, instead save the file into the tb or locally, and then ocr
# Todo: when the job actually executes (this version of the implementation is temporary)
if status == 200:
job, is_old_job = JobManager.create_job(1, txt)
if job is None:
return "Something went wrong", 500
ret = {'check_job_url': f'{connexion.request.url_root}/job/{job.id}'}
return ret, 200
else:
return "Something went wrong", 500
swagger_server/models/__init__.py
+2
View File
@@ -11,3 +11,5 @@ from swagger_server.models.datoteka_v_besedilo_body import DatotekaVBesediloBody
from swagger_server.models.datoteka_v_besedilo_ocr_body import DatotekaVBesediloOcrBody
from swagger_server.models.datoteka_v_conllu_sync_body import DatotekaVConlluSyncBody
from swagger_server.models.datoteka_v_conllu_sync_ocr_body import DatotekaVConlluSyncOcrBody
from swagger_server.models.pretvori_datoteko_in_oznaci_async_body import PretvoriDatotekoInOznaciAsyncBody
from swagger_server.models.pretvori_datoteko_in_oznaci_async_ocr_body import PretvoriDatotekoInOznaciAsyncOcrBody
swagger_server/models/pretvori_datoteko_in_oznaci_async_body.py
+64
View File
@@ -0,0 +1,64 @@
# coding: utf-8
from __future__ import absolute_import
from datetime import date, datetime # noqa: F401
from typing import List, Dict # noqa: F401
from swagger_server.models.base_model_ import Model
from swagger_server import util
class PretvoriDatotekoInOznaciAsyncBody(Model):
"""NOTE: This class is auto generated by the swagger code generator program.
Do not edit the class manually.
"""
def __init__(self, file: str=None): # noqa: E501
"""PretvoriDatotekoInOznaciAsyncBody - a model defined in Swagger
:param file: The file of this PretvoriDatotekoInOznaciAsyncBody. # noqa: E501
:type file: str
"""
self.swagger_types = {
'file': str
}
self.attribute_map = {
'file': 'file'
}
self._file = file
@classmethod
def from_dict(cls, dikt) -> 'PretvoriDatotekoInOznaciAsyncBody':
"""Returns the dict as a model
:param dikt: A dict.
:type: dict
:return: The pretvoriDatotekoInOznaciAsync_body of this PretvoriDatotekoInOznaciAsyncBody. # noqa: E501
:rtype: PretvoriDatotekoInOznaciAsyncBody
"""
return util.deserialize_model(dikt, cls)
@property
def file(self) -> str:
"""Gets the file of this PretvoriDatotekoInOznaciAsyncBody.
:return: The file of this PretvoriDatotekoInOznaciAsyncBody.
:rtype: str
"""
return self._file
@file.setter
def file(self, file: str):
"""Sets the file of this PretvoriDatotekoInOznaciAsyncBody.
:param file: The file of this PretvoriDatotekoInOznaciAsyncBody.
:type file: str
"""
if file is None:
raise ValueError("Invalid value for `file`, must not be `None`") # noqa: E501
self._file = file
swagger_server/models/pretvori_datoteko_in_oznaci_async_ocr_body.py
+64
View File
@@ -0,0 +1,64 @@
# coding: utf-8
from __future__ import absolute_import
from datetime import date, datetime # noqa: F401
from typing import List, Dict # noqa: F401
from swagger_server.models.base_model_ import Model
from swagger_server import util
class PretvoriDatotekoInOznaciAsyncOcrBody(Model):
"""NOTE: This class is auto generated by the swagger code generator program.
Do not edit the class manually.
"""
def __init__(self, file: str=None): # noqa: E501
"""PretvoriDatotekoInOznaciAsyncOcrBody - a model defined in Swagger
:param file: The file of this PretvoriDatotekoInOznaciAsyncOcrBody. # noqa: E501
:type file: str
"""
self.swagger_types = {
'file': str
}
self.attribute_map = {
'file': 'file'
}
self._file = file
@classmethod
def from_dict(cls, dikt) -> 'PretvoriDatotekoInOznaciAsyncOcrBody':
"""Returns the dict as a model
:param dikt: A dict.
:type: dict
:return: The pretvoriDatotekoInOznaciAsync_ocr_body of this PretvoriDatotekoInOznaciAsyncOcrBody. # noqa: E501
:rtype: PretvoriDatotekoInOznaciAsyncOcrBody
"""
return util.deserialize_model(dikt, cls)
@property
def file(self) -> str:
"""Gets the file of this PretvoriDatotekoInOznaciAsyncOcrBody.
:return: The file of this PretvoriDatotekoInOznaciAsyncOcrBody.
:rtype: str
"""
return self._file
@file.setter
def file(self, file: str):
"""Sets the file of this PretvoriDatotekoInOznaciAsyncOcrBody.
:param file: The file of this PretvoriDatotekoInOznaciAsyncOcrBody.
:type file: str
"""
if file is None:
raise ValueError("Invalid value for `file`, must not be `None`") # noqa: E501
self._file = file
swagger_server/requets_db/models/vrsta.py
+6
View File
@@ -45,6 +45,12 @@ class JobManager:
@staticmethod
def create_job(job_type, job_input) -> Tuple(Job, bool):
"""
:param: job_type
:possibilities:
1 = txt to classla
2 = file to txt and then mark with classla
3 = file with ocr then to classla
:return: Job object, Did already exist boolean
"""
try:
swagger_server/swagger/swagger.yaml
+58
View File
@@ -60,6 +60,48 @@ paths:
type: string
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.marktext_controller
/pretvoriDatotekoInOznaciAsync:
post:
tags:
- marktext
summary: Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih
modelov ter vrne conll-u format
operationId: get_text_from_file
requestBody:
content:
multipart/form-data:
schema:
$ref: '#/components/schemas/pretvoriDatotekoInOznaciAsync_body'
responses:
"200":
description: OK
content:
'*/*':
schema:
type: string
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.marktext_controller
/pretvoriDatotekoInOznaciAsync/ocr:
post:
tags:
- marktext
summary: Pretvori datoteko v besedilo s pomočjo ocr razpoznavanja in označi
s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
operationId: get_text_from_file_ocr
requestBody:
content:
multipart/form-data:
schema:
$ref: '#/components/schemas/pretvoriDatotekoInOznaciAsync_ocr_body'
responses:
"200":
description: OK
content:
'*/*':
schema:
type: string
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.marktext_controller
/izlusci:
post:
tags:
@@ -583,6 +625,22 @@ components:
properties:
besedilo:
type: string
pretvoriDatotekoInOznaciAsync_body:
required:
- file
type: object
properties:
file:
type: string
format: binary
pretvoriDatotekoInOznaciAsync_ocr_body:
required:
- file
type: object
properties:
file:
type: string
format: binary
izlusci_body:
type: object
properties: