webapps/term_extractor
18
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added two more async endpoints (temporary implementation)

Browse Source
This commit is contained in:
Kikimanox
2022-09-01 18:30:58 +02:00
parent fcf192b00b
commit 729dfb14cf
8 changed files with 589 additions and 264 deletions
Download Patch File Download Diff File Expand all files Collapse all files
swagger_server/controllers/doc2text_controller.py
+6 -6
View File
@@ -24,7 +24,7 @@ tika_server = "http://tika2:9999/tika_NAROBENURLZANALASC"
# tika_server = "http://rsdo.lhrs.feri.um.si:9998/tika"
def extract_text(file):
def extract_text_prepResp(file):
if tika_responding():
response = requests.put(tika_server, data=file)
return response.text, 200
@@ -47,7 +47,7 @@ def extract_text(file):
return content, 200
def ocr_text(file):
def ocr_text_prepResp(file):
if tika_responding():
response = requests.put(tika_server, data=file,
headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng"})
@@ -87,7 +87,7 @@ def datoteka_v_besedilo_post(file=None): # noqa: E501
if file is None:
return "No file provided", 400
try:
return extract_text(file)
return extract_text_prepResp(file)
except Exception as e:
return str(e), 500
@@ -105,7 +105,7 @@ def get_text_ocr(file=None): # noqa: E501
if file is None:
return "No file provided", 400
try:
return ocr_text(file)
return ocr_text_prepResp(file)
except Exception as e:
return str(e), 500
@@ -125,7 +125,7 @@ def datoteka_v_besedilo_in_classla(file=None): # noqa: E501
if file is None:
return "No file provided", 400
try:
txt, _ = extract_text(file)
txt, _ = extract_text_prepResp(file)
return cl_utils.raw_text_to_conllu(txt)
except Exception as e:
return str(e), 500
@@ -151,7 +151,7 @@ def get_conllu_ocr(file=None): # noqa: E501
headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng"})
return cl_utils.raw_text_to_conllu(response.text)
else:
txt, _ = ocr_text(file)
txt, _ = ocr_text_prepResp(file)
return cl_utils.raw_text_to_conllu(txt)
except Exception as e:
return str(e), 500
swagger_server/controllers/marktext_controller.py
+54 -1
View File
@@ -5,6 +5,7 @@ from swagger_server.models.oznaci_besedilo_async_body import OznaciBesediloAsync
from swagger_server import util
from swagger_server.classla import cl_utils
from swagger_server.requets_db.models.vrsta import (Job, JobManager)
import swagger_server.controllers.doc2text_controller as d2t
def get_text(body): # noqa: E501
@@ -26,4 +27,56 @@ def get_text(body): # noqa: E501
return "Something went wrong", 500
ret = {'check_job_url': f'{connexion.request.url_root}/job/{job.id}'}
return ret # Todo: Update swagger to the newest response template later
return ret, 200 # Todo: Update swagger to the newest response template later
def get_text_from_file(file=None): # noqa: E501
"""Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
if file is None:
return "No file provided", 400
txt, status = d2t.extract_text_prepResp(file)
# Todo: instead of parsing text here, instead save the file into the tb or locally, and then parsing
# Todo: when the job actually executes (this version of the implementation is temporary)
if status == 200:
job, is_old_job = JobManager.create_job(1, txt)
if job is None:
return "Something went wrong", 500
ret = {'check_job_url': f'{connexion.request.url_root}/job/{job.id}'}
return ret, 200
else:
return "Something went wrong", 500
def get_text_from_file_ocr(file=None): # noqa: E501
"""Pretvori datoteko v besedilo s pomočjo ocr razpoznavanja in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
if file is None:
return "No file provided", 400
txt, status = d2t.ocr_text_prepResp(file)
# Todo: instead of ocr-ing text here, instead save the file into the tb or locally, and then ocr
# Todo: when the job actually executes (this version of the implementation is temporary)
if status == 200:
job, is_old_job = JobManager.create_job(1, txt)
if job is None:
return "Something went wrong", 500
ret = {'check_job_url': f'{connexion.request.url_root}/job/{job.id}'}
return ret, 200
else:
return "Something went wrong", 500
swagger_server/models/__init__.py
+2
View File
@@ -11,3 +11,5 @@ from swagger_server.models.datoteka_v_besedilo_body import DatotekaVBesediloBody
from swagger_server.models.datoteka_v_besedilo_ocr_body import DatotekaVBesediloOcrBody
from swagger_server.models.datoteka_v_conllu_sync_body import DatotekaVConlluSyncBody
from swagger_server.models.datoteka_v_conllu_sync_ocr_body import DatotekaVConlluSyncOcrBody
from swagger_server.models.pretvori_datoteko_in_oznaci_async_body import PretvoriDatotekoInOznaciAsyncBody
from swagger_server.models.pretvori_datoteko_in_oznaci_async_ocr_body import PretvoriDatotekoInOznaciAsyncOcrBody
swagger_server/models/pretvori_datoteko_in_oznaci_async_body.py
+64
View File
@@ -0,0 +1,64 @@
# coding: utf-8
from __future__ import absolute_import
from datetime import date, datetime # noqa: F401
from typing import List, Dict # noqa: F401
from swagger_server.models.base_model_ import Model
from swagger_server import util
class PretvoriDatotekoInOznaciAsyncBody(Model):
"""NOTE: This class is auto generated by the swagger code generator program.
Do not edit the class manually.
"""
def __init__(self, file: str=None): # noqa: E501
"""PretvoriDatotekoInOznaciAsyncBody - a model defined in Swagger
:param file: The file of this PretvoriDatotekoInOznaciAsyncBody. # noqa: E501
:type file: str
"""
self.swagger_types = {
'file': str
}
self.attribute_map = {
'file': 'file'
}
self._file = file
@classmethod
def from_dict(cls, dikt) -> 'PretvoriDatotekoInOznaciAsyncBody':
"""Returns the dict as a model
:param dikt: A dict.
:type: dict
:return: The pretvoriDatotekoInOznaciAsync_body of this PretvoriDatotekoInOznaciAsyncBody. # noqa: E501
:rtype: PretvoriDatotekoInOznaciAsyncBody
"""
return util.deserialize_model(dikt, cls)
@property
def file(self) -> str:
"""Gets the file of this PretvoriDatotekoInOznaciAsyncBody.
:return: The file of this PretvoriDatotekoInOznaciAsyncBody.
:rtype: str
"""
return self._file
@file.setter
def file(self, file: str):
"""Sets the file of this PretvoriDatotekoInOznaciAsyncBody.
:param file: The file of this PretvoriDatotekoInOznaciAsyncBody.
:type file: str
"""
if file is None:
raise ValueError("Invalid value for `file`, must not be `None`") # noqa: E501
self._file = file
swagger_server/models/pretvori_datoteko_in_oznaci_async_ocr_body.py
+64
View File
@@ -0,0 +1,64 @@
# coding: utf-8
from __future__ import absolute_import
from datetime import date, datetime # noqa: F401
from typing import List, Dict # noqa: F401
from swagger_server.models.base_model_ import Model
from swagger_server import util
class PretvoriDatotekoInOznaciAsyncOcrBody(Model):
"""NOTE: This class is auto generated by the swagger code generator program.
Do not edit the class manually.
"""
def __init__(self, file: str=None): # noqa: E501
"""PretvoriDatotekoInOznaciAsyncOcrBody - a model defined in Swagger
:param file: The file of this PretvoriDatotekoInOznaciAsyncOcrBody. # noqa: E501
:type file: str
"""
self.swagger_types = {
'file': str
}
self.attribute_map = {
'file': 'file'
}
self._file = file
@classmethod
def from_dict(cls, dikt) -> 'PretvoriDatotekoInOznaciAsyncOcrBody':
"""Returns the dict as a model
:param dikt: A dict.
:type: dict
:return: The pretvoriDatotekoInOznaciAsync_ocr_body of this PretvoriDatotekoInOznaciAsyncOcrBody. # noqa: E501
:rtype: PretvoriDatotekoInOznaciAsyncOcrBody
"""
return util.deserialize_model(dikt, cls)
@property
def file(self) -> str:
"""Gets the file of this PretvoriDatotekoInOznaciAsyncOcrBody.
:return: The file of this PretvoriDatotekoInOznaciAsyncOcrBody.
:rtype: str
"""
return self._file
@file.setter
def file(self, file: str):
"""Sets the file of this PretvoriDatotekoInOznaciAsyncOcrBody.
:param file: The file of this PretvoriDatotekoInOznaciAsyncOcrBody.
:type file: str
"""
if file is None:
raise ValueError("Invalid value for `file`, must not be `None`") # noqa: E501
self._file = file
swagger_server/requets_db/models/vrsta.py
+6
View File
@@ -45,6 +45,12 @@ class JobManager:
@staticmethod
def create_job(job_type, job_input) -> Tuple(Job, bool):
"""
:param: job_type
:possibilities:
1 = txt to classla
2 = file to txt and then mark with classla
3 = file with ocr then to classla
:return: Job object, Did already exist boolean
"""
try:
swagger_server/swagger/swagger.yaml
+315 -257
View File
@@ -3,33 +3,33 @@ info:
title: OpenAPI definition
version: v0
servers:
- url: http://localhost:8089
description: Generated server url
- url: http://localhost:8089
description: Generated server url
paths:
/job/{job_id}:
get:
tags:
- jobs
- jobs
summary: Vrne status
operationId: get_job_status
parameters:
- name: job_id
in: path
required: true
style: simple
explode: false
schema:
type: integer
format: int64
- name: show_estimated_completion
in: query
description: Calculate estimate time remaining based on various factors (could
be inaccurate)
required: false
style: form
explode: true
schema:
type: boolean
- name: job_id
in: path
required: true
style: simple
explode: false
schema:
type: integer
format: int64
- name: show_estimated_completion
in: query
description: Calculate estimate time remaining based on various factors (could
be inaccurate)
required: false
style: form
explode: true
schema:
type: boolean
responses:
"200":
description: OK
@@ -41,7 +41,7 @@ paths:
/oznaciBesediloAsync:
post:
tags:
- marktext
- marktext
summary: Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne
conll-u format
operationId: get_text
@@ -60,10 +60,52 @@ paths:
type: string
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.marktext_controller
/pretvoriDatotekoInOznaciAsync:
post:
tags:
- marktext
summary: Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih
modelov ter vrne conll-u format
operationId: get_text_from_file
requestBody:
content:
multipart/form-data:
schema:
$ref: '#/components/schemas/pretvoriDatotekoInOznaciAsync_body'
responses:
"200":
description: OK
content:
'*/*':
schema:
type: string
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.marktext_controller
/pretvoriDatotekoInOznaciAsync/ocr:
post:
tags:
- marktext
summary: Pretvori datoteko v besedilo s pomočjo ocr razpoznavanja in označi
s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
operationId: get_text_from_file_ocr
requestBody:
content:
multipart/form-data:
schema:
$ref: '#/components/schemas/pretvoriDatotekoInOznaciAsync_ocr_body'
responses:
"200":
description: OK
content:
'*/*':
schema:
type: string
x-content-type: '*/*'
x-openapi-router-controller: swagger_server.controllers.marktext_controller
/izlusci:
post:
tags:
- extract
- extract
summary: Izlusci terminološke kandidate iz seznama besedil v conllu obliki
operationId: get_candidates
requestBody:
@@ -86,7 +128,7 @@ paths:
/datotekaVBesedilo:
post:
tags:
- doc-2text
- doc-2text
summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vraca besedilo"
operationId: datoteka_v_besedilo_post
requestBody:
@@ -106,7 +148,7 @@ paths:
/datotekaVBesedilo/ocr:
post:
tags:
- doc-2text
- doc-2text
summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo\
\ s pomočjo ocr razpoznavanja"
operationId: get_text_ocr
@@ -127,7 +169,7 @@ paths:
/datotekaVConlluSync:
post:
tags:
- doc-2text
- doc-2text
summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vraca conllu"
operationId: datoteka_v_besedilo_in_classla
requestBody:
@@ -147,7 +189,7 @@ paths:
/datotekaVConlluSync/ocr:
post:
tags:
- doc-2text
- doc-2text
summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v conllu s\
\ pomočjo ocr razpoznavanja"
operationId: get_conllu_ocr
@@ -168,48 +210,48 @@ paths:
/oss/steviloBesedilPoIskanju:
get:
tags:
- oss
- oss
summary: Vrne število besedil glede na iskalne pogoje
operationId: get_number_texts
parameters:
- name: leta
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: vrste
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: kljucnebesede
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: cerifpodrocja
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: leta
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: vrste
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: kljucnebesede
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: cerifpodrocja
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
responses:
"200":
description: OK
@@ -223,48 +265,48 @@ paths:
/oss/izlusciPoIskanju:
get:
tags:
- oss
- oss
summary: 'Vrne terminloške kandidate glede na '
operationId: get_extracted_words
parameters:
- name: leta
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: vrste
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: kljucnebesede
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: cerifpodrocja
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: leta
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: vrste
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: kljucnebesede
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: cerifpodrocja
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
responses:
"200":
description: OK
@@ -279,49 +321,49 @@ paths:
/oss/datotekePoIskanju:
get:
tags:
- oss
- oss
summary: Vrne seznam binarnih zapisov v originalnem formatu glede na iskalne
pogoje
operationId: get_files
parameters:
- name: leta
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: vrste
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: kljucnebesede
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: cerifpodrocja
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: leta
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: vrste
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: kljucnebesede
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: cerifpodrocja
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
responses:
"200":
description: OK
@@ -339,18 +381,18 @@ paths:
/oss/datotekaPoId:
get:
tags:
- oss
- oss
summary: Vrne binarni zapis v originalnem formatu po id-ju datoteke
operationId: get_file
parameters:
- name: file_id
in: query
required: true
style: form
explode: true
schema:
type: integer
format: int64
- name: file_id
in: query
required: true
style: form
explode: true
schema:
type: integer
format: int64
responses:
"200":
description: OK
@@ -366,48 +408,48 @@ paths:
/oss/conlluPoIskanju:
get:
tags:
- oss
- oss
summary: Vrne seznam CoNNL-U-jev glede na iskalne pogoje
operationId: get_conllus
parameters:
- name: leta
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: vrste
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: kljucnebesede
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: cerifpodrocja
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: leta
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: vrste
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: kljucnebesede
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: cerifpodrocja
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
responses:
"200":
description: OK
@@ -422,18 +464,18 @@ paths:
/oss/conlluPoId:
get:
tags:
- oss
- oss
summary: Vrne CoNNL-U po id-ju datoteke
operationId: get_conllu
parameters:
- name: file_id
in: query
required: true
style: form
explode: true
schema:
type: integer
format: int64
- name: file_id
in: query
required: true
style: form
explode: true
schema:
type: integer
format: int64
responses:
"200":
description: OK
@@ -446,18 +488,18 @@ paths:
/oss/besediloPoId:
get:
tags:
- oss
- oss
summary: Vrne besedilo po id-ju datoteke
operationId: oss_besedilo_po_id_get
parameters:
- name: file_id
in: query
required: true
style: form
explode: true
schema:
type: integer
format: int64
- name: file_id
in: query
required: true
style: form
explode: true
schema:
type: integer
format: int64
responses:
"200":
description: OK
@@ -470,48 +512,48 @@ paths:
/oss/besedilaPoIskanju:
get:
tags:
- oss
- oss
summary: Vrne seznam besedil glede na iskalne pogoje
operationId: get_texts
parameters:
- name: leta
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: vrste
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: kljucnebesede
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: cerifpodrocja
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: leta
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: vrste
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: kljucnebesede
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: cerifpodrocja
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
responses:
"200":
description: OK
@@ -549,18 +591,18 @@ components:
format: int64
example:
podporneutezi:
- 6.0274563
- 6.0274563
- 6.0274563
- 6.0274563
pogostostpojavljanja:
- 1
- 1
- 1
- 1
POSoznake: POSoznake
kanonicnaoblika: kanonicnaoblika
nosilnautez: 0.8008282
kandidat: kandidat
JobResponse:
required:
- finished_job
- finished_job
type: object
properties:
finished_job:
@@ -583,6 +625,22 @@ components:
properties:
besedilo:
type: string
pretvoriDatotekoInOznaciAsync_body:
required:
- file
type: object
properties:
file:
type: string
format: binary
pretvoriDatotekoInOznaciAsync_ocr_body:
required:
- file
type: object
properties:
file:
type: string
format: binary
izlusci_body:
type: object
properties:
@@ -596,7 +654,7 @@ components:
type: string
datotekaVBesedilo_body:
required:
- file
- file
type: object
properties:
file:
@@ -604,7 +662,7 @@ components:
format: binary
datotekaVBesedilo_ocr_body:
required:
- file
- file
type: object
properties:
file:
@@ -612,7 +670,7 @@ components:
format: binary
datotekaVConlluSync_body:
required:
- file
- file
type: object
properties:
file:
@@ -620,7 +678,7 @@ components:
format: binary
datotekaVConlluSync_ocr_body:
required:
- file
- file
type: object
properties:
file: