From f16c78bbdc1f8bacc748cac1dc6836d1003205b6 Mon Sep 17 00:00:00 2001 From: Kikimanox Date: Wed, 31 Aug 2022 15:37:04 +0200 Subject: [PATCH] Dodana endpointa za sync pridobivanje connlu-jev, ali preprosto file -> text, ali file -> ocr -> text. --- rsdo5.json | 82 ++++++++++++++++++- .../controllers/doc2text_controller.py | 58 +++++++++++-- swagger_server/models/__init__.py | 6 +- .../models/datoteka_v_conllu_sync_body.py | 64 +++++++++++++++ .../models/datoteka_v_conllu_sync_ocr_body.py | 64 +++++++++++++++ swagger_server/swagger/swagger.yaml | 75 ++++++++++++++--- 6 files changed, 323 insertions(+), 26 deletions(-) create mode 100644 swagger_server/models/datoteka_v_conllu_sync_body.py create mode 100644 swagger_server/models/datoteka_v_conllu_sync_ocr_body.py diff --git a/rsdo5.json b/rsdo5.json index 1a16fcd..e90634c 100644 --- a/rsdo5.json +++ b/rsdo5.json @@ -135,6 +135,44 @@ } } }, + "/datotekaVBesedilo": { + "post": { + "tags": [ + "doc-2text" + ], + "summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vraca besedilo", + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "required": [ + "file" + ], + "type": "object", + "properties": { + "file": { + "type": "string", + "format": "binary" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, "/datotekaVBesedilo/ocr": { "post": { "tags": [ @@ -174,12 +212,52 @@ } } }, - "/datotekaVBesedilo/": { + "/datotekaVConlluSync": { "post": { "tags": [ "doc-2text" ], - "summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vraca besedilo", + "summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vraca conllu", + "operationId": "datotekaVBesediloInClassla", + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "required": [ + "file" + ], + "type": "object", + "properties": { + "file": { + "type": "string", + "format": "binary" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/datotekaVConlluSync/ocr": { + "post": { + "tags": [ + "doc-2text" + ], + "summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v conllu s pomočjo ocr razpoznavanja", + "operationId": "getConlluOcr", "requestBody": { "content": { "multipart/form-data": { diff --git a/swagger_server/controllers/doc2text_controller.py b/swagger_server/controllers/doc2text_controller.py index 1e4f2c4..cd4dc2f 100644 --- a/swagger_server/controllers/doc2text_controller.py +++ b/swagger_server/controllers/doc2text_controller.py @@ -2,7 +2,6 @@ from tempfile import TemporaryFile import connexion import six -# import tika import requests import json from swagger_server import util @@ -11,19 +10,20 @@ import docx import codecs import xml.etree.ElementTree as ET from PyPDF2 import PdfReader +from swagger_server.classla import cl_utils tika_server = "http://tika:9998/tika" - -tika_server = "http://rsdo.lhrs.feri.um.si:9998/tika" +# endpoint below to be used only for development purposes (don't need to run docker) +# tika_server = "http://rsdo.lhrs.feri.um.si:9998/tika" def datoteka_v_besedilo_post(file=None): # noqa: E501 - """Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo + """Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vraca besedilo # noqa: E501 - :param file: + :param file: :type file: strstr :rtype: str @@ -37,10 +37,9 @@ def datoteka_v_besedilo_post(file=None): # noqa: E501 return str(e), 500 -def get_text_ocr(file=None): +def get_text_ocr(file=None): # noqa: E501 """Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo s pomočjo ocr razpoznavanja - # noqa: E501 :param file: @@ -48,13 +47,54 @@ def get_text_ocr(file=None): :rtype: str """ - if file is None: return "No file provided", 400 try: response = requests.put(tika_server, data=file, - headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng"}) + headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng"}) return response.text, 200 except Exception as e: return str(e), 500 + +def datoteka_v_besedilo_in_classla(file=None): # noqa: E501 + """Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vraca conllu + + # noqa: E501 + + :param file: + :type file: strstr + + :rtype: str + """ + if not cl_utils.nlp_loaded: + return "NLP Models still loading up since server restart, please retry later.", 500 + if file is None: + return "No file provided", 400 + try: + response = requests.put(tika_server, data=file) + return cl_utils.raw_text_to_conllu(response.text) + except Exception as e: + return str(e), 500 + + +def get_conllu_ocr(file=None): # noqa: E501 + """Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v conllu s pomočjo ocr razpoznavanja + + # noqa: E501 + + :param file: + :type file: strstr + + :rtype: str + """ + if not cl_utils.nlp_loaded: + return "NLP Models still loading up since server restart, please retry later.", 500 + if file is None: + return "No file provided", 400 + try: + response = requests.put(tika_server, data=file, + headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng"}) + return cl_utils.raw_text_to_conllu(response.text) + except Exception as e: + return str(e), 500 diff --git a/swagger_server/models/__init__.py b/swagger_server/models/__init__.py index 49ce18d..dade2a3 100644 --- a/swagger_server/models/__init__.py +++ b/swagger_server/models/__init__.py @@ -3,9 +3,11 @@ # flake8: noqa from __future__ import absolute_import # import models into model package -from swagger_server.models.datoteka_v_besedilo_body import DatotekaVBesediloBody -from swagger_server.models.datoteka_v_besedilo_ocr_body import DatotekaVBesediloOcrBody from swagger_server.models.izlusci_body import IzlusciBody from swagger_server.models.oznaci_besedilo_body import OznaciBesediloBody from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat from swagger_server.models.job_response import JobResponse +from swagger_server.models.datoteka_v_besedilo_body import DatotekaVBesediloBody +from swagger_server.models.datoteka_v_besedilo_ocr_body import DatotekaVBesediloOcrBody +from swagger_server.models.datoteka_v_conllu_sync_body import DatotekaVConlluSyncBody +from swagger_server.models.datoteka_v_conllu_sync_ocr_body import DatotekaVConlluSyncOcrBody diff --git a/swagger_server/models/datoteka_v_conllu_sync_body.py b/swagger_server/models/datoteka_v_conllu_sync_body.py new file mode 100644 index 0000000..0932680 --- /dev/null +++ b/swagger_server/models/datoteka_v_conllu_sync_body.py @@ -0,0 +1,64 @@ +# coding: utf-8 + +from __future__ import absolute_import +from datetime import date, datetime # noqa: F401 + +from typing import List, Dict # noqa: F401 + +from swagger_server.models.base_model_ import Model +from swagger_server import util + + +class DatotekaVConlluSyncBody(Model): + """NOTE: This class is auto generated by the swagger code generator program. + + Do not edit the class manually. + """ + def __init__(self, file: str=None): # noqa: E501 + """DatotekaVConlluSyncBody - a model defined in Swagger + + :param file: The file of this DatotekaVConlluSyncBody. # noqa: E501 + :type file: str + """ + self.swagger_types = { + 'file': str + } + + self.attribute_map = { + 'file': 'file' + } + self._file = file + + @classmethod + def from_dict(cls, dikt) -> 'DatotekaVConlluSyncBody': + """Returns the dict as a model + + :param dikt: A dict. + :type: dict + :return: The datotekaVConlluSync_body of this DatotekaVConlluSyncBody. # noqa: E501 + :rtype: DatotekaVConlluSyncBody + """ + return util.deserialize_model(dikt, cls) + + @property + def file(self) -> str: + """Gets the file of this DatotekaVConlluSyncBody. + + + :return: The file of this DatotekaVConlluSyncBody. + :rtype: str + """ + return self._file + + @file.setter + def file(self, file: str): + """Sets the file of this DatotekaVConlluSyncBody. + + + :param file: The file of this DatotekaVConlluSyncBody. + :type file: str + """ + if file is None: + raise ValueError("Invalid value for `file`, must not be `None`") # noqa: E501 + + self._file = file diff --git a/swagger_server/models/datoteka_v_conllu_sync_ocr_body.py b/swagger_server/models/datoteka_v_conllu_sync_ocr_body.py new file mode 100644 index 0000000..44040b1 --- /dev/null +++ b/swagger_server/models/datoteka_v_conllu_sync_ocr_body.py @@ -0,0 +1,64 @@ +# coding: utf-8 + +from __future__ import absolute_import +from datetime import date, datetime # noqa: F401 + +from typing import List, Dict # noqa: F401 + +from swagger_server.models.base_model_ import Model +from swagger_server import util + + +class DatotekaVConlluSyncOcrBody(Model): + """NOTE: This class is auto generated by the swagger code generator program. + + Do not edit the class manually. + """ + def __init__(self, file: str=None): # noqa: E501 + """DatotekaVConlluSyncOcrBody - a model defined in Swagger + + :param file: The file of this DatotekaVConlluSyncOcrBody. # noqa: E501 + :type file: str + """ + self.swagger_types = { + 'file': str + } + + self.attribute_map = { + 'file': 'file' + } + self._file = file + + @classmethod + def from_dict(cls, dikt) -> 'DatotekaVConlluSyncOcrBody': + """Returns the dict as a model + + :param dikt: A dict. + :type: dict + :return: The datotekaVConlluSync_ocr_body of this DatotekaVConlluSyncOcrBody. # noqa: E501 + :rtype: DatotekaVConlluSyncOcrBody + """ + return util.deserialize_model(dikt, cls) + + @property + def file(self) -> str: + """Gets the file of this DatotekaVConlluSyncOcrBody. + + + :return: The file of this DatotekaVConlluSyncOcrBody. + :rtype: str + """ + return self._file + + @file.setter + def file(self, file: str): + """Sets the file of this DatotekaVConlluSyncOcrBody. + + + :param file: The file of this DatotekaVConlluSyncOcrBody. + :type file: str + """ + if file is None: + raise ValueError("Invalid value for `file`, must not be `None`") # noqa: E501 + + self._file = file diff --git a/swagger_server/swagger/swagger.yaml b/swagger_server/swagger/swagger.yaml index 323ab8d..0220cf3 100644 --- a/swagger_server/swagger/swagger.yaml +++ b/swagger_server/swagger/swagger.yaml @@ -83,6 +83,26 @@ paths: $ref: '#/components/schemas/TerminoloskiKandidat' x-content-type: '*/*' x-openapi-router-controller: swagger_server.controllers.extract_controller + /datotekaVBesedilo: + post: + tags: + - doc-2text + summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vraca besedilo" + operationId: datoteka_v_besedilo_post + requestBody: + content: + multipart/form-data: + schema: + $ref: '#/components/schemas/datotekaVBesedilo_body' + responses: + "200": + description: OK + content: + '*/*': + schema: + type: string + x-content-type: '*/*' + x-openapi-router-controller: swagger_server.controllers.doc2text_controller /datotekaVBesedilo/ocr: post: tags: @@ -104,17 +124,38 @@ paths: type: string x-content-type: '*/*' x-openapi-router-controller: swagger_server.controllers.doc2text_controller - /datotekaVBesedilo/: + /datotekaVConlluSync: post: tags: - doc-2text - summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo" - operationId: datoteka_v_besedilo_post + summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vraca conllu" + operationId: datoteka_v_besedilo_in_classla requestBody: content: multipart/form-data: schema: - $ref: '#/components/schemas/datotekaVBesedilo_body' + $ref: '#/components/schemas/datotekaVConlluSync_body' + responses: + "200": + description: OK + content: + '*/*': + schema: + type: string + x-content-type: '*/*' + x-openapi-router-controller: swagger_server.controllers.doc2text_controller + /datotekaVConlluSync/ocr: + post: + tags: + - doc-2text + summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v conllu s\ + \ pomočjo ocr razpoznavanja" + operationId: get_conllu_ocr + requestBody: + content: + multipart/form-data: + schema: + $ref: '#/components/schemas/datotekaVConlluSync_ocr_body' responses: "200": description: OK @@ -553,14 +594,6 @@ components: type: array items: type: string - datotekaVBesedilo_ocr_body: - required: - - file - type: object - properties: - file: - type: string - format: binary datotekaVBesedilo_body: required: - file @@ -569,7 +602,23 @@ components: file: type: string format: binary - datotekaVBesediloSync_body: + datotekaVBesedilo_ocr_body: + required: + - file + type: object + properties: + file: + type: string + format: binary + datotekaVConlluSync_body: + required: + - file + type: object + properties: + file: + type: string + format: binary + datotekaVConlluSync_ocr_body: required: - file type: object