From c30b014a689dfadc0b32458f24648b65e3f66621 Mon Sep 17 00:00:00 2001 From: Kikimanox Date: Wed, 31 Aug 2022 14:49:47 +0200 Subject: [PATCH] Dodana tika & format (v swagger definiciji se ni nic novega v tem commitu) --- rsdo5.json | 2 +- .../controllers/doc2text_controller.py | 45 +- .../controllers/extract_controller.py | 49 +- swagger_server/swagger/swagger.yaml | 514 +++++++++--------- 4 files changed, 316 insertions(+), 294 deletions(-) diff --git a/rsdo5.json b/rsdo5.json index 935b497..1a16fcd 100644 --- a/rsdo5.json +++ b/rsdo5.json @@ -179,7 +179,7 @@ "tags": [ "doc-2text" ], - "summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo", + "summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vraca besedilo", "requestBody": { "content": { "multipart/form-data": { diff --git a/swagger_server/controllers/doc2text_controller.py b/swagger_server/controllers/doc2text_controller.py index fbfb935..1e4f2c4 100644 --- a/swagger_server/controllers/doc2text_controller.py +++ b/swagger_server/controllers/doc2text_controller.py @@ -1,8 +1,10 @@ +from tempfile import TemporaryFile + import connexion import six # import tika import requests - +import json from swagger_server import util import pandas as pd import docx @@ -10,6 +12,11 @@ import codecs import xml.etree.ElementTree as ET from PyPDF2 import PdfReader +tika_server = "http://tika:9998/tika" + + +tika_server = "http://rsdo.lhrs.feri.um.si:9998/tika" + def datoteka_v_besedilo_post(file=None): # noqa: E501 """Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo @@ -21,25 +28,33 @@ def datoteka_v_besedilo_post(file=None): # noqa: E501 :rtype: str """ - #zakaj ? tika server se požene zraven v dodatnem containerju in se datoteka samo pošlje tja, nazaj pa se dobi čisto besedilo... - #tika ima rest api if file is None: return "No file provided", 400 - - tika_server="http://tika:9998/tika" - response=requests.post(tika_server, data=file) - return response.text, 200 + try: + response = requests.put(tika_server, data=file) + return response.text, 200 + except Exception as e: + return str(e), 500 -def get_text_ocr(file=None): # noqa: E501 - +def get_text_ocr(file=None): + """Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo s pomočjo ocr razpoznavanja + + + # noqa: E501 + + :param file: + :type file: strstr + + :rtype: str + """ if file is None: return "No file provided", 400 - - tika_server="http://tika:9998/tika" - #za ocr se zraven samo doda header - response=requests.post(tika_server, data=file,headers={"X-Tika-PDFOcrStrategy":"ocr_only","X-Tika-OCRLanguage":"slv+eng"}) - return response.text, 200 - + try: + response = requests.put(tika_server, data=file, + headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng"}) + return response.text, 200 + except Exception as e: + return str(e), 500 diff --git a/swagger_server/controllers/extract_controller.py b/swagger_server/controllers/extract_controller.py index b8ec2e2..fbe9849 100644 --- a/swagger_server/controllers/extract_controller.py +++ b/swagger_server/controllers/extract_controller.py @@ -19,53 +19,52 @@ def get_candidates(body): # noqa: E501 if connexion.request.is_json: body = IzlusciBody.from_dict(connexion.request.get_json()) # noqa: E501 - - data={ - terminoloski_kandidati:[ - { + data = { + "terminoloski_kandidati": [ + { "POSoznake": "Ncmsn", "kandidat": "vpliv", "kanonicnaoblika": "vpliv", "nosilnautez": 0.8008282, "podporneutezi": [ - 6.0274563, - 6.0274563 + 6.0274563, + 6.0274563 ], - "pogostostpojavljanja": [101,71] - }, - { + "pogostostpojavljanja": [101, 71] + }, + { "POSoznake": "Agpnsg Ncnsg", "kandidat": "bivalen okolje", "kanonicnaoblika": "bivalno okolje", "nosilnautez": 0.60254, "podporneutezi": [ - 3.263, - 2.134 + 3.263, + 2.134 ], - "pogostostpojavljanja": [27,11] - }, - { + "pogostostpojavljanja": [27, 11] + }, + { "POSoznake": "Agpmsny Ncmsn Ncmpg", "kandidat": "motorični status otrok", "kanonicnaoblika": "motorični status otrok", "nosilnautez": 0.3324, "podporneutezi": [ - 1.221, - 3.323 + 1.221, + 3.323 ], - "pogostostpojavljanja": [31,51] - }, - { + "pogostostpojavljanja": [31, 51] + }, + { "POSoznake": "Agpnsn Ncnsn", "kandidat": "diplomsko delo", "kanonicnaoblika": "diplomsko delo", "nosilnautez": 0.2008282, "podporneutezi": [ - 0.883, - 1.02 + 0.883, + 1.02 ], - "pogostostpojavljanja": [1241,111] - } - ] - } + "pogostostpojavljanja": [1241, 111] + } + ] + } return data diff --git a/swagger_server/swagger/swagger.yaml b/swagger_server/swagger/swagger.yaml index be9680a..323ab8d 100644 --- a/swagger_server/swagger/swagger.yaml +++ b/swagger_server/swagger/swagger.yaml @@ -3,33 +3,33 @@ info: title: OpenAPI definition version: v0 servers: -- url: http://localhost:8089 - description: Generated server url + - url: http://localhost:8089 + description: Generated server url paths: /job/{job_id}: get: tags: - - jobs + - jobs summary: Vrne status operationId: get_job_status parameters: - - name: job_id - in: path - required: true - style: simple - explode: false - schema: - type: integer - format: int64 - - name: show_estimated_completion - in: query - description: Calculate estimate time remaining based on various factors (could - be inaccurate) - required: false - style: form - explode: true - schema: - type: boolean + - name: job_id + in: path + required: true + style: simple + explode: false + schema: + type: integer + format: int64 + - name: show_estimated_completion + in: query + description: Calculate estimate time remaining based on various factors (could + be inaccurate) + required: false + style: form + explode: true + schema: + type: boolean responses: "200": description: OK @@ -41,7 +41,7 @@ paths: /oznaciBesedilo: post: tags: - - marktext + - marktext summary: Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format operationId: get_text @@ -63,7 +63,7 @@ paths: /izlusci: post: tags: - - extract + - extract summary: Izlusci terminološke kandidate iz seznama besedil v conllu obliki operationId: get_candidates requestBody: @@ -86,7 +86,7 @@ paths: /datotekaVBesedilo/ocr: post: tags: - - doc-2text + - doc-2text summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo\ \ s pomočjo ocr razpoznavanja" operationId: get_text_ocr @@ -107,7 +107,7 @@ paths: /datotekaVBesedilo/: post: tags: - - doc-2text + - doc-2text summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo" operationId: datoteka_v_besedilo_post requestBody: @@ -127,48 +127,48 @@ paths: /oss/steviloBesedilPoIskanju: get: tags: - - oss + - oss summary: Vrne število besedil glede na iskalne pogoje operationId: get_number_texts parameters: - - name: leta - in: query - required: true - style: form - explode: true - schema: - type: array - items: - type: integer - format: int64 - - name: vrste - in: query - required: true - style: form - explode: true - schema: - type: array - items: - type: string - - name: kljucnebesede - in: query - required: true - style: form - explode: true - schema: - type: array - items: - type: string - - name: cerifpodrocja - in: query - required: true - style: form - explode: true - schema: - type: array - items: - type: integer - format: int64 + - name: leta + in: query + required: true + style: form + explode: true + schema: + type: array + items: + type: integer + format: int64 + - name: vrste + in: query + required: true + style: form + explode: true + schema: + type: array + items: + type: string + - name: kljucnebesede + in: query + required: true + style: form + explode: true + schema: + type: array + items: + type: string + - name: cerifpodrocja + in: query + required: true + style: form + explode: true + schema: + type: array + items: + type: integer + format: int64 responses: "200": description: OK @@ -182,48 +182,48 @@ paths: /oss/izlusciPoIskanju: get: tags: - - oss + - oss summary: 'Vrne terminloške kandidate glede na ' operationId: get_extracted_words parameters: - - name: leta - in: query - required: true - style: form - explode: true - schema: - type: array - items: - type: integer - format: int64 - - name: vrste - in: query - required: true - style: form - explode: true - schema: - type: array - items: - type: string - - name: kljucnebesede - in: query - required: true - style: form - explode: true - schema: - type: array - items: - type: string - - name: cerifpodrocja - in: query - required: true - style: form - explode: true - schema: - type: array - items: - type: integer - format: int64 + - name: leta + in: query + required: true + style: form + explode: true + schema: + type: array + items: + type: integer + format: int64 + - name: vrste + in: query + required: true + style: form + explode: true + schema: + type: array + items: + type: string + - name: kljucnebesede + in: query + required: true + style: form + explode: true + schema: + type: array + items: + type: string + - name: cerifpodrocja + in: query + required: true + style: form + explode: true + schema: + type: array + items: + type: integer + format: int64 responses: "200": description: OK @@ -238,49 +238,49 @@ paths: /oss/datotekePoIskanju: get: tags: - - oss + - oss summary: Vrne seznam binarnih zapisov v originalnem formatu glede na iskalne pogoje operationId: get_files parameters: - - name: leta - in: query - required: true - style: form - explode: true - schema: - type: array - items: - type: integer - format: int64 - - name: vrste - in: query - required: true - style: form - explode: true - schema: - type: array - items: - type: string - - name: kljucnebesede - in: query - required: true - style: form - explode: true - schema: - type: array - items: - type: string - - name: cerifpodrocja - in: query - required: true - style: form - explode: true - schema: - type: array - items: - type: integer - format: int64 + - name: leta + in: query + required: true + style: form + explode: true + schema: + type: array + items: + type: integer + format: int64 + - name: vrste + in: query + required: true + style: form + explode: true + schema: + type: array + items: + type: string + - name: kljucnebesede + in: query + required: true + style: form + explode: true + schema: + type: array + items: + type: string + - name: cerifpodrocja + in: query + required: true + style: form + explode: true + schema: + type: array + items: + type: integer + format: int64 responses: "200": description: OK @@ -298,18 +298,18 @@ paths: /oss/datotekaPoId: get: tags: - - oss + - oss summary: Vrne binarni zapis v originalnem formatu po id-ju datoteke operationId: get_file parameters: - - name: file_id - in: query - required: true - style: form - explode: true - schema: - type: integer - format: int64 + - name: file_id + in: query + required: true + style: form + explode: true + schema: + type: integer + format: int64 responses: "200": description: OK @@ -325,48 +325,48 @@ paths: /oss/conlluPoIskanju: get: tags: - - oss + - oss summary: Vrne seznam CoNNL-U-jev glede na iskalne pogoje operationId: get_conllus parameters: - - name: leta - in: query - required: true - style: form - explode: true - schema: - type: array - items: - type: integer - format: int64 - - name: vrste - in: query - required: true - style: form - explode: true - schema: - type: array - items: - type: string - - name: kljucnebesede - in: query - required: true - style: form - explode: true - schema: - type: array - items: - type: string - - name: cerifpodrocja - in: query - required: true - style: form - explode: true - schema: - type: array - items: - type: integer - format: int64 + - name: leta + in: query + required: true + style: form + explode: true + schema: + type: array + items: + type: integer + format: int64 + - name: vrste + in: query + required: true + style: form + explode: true + schema: + type: array + items: + type: string + - name: kljucnebesede + in: query + required: true + style: form + explode: true + schema: + type: array + items: + type: string + - name: cerifpodrocja + in: query + required: true + style: form + explode: true + schema: + type: array + items: + type: integer + format: int64 responses: "200": description: OK @@ -381,18 +381,18 @@ paths: /oss/conlluPoId: get: tags: - - oss + - oss summary: Vrne CoNNL-U po id-ju datoteke operationId: get_conllu parameters: - - name: file_id - in: query - required: true - style: form - explode: true - schema: - type: integer - format: int64 + - name: file_id + in: query + required: true + style: form + explode: true + schema: + type: integer + format: int64 responses: "200": description: OK @@ -405,18 +405,18 @@ paths: /oss/besediloPoId: get: tags: - - oss + - oss summary: Vrne besedilo po id-ju datoteke operationId: oss_besedilo_po_id_get parameters: - - name: file_id - in: query - required: true - style: form - explode: true - schema: - type: integer - format: int64 + - name: file_id + in: query + required: true + style: form + explode: true + schema: + type: integer + format: int64 responses: "200": description: OK @@ -429,48 +429,48 @@ paths: /oss/besedilaPoIskanju: get: tags: - - oss + - oss summary: Vrne seznam besedil glede na iskalne pogoje operationId: get_texts parameters: - - name: leta - in: query - required: true - style: form - explode: true - schema: - type: array - items: - type: integer - format: int64 - - name: vrste - in: query - required: true - style: form - explode: true - schema: - type: array - items: - type: string - - name: kljucnebesede - in: query - required: true - style: form - explode: true - schema: - type: array - items: - type: string - - name: cerifpodrocja - in: query - required: true - style: form - explode: true - schema: - type: array - items: - type: integer - format: int64 + - name: leta + in: query + required: true + style: form + explode: true + schema: + type: array + items: + type: integer + format: int64 + - name: vrste + in: query + required: true + style: form + explode: true + schema: + type: array + items: + type: string + - name: kljucnebesede + in: query + required: true + style: form + explode: true + schema: + type: array + items: + type: string + - name: cerifpodrocja + in: query + required: true + style: form + explode: true + schema: + type: array + items: + type: integer + format: int64 responses: "200": description: OK @@ -508,18 +508,18 @@ components: format: int64 example: podporneutezi: - - 6.0274563 - - 6.0274563 + - 6.0274563 + - 6.0274563 pogostostpojavljanja: - - 1 - - 1 + - 1 + - 1 POSoznake: POSoznake kanonicnaoblika: kanonicnaoblika nosilnautez: 0.8008282 kandidat: kandidat JobResponse: required: - - finished_job + - finished_job type: object properties: finished_job: @@ -555,7 +555,7 @@ components: type: string datotekaVBesedilo_ocr_body: required: - - file + - file type: object properties: file: @@ -563,7 +563,15 @@ components: format: binary datotekaVBesedilo_body: required: - - file + - file + type: object + properties: + file: + type: string + format: binary + datotekaVBesediloSync_body: + required: + - file type: object properties: file: