diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..5ab638a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,10 @@ +version: '3.1' +services: + flask-serv: + build: . + restart: always + image: flask-server + ports: + - 8080:8080 + volumes: # when pushing to server, remove the first dot here in front of /mnt + - ./mnt/ssd/ds_ftp/classla_OS2022:/usr/src/app/classla_OS2022 \ No newline at end of file diff --git a/rsdo5.json b/rsdo5.json index 77858b9..0ed681d 100644 --- a/rsdo5.json +++ b/rsdo5.json @@ -4,21 +4,30 @@ "title": "OpenAPI definition", "version": "v0" }, - "servers": [{ - "url": "http://localhost:8089", - "description": "Generated server url" - }], + "servers": [ + { + "url": "http://localhost:8089", + "description": "Generated server url" + } + ], "paths": { "/oznaciBesedilo": { "post": { - "tags": ["marktext"], + "tags": [ + "marktext" + ], "summary": "Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format", "operationId": "getText", "requestBody": { "content": { "application/json": { "schema": { - "type": "string" + "type": "object", + "properties": { + "besedilo": { + "type": "string" + } + } } } }, @@ -40,7 +49,9 @@ }, "/izlusci": { "post": { - "tags": ["extract"], + "tags": [ + "extract" + ], "summary": "Izlusci terminološke kandidate iz seznama besedil v conllu obliki", "operationId": "getCandidates", "requestBody": { @@ -86,14 +97,18 @@ }, "/datotekaVBesedilo/ocr": { "post": { - "tags": ["doc-2text"], + "tags": [ + "doc-2text" + ], "summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo s pomočjo ocr razpoznavanja", "operationId": "getTextOcr", "requestBody": { "content": { "multipart/form-data": { "schema": { - "required": ["file"], + "required": [ + "file" + ], "type": "object", "properties": { "file": { @@ -121,13 +136,17 @@ }, "/datotekaVBesedilo/": { "post": { - "tags": ["doc-2text"], + "tags": [ + "doc-2text" + ], "summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo", "requestBody": { "content": { "multipart/form-data": { "schema": { - "required": ["file"], + "required": [ + "file" + ], "type": "object", "properties": { "file": { @@ -155,52 +174,59 @@ }, "/oss/steviloBesedilPoIskanju": { "get": { - "tags": ["oss"], + "tags": [ + "oss" + ], "summary": "Vrne število besedil glede na iskalne pogoje", "operationId": "getNumberTexts", - "parameters": [{ - "name": "leta", - "in": "query", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "integer", - "format": "int64" + "parameters": [ + { + "name": "leta", + "in": "query", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + } + } + }, + { + "name": "vrste", + "in": "query", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "kljucnebesede", + "in": "query", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "cerifpodrocja", + "in": "query", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + } } } - }, { - "name": "vrste", - "in": "query", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - }, { - "name": "kljucnebesede", - "in": "query", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - }, { - "name": "cerifpodrocja", - "in": "query", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "integer", - "format": "int64" - } - } - }], + ], "responses": { "200": { "description": "OK", @@ -218,52 +244,59 @@ }, "/oss/izlusciPoIskanju": { "get": { - "tags": ["oss"], + "tags": [ + "oss" + ], "summary": "Vrne terminloške kandidate glede na ", "operationId": "getExtractedWords", - "parameters": [{ - "name": "leta", - "in": "query", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "integer", - "format": "int64" + "parameters": [ + { + "name": "leta", + "in": "query", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + } + } + }, + { + "name": "vrste", + "in": "query", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "kljucnebesede", + "in": "query", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "cerifpodrocja", + "in": "query", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + } } } - }, { - "name": "vrste", - "in": "query", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - }, { - "name": "kljucnebesede", - "in": "query", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - }, { - "name": "cerifpodrocja", - "in": "query", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "integer", - "format": "int64" - } - } - }], + ], "responses": { "200": { "description": "OK", @@ -283,52 +316,59 @@ }, "/oss/datotekePoIskanju": { "get": { - "tags": ["oss"], + "tags": [ + "oss" + ], "summary": "Vrne seznam binarnih zapisov v originalnem formatu glede na iskalne pogoje", "operationId": "getFiles", - "parameters": [{ - "name": "leta", - "in": "query", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "integer", - "format": "int64" + "parameters": [ + { + "name": "leta", + "in": "query", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + } + } + }, + { + "name": "vrste", + "in": "query", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "kljucnebesede", + "in": "query", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "cerifpodrocja", + "in": "query", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + } } } - }, { - "name": "vrste", - "in": "query", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - }, { - "name": "kljucnebesede", - "in": "query", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - }, { - "name": "cerifpodrocja", - "in": "query", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "integer", - "format": "int64" - } - } - }], + ], "responses": { "200": { "description": "OK", @@ -352,18 +392,22 @@ }, "/oss/datotekaPoId": { "get": { - "tags": ["oss"], + "tags": [ + "oss" + ], "summary": "Vrne binarni zapis v originalnem formatu po id-ju datoteke", "operationId": "getFile", - "parameters": [{ - "name": "file_id", - "in": "query", - "required": true, - "schema": { - "type": "integer", - "format": "int64" + "parameters": [ + { + "name": "file_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } } - }], + ], "responses": { "200": { "description": "OK", @@ -384,52 +428,59 @@ }, "/oss/conlluPoIskanju": { "get": { - "tags": ["oss"], + "tags": [ + "oss" + ], "summary": "Vrne seznam CoNNL-U-jev glede na iskalne pogoje", "operationId": "getConllus", - "parameters": [{ - "name": "leta", - "in": "query", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "integer", - "format": "int64" + "parameters": [ + { + "name": "leta", + "in": "query", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + } + } + }, + { + "name": "vrste", + "in": "query", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "kljucnebesede", + "in": "query", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "cerifpodrocja", + "in": "query", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + } } } - }, { - "name": "vrste", - "in": "query", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - }, { - "name": "kljucnebesede", - "in": "query", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - }, { - "name": "cerifpodrocja", - "in": "query", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "integer", - "format": "int64" - } - } - }], + ], "responses": { "200": { "description": "OK", @@ -449,18 +500,22 @@ }, "/oss/conlluPoId": { "get": { - "tags": ["oss"], + "tags": [ + "oss" + ], "summary": "Vrne CoNNL-U po id-ju datoteke", "operationId": "getConllu", - "parameters": [{ - "name": "file_id", - "in": "query", - "required": true, - "schema": { - "type": "integer", - "format": "int64" + "parameters": [ + { + "name": "file_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } } - }], + ], "responses": { "200": { "description": "OK", @@ -477,17 +532,21 @@ }, "/oss/besediloPoId": { "get": { - "tags": ["oss"], + "tags": [ + "oss" + ], "summary": "Vrne besedilo po id-ju datoteke", - "parameters": [{ - "name": "file_id", - "in": "query", - "required": true, - "schema": { - "type": "integer", - "format": "int64" + "parameters": [ + { + "name": "file_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } } - }], + ], "responses": { "200": { "description": "OK", @@ -504,52 +563,59 @@ }, "/oss/besedilaPoIskanju": { "get": { - "tags": ["oss"], + "tags": [ + "oss" + ], "summary": "Vrne seznam besedil glede na iskalne pogoje", "operationId": "getTexts", - "parameters": [{ - "name": "leta", - "in": "query", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "integer", - "format": "int64" + "parameters": [ + { + "name": "leta", + "in": "query", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + } + } + }, + { + "name": "vrste", + "in": "query", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "kljucnebesede", + "in": "query", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "cerifpodrocja", + "in": "query", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + } } } - }, { - "name": "vrste", - "in": "query", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - }, { - "name": "kljucnebesede", - "in": "query", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - }, { - "name": "cerifpodrocja", - "in": "query", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "integer", - "format": "int64" - } - } - }], + ], "responses": { "200": { "description": "OK", diff --git a/swagger_server/controllers/marktext_controller.py b/swagger_server/controllers/marktext_controller.py index 16fec35..c3da9d4 100644 --- a/swagger_server/controllers/marktext_controller.py +++ b/swagger_server/controllers/marktext_controller.py @@ -1,7 +1,9 @@ import connexion import six +from swagger_server.models.oznaci_besedilo_body import OznaciBesediloBody # noqa: E501 from swagger_server import util +from swagger_server.classla import cl_utils def get_text(body): # noqa: E501 @@ -9,11 +11,13 @@ def get_text(body): # noqa: E501 # noqa: E501 - :param body: + :param body: :type body: dict | bytes :rtype: str """ if connexion.request.is_json: - body = str.from_dict(connexion.request.get_json()) # noqa: E501 - return 'do some magic!' + body = OznaciBesediloBody.from_dict(connexion.request.get_json()) # noqa: E501 + conllu = cl_utils.raw_text_to_conllu(body.besedilo) + return conllu + diff --git a/swagger_server/controllers/oss_controller.py b/swagger_server/controllers/oss_controller.py index ca6a8a8..ae0ad79 100644 --- a/swagger_server/controllers/oss_controller.py +++ b/swagger_server/controllers/oss_controller.py @@ -1,8 +1,11 @@ import connexion import six +import os from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat # noqa: E501 from swagger_server import util +from flask import send_file +from swagger_server.db_utils import Ngrams_Manager def get_conllu(file_id): # noqa: E501 @@ -15,7 +18,17 @@ def get_conllu(file_id): # noqa: E501 :rtype: str """ - return 'do some magic!' + file_path = util.get_conllu_path_by_id(file_id) + + a = Ngrams_Manager.get_by_file_id(file_id) + print(a) + + try: + return send_file(file_path, attachment_filename=f'{file_id}.conllu', as_attachment=True) + except Exception as e: + if "The system cannot find the file specified" in str(e): + return "The conllu with this ID doesn't exist.", 404 + return str(e), 400 def get_conllus(leta, vrste, kljucnebesede, cerifpodrocja): # noqa: E501 diff --git a/swagger_server/db_utils.py b/swagger_server/db_utils.py new file mode 100644 index 0000000..f0b8725 --- /dev/null +++ b/swagger_server/db_utils.py @@ -0,0 +1,52 @@ +import mariadb +import sys + + +# todo: DO NOT PUSH THIS TO GIT +database_info = { + 'database': 'TEMPORARYVALJUSTFORTHECOMMITPURPOSE', + 'host': 'TEMPORARYVALJUSTFORTHECOMMITPURPOSE', + 'port': 'TEMPORARYVALJUSTFORTHECOMMITPURPOSE', + 'user': 'TEMPORARYVALJUSTFORTHECOMMITPURPOSE', + 'password': 'TEMPORARYVALJUSTFORTHECOMMITPURPOSE' +} + +# Connect to MariaDB Platform +try: + conn = mariadb.connect(**database_info) +except mariadb.Error as e: + print(f"Error connecting to MariaDB Platform: {e}") + sys.exit(1) + +# Get Cursor +cur = conn.cursor() + +# class BaseModel(Model): +# class Meta: +# database = db +# +# +# class os2022_ngrams(BaseModel): +# file_id = IntegerField() +# sent_id = FloatField() +# ngram_len = IntegerField() +# frequency_g_t = IntegerField() +# gram_text = TextField() +# lemma_text = TextField() +# xpos_text = TextField() +# upos_text = TextField() +# +# db.connect() + + +class Ngrams_Manager: + @staticmethod + def get_by_file_id(file_id): + try: + # cur.execute(f'SELECT * from os2022_ngrams WHERE file_id = {file_id}') + # cur.execute(f'SELECT COUNT(*) FROM os2022_ngrams') + # return list(cur) + return 1 + except Exception as e: + print(e, 'EXC') + return 0 diff --git a/swagger_server/models/__init__.py b/swagger_server/models/__init__.py index 83cc770..a9718ec 100644 --- a/swagger_server/models/__init__.py +++ b/swagger_server/models/__init__.py @@ -6,4 +6,5 @@ from __future__ import absolute_import from swagger_server.models.datoteka_v_besedilo_body import DatotekaVBesediloBody from swagger_server.models.datoteka_v_besedilo_ocr_body import DatotekaVBesediloOcrBody from swagger_server.models.izlusci_body import IzlusciBody +from swagger_server.models.oznaci_besedilo_body import OznaciBesediloBody from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat diff --git a/swagger_server/models/oznaci_besedilo_body.py b/swagger_server/models/oznaci_besedilo_body.py new file mode 100644 index 0000000..18de203 --- /dev/null +++ b/swagger_server/models/oznaci_besedilo_body.py @@ -0,0 +1,62 @@ +# coding: utf-8 + +from __future__ import absolute_import +from datetime import date, datetime # noqa: F401 + +from typing import List, Dict # noqa: F401 + +from swagger_server.models.base_model_ import Model +from swagger_server import util + + +class OznaciBesediloBody(Model): + """NOTE: This class is auto generated by the swagger code generator program. + + Do not edit the class manually. + """ + def __init__(self, besedilo: str=None): # noqa: E501 + """OznaciBesediloBody - a model defined in Swagger + + :param besedilo: The besedilo of this OznaciBesediloBody. # noqa: E501 + :type besedilo: str + """ + self.swagger_types = { + 'besedilo': str + } + + self.attribute_map = { + 'besedilo': 'besedilo' + } + self._besedilo = besedilo + + @classmethod + def from_dict(cls, dikt) -> 'OznaciBesediloBody': + """Returns the dict as a model + + :param dikt: A dict. + :type: dict + :return: The oznaciBesedilo_body of this OznaciBesediloBody. # noqa: E501 + :rtype: OznaciBesediloBody + """ + return util.deserialize_model(dikt, cls) + + @property + def besedilo(self) -> str: + """Gets the besedilo of this OznaciBesediloBody. + + + :return: The besedilo of this OznaciBesediloBody. + :rtype: str + """ + return self._besedilo + + @besedilo.setter + def besedilo(self, besedilo: str): + """Sets the besedilo of this OznaciBesediloBody. + + + :param besedilo: The besedilo of this OznaciBesediloBody. + :type besedilo: str + """ + + self._besedilo = besedilo diff --git a/swagger_server/swagger/swagger.yaml b/swagger_server/swagger/swagger.yaml index 7e4d258..abcf60b 100644 --- a/swagger_server/swagger/swagger.yaml +++ b/swagger_server/swagger/swagger.yaml @@ -17,7 +17,7 @@ paths: content: application/json: schema: - type: string + $ref: '#/components/schemas/oznaciBesedilo_body' required: true responses: "200": @@ -485,6 +485,11 @@ components: kanonicnaoblika: kanonicnaoblika nosilnautez: 0.8008282 kandidat: kandidat + oznaciBesedilo_body: + type: object + properties: + besedilo: + type: string izlusci_body: type: object properties: diff --git a/swagger_server/test/test_marktext_controller.py b/swagger_server/test/test_marktext_controller.py index 03196e7..dca58b2 100644 --- a/swagger_server/test/test_marktext_controller.py +++ b/swagger_server/test/test_marktext_controller.py @@ -5,6 +5,7 @@ from __future__ import absolute_import from flask import json from six import BytesIO +from swagger_server.models.oznaci_besedilo_body import OznaciBesediloBody # noqa: E501 from swagger_server.test import BaseTestCase @@ -16,7 +17,7 @@ class TestMarktextController(BaseTestCase): Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format """ - body = 'body_example' + body = OznaciBesediloBody() response = self.client.open( '/oznaciBesedilo', method='POST', diff --git a/swagger_server/util.py b/swagger_server/util.py index 812b2ad..381939e 100644 --- a/swagger_server/util.py +++ b/swagger_server/util.py @@ -140,3 +140,7 @@ def _deserialize_dict(data, boxed_type): """ return {k: _deserialize(v, boxed_type) for k, v in six.iteritems(data)} + + +def get_conllu_path_by_id(file_id): + return f'..\\mnt\\ssd\\ds_ftp\\classla_OS2022\\conll\\rsdo_doc-{file_id}.plainText.conllu'