- Popravljena swagger definicija in json (/oznaciBesedilo ni delal)

- Dodani resourci za classlo
- Priprava util-ov za podatkovno bazo (se v zelo surovem stanju)
- Dodan docker-compose, ki bo potreben za nadaljni development
- Popravilejni controllerji
This commit is contained in:
Kristjan Žagar
2022-07-08 15:54:52 +02:00
parent aa2946ee4a
commit c8e4f7da17
10 changed files with 472 additions and 254 deletions
+10
View File
@@ -0,0 +1,10 @@
version: '3.1'
services:
flask-serv:
build: .
restart: always
image: flask-server
ports:
- 8080:8080
volumes: # when pushing to server, remove the first dot here in front of /mnt
- ./mnt/ssd/ds_ftp/classla_OS2022:/usr/src/app/classla_OS2022
+314 -248
View File
@@ -4,21 +4,30 @@
"title": "OpenAPI definition",
"version": "v0"
},
"servers": [{
"url": "http://localhost:8089",
"description": "Generated server url"
}],
"servers": [
{
"url": "http://localhost:8089",
"description": "Generated server url"
}
],
"paths": {
"/oznaciBesedilo": {
"post": {
"tags": ["marktext"],
"tags": [
"marktext"
],
"summary": "Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format",
"operationId": "getText",
"requestBody": {
"content": {
"application/json": {
"schema": {
"type": "string"
"type": "object",
"properties": {
"besedilo": {
"type": "string"
}
}
}
}
},
@@ -40,7 +49,9 @@
},
"/izlusci": {
"post": {
"tags": ["extract"],
"tags": [
"extract"
],
"summary": "Izlusci terminološke kandidate iz seznama besedil v conllu obliki",
"operationId": "getCandidates",
"requestBody": {
@@ -86,14 +97,18 @@
},
"/datotekaVBesedilo/ocr": {
"post": {
"tags": ["doc-2text"],
"tags": [
"doc-2text"
],
"summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo s pomočjo ocr razpoznavanja",
"operationId": "getTextOcr",
"requestBody": {
"content": {
"multipart/form-data": {
"schema": {
"required": ["file"],
"required": [
"file"
],
"type": "object",
"properties": {
"file": {
@@ -121,13 +136,17 @@
},
"/datotekaVBesedilo/": {
"post": {
"tags": ["doc-2text"],
"tags": [
"doc-2text"
],
"summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo",
"requestBody": {
"content": {
"multipart/form-data": {
"schema": {
"required": ["file"],
"required": [
"file"
],
"type": "object",
"properties": {
"file": {
@@ -155,52 +174,59 @@
},
"/oss/steviloBesedilPoIskanju": {
"get": {
"tags": ["oss"],
"tags": [
"oss"
],
"summary": "Vrne število besedil glede na iskalne pogoje",
"operationId": "getNumberTexts",
"parameters": [{
"name": "leta",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
"parameters": [
{
"name": "leta",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
}
}
},
{
"name": "vrste",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"name": "kljucnebesede",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"name": "cerifpodrocja",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
}
}
}
}, {
"name": "vrste",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}, {
"name": "kljucnebesede",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}, {
"name": "cerifpodrocja",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
}
}
}],
],
"responses": {
"200": {
"description": "OK",
@@ -218,52 +244,59 @@
},
"/oss/izlusciPoIskanju": {
"get": {
"tags": ["oss"],
"tags": [
"oss"
],
"summary": "Vrne terminloške kandidate glede na ",
"operationId": "getExtractedWords",
"parameters": [{
"name": "leta",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
"parameters": [
{
"name": "leta",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
}
}
},
{
"name": "vrste",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"name": "kljucnebesede",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"name": "cerifpodrocja",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
}
}
}
}, {
"name": "vrste",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}, {
"name": "kljucnebesede",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}, {
"name": "cerifpodrocja",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
}
}
}],
],
"responses": {
"200": {
"description": "OK",
@@ -283,52 +316,59 @@
},
"/oss/datotekePoIskanju": {
"get": {
"tags": ["oss"],
"tags": [
"oss"
],
"summary": "Vrne seznam binarnih zapisov v originalnem formatu glede na iskalne pogoje",
"operationId": "getFiles",
"parameters": [{
"name": "leta",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
"parameters": [
{
"name": "leta",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
}
}
},
{
"name": "vrste",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"name": "kljucnebesede",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"name": "cerifpodrocja",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
}
}
}
}, {
"name": "vrste",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}, {
"name": "kljucnebesede",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}, {
"name": "cerifpodrocja",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
}
}
}],
],
"responses": {
"200": {
"description": "OK",
@@ -352,18 +392,22 @@
},
"/oss/datotekaPoId": {
"get": {
"tags": ["oss"],
"tags": [
"oss"
],
"summary": "Vrne binarni zapis v originalnem formatu po id-ju datoteke",
"operationId": "getFile",
"parameters": [{
"name": "file_id",
"in": "query",
"required": true,
"schema": {
"type": "integer",
"format": "int64"
"parameters": [
{
"name": "file_id",
"in": "query",
"required": true,
"schema": {
"type": "integer",
"format": "int64"
}
}
}],
],
"responses": {
"200": {
"description": "OK",
@@ -384,52 +428,59 @@
},
"/oss/conlluPoIskanju": {
"get": {
"tags": ["oss"],
"tags": [
"oss"
],
"summary": "Vrne seznam CoNNL-U-jev glede na iskalne pogoje",
"operationId": "getConllus",
"parameters": [{
"name": "leta",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
"parameters": [
{
"name": "leta",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
}
}
},
{
"name": "vrste",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"name": "kljucnebesede",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"name": "cerifpodrocja",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
}
}
}
}, {
"name": "vrste",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}, {
"name": "kljucnebesede",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}, {
"name": "cerifpodrocja",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
}
}
}],
],
"responses": {
"200": {
"description": "OK",
@@ -449,18 +500,22 @@
},
"/oss/conlluPoId": {
"get": {
"tags": ["oss"],
"tags": [
"oss"
],
"summary": "Vrne CoNNL-U po id-ju datoteke",
"operationId": "getConllu",
"parameters": [{
"name": "file_id",
"in": "query",
"required": true,
"schema": {
"type": "integer",
"format": "int64"
"parameters": [
{
"name": "file_id",
"in": "query",
"required": true,
"schema": {
"type": "integer",
"format": "int64"
}
}
}],
],
"responses": {
"200": {
"description": "OK",
@@ -477,17 +532,21 @@
},
"/oss/besediloPoId": {
"get": {
"tags": ["oss"],
"tags": [
"oss"
],
"summary": "Vrne besedilo po id-ju datoteke",
"parameters": [{
"name": "file_id",
"in": "query",
"required": true,
"schema": {
"type": "integer",
"format": "int64"
"parameters": [
{
"name": "file_id",
"in": "query",
"required": true,
"schema": {
"type": "integer",
"format": "int64"
}
}
}],
],
"responses": {
"200": {
"description": "OK",
@@ -504,52 +563,59 @@
},
"/oss/besedilaPoIskanju": {
"get": {
"tags": ["oss"],
"tags": [
"oss"
],
"summary": "Vrne seznam besedil glede na iskalne pogoje",
"operationId": "getTexts",
"parameters": [{
"name": "leta",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
"parameters": [
{
"name": "leta",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
}
}
},
{
"name": "vrste",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"name": "kljucnebesede",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"name": "cerifpodrocja",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
}
}
}
}, {
"name": "vrste",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}, {
"name": "kljucnebesede",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}, {
"name": "cerifpodrocja",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
}
}
}],
],
"responses": {
"200": {
"description": "OK",
@@ -1,7 +1,9 @@
import connexion
import six
from swagger_server.models.oznaci_besedilo_body import OznaciBesediloBody # noqa: E501
from swagger_server import util
from swagger_server.classla import cl_utils
def get_text(body): # noqa: E501
@@ -9,11 +11,13 @@ def get_text(body): # noqa: E501
# noqa: E501
:param body:
:param body:
:type body: dict | bytes
:rtype: str
"""
if connexion.request.is_json:
body = str.from_dict(connexion.request.get_json()) # noqa: E501
return 'do some magic!'
body = OznaciBesediloBody.from_dict(connexion.request.get_json()) # noqa: E501
conllu = cl_utils.raw_text_to_conllu(body.besedilo)
return conllu
+14 -1
View File
@@ -1,8 +1,11 @@
import connexion
import six
import os
from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat # noqa: E501
from swagger_server import util
from flask import send_file
from swagger_server.db_utils import Ngrams_Manager
def get_conllu(file_id): # noqa: E501
@@ -15,7 +18,17 @@ def get_conllu(file_id): # noqa: E501
:rtype: str
"""
return 'do some magic!'
file_path = util.get_conllu_path_by_id(file_id)
a = Ngrams_Manager.get_by_file_id(file_id)
print(a)
try:
return send_file(file_path, attachment_filename=f'{file_id}.conllu', as_attachment=True)
except Exception as e:
if "The system cannot find the file specified" in str(e):
return "The conllu with this ID doesn't exist.", 404
return str(e), 400
def get_conllus(leta, vrste, kljucnebesede, cerifpodrocja): # noqa: E501
+52
View File
@@ -0,0 +1,52 @@
import mariadb
import sys
# todo: DO NOT PUSH THIS TO GIT
database_info = {
'database': 'TEMPORARYVALJUSTFORTHECOMMITPURPOSE',
'host': 'TEMPORARYVALJUSTFORTHECOMMITPURPOSE',
'port': 'TEMPORARYVALJUSTFORTHECOMMITPURPOSE',
'user': 'TEMPORARYVALJUSTFORTHECOMMITPURPOSE',
'password': 'TEMPORARYVALJUSTFORTHECOMMITPURPOSE'
}
# Connect to MariaDB Platform
try:
conn = mariadb.connect(**database_info)
except mariadb.Error as e:
print(f"Error connecting to MariaDB Platform: {e}")
sys.exit(1)
# Get Cursor
cur = conn.cursor()
# class BaseModel(Model):
# class Meta:
# database = db
#
#
# class os2022_ngrams(BaseModel):
# file_id = IntegerField()
# sent_id = FloatField()
# ngram_len = IntegerField()
# frequency_g_t = IntegerField()
# gram_text = TextField()
# lemma_text = TextField()
# xpos_text = TextField()
# upos_text = TextField()
#
# db.connect()
class Ngrams_Manager:
@staticmethod
def get_by_file_id(file_id):
try:
# cur.execute(f'SELECT * from os2022_ngrams WHERE file_id = {file_id}')
# cur.execute(f'SELECT COUNT(*) FROM os2022_ngrams')
# return list(cur)
return 1
except Exception as e:
print(e, 'EXC')
return 0
+1
View File
@@ -6,4 +6,5 @@ from __future__ import absolute_import
from swagger_server.models.datoteka_v_besedilo_body import DatotekaVBesediloBody
from swagger_server.models.datoteka_v_besedilo_ocr_body import DatotekaVBesediloOcrBody
from swagger_server.models.izlusci_body import IzlusciBody
from swagger_server.models.oznaci_besedilo_body import OznaciBesediloBody
from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat
@@ -0,0 +1,62 @@
# coding: utf-8
from __future__ import absolute_import
from datetime import date, datetime # noqa: F401
from typing import List, Dict # noqa: F401
from swagger_server.models.base_model_ import Model
from swagger_server import util
class OznaciBesediloBody(Model):
"""NOTE: This class is auto generated by the swagger code generator program.
Do not edit the class manually.
"""
def __init__(self, besedilo: str=None): # noqa: E501
"""OznaciBesediloBody - a model defined in Swagger
:param besedilo: The besedilo of this OznaciBesediloBody. # noqa: E501
:type besedilo: str
"""
self.swagger_types = {
'besedilo': str
}
self.attribute_map = {
'besedilo': 'besedilo'
}
self._besedilo = besedilo
@classmethod
def from_dict(cls, dikt) -> 'OznaciBesediloBody':
"""Returns the dict as a model
:param dikt: A dict.
:type: dict
:return: The oznaciBesedilo_body of this OznaciBesediloBody. # noqa: E501
:rtype: OznaciBesediloBody
"""
return util.deserialize_model(dikt, cls)
@property
def besedilo(self) -> str:
"""Gets the besedilo of this OznaciBesediloBody.
:return: The besedilo of this OznaciBesediloBody.
:rtype: str
"""
return self._besedilo
@besedilo.setter
def besedilo(self, besedilo: str):
"""Sets the besedilo of this OznaciBesediloBody.
:param besedilo: The besedilo of this OznaciBesediloBody.
:type besedilo: str
"""
self._besedilo = besedilo
+6 -1
View File
@@ -17,7 +17,7 @@ paths:
content:
application/json:
schema:
type: string
$ref: '#/components/schemas/oznaciBesedilo_body'
required: true
responses:
"200":
@@ -485,6 +485,11 @@ components:
kanonicnaoblika: kanonicnaoblika
nosilnautez: 0.8008282
kandidat: kandidat
oznaciBesedilo_body:
type: object
properties:
besedilo:
type: string
izlusci_body:
type: object
properties:
@@ -5,6 +5,7 @@ from __future__ import absolute_import
from flask import json
from six import BytesIO
from swagger_server.models.oznaci_besedilo_body import OznaciBesediloBody # noqa: E501
from swagger_server.test import BaseTestCase
@@ -16,7 +17,7 @@ class TestMarktextController(BaseTestCase):
Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
"""
body = 'body_example'
body = OznaciBesediloBody()
response = self.client.open(
'/oznaciBesedilo',
method='POST',
+4
View File
@@ -140,3 +140,7 @@ def _deserialize_dict(data, boxed_type):
"""
return {k: _deserialize(v, boxed_type)
for k, v in six.iteritems(data)}
def get_conllu_path_by_id(file_id):
return f'..\\mnt\\ssd\\ds_ftp\\classla_OS2022\\conll\\rsdo_doc-{file_id}.plainText.conllu'