webapps/term_extractor
18
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- Popravljena swagger definicija in json (/oznaciBesedilo ni delal)

Browse Source 
- Dodani resourci za classlo
- Priprava util-ov za podatkovno bazo (se v zelo surovem stanju)
- Dodan docker-compose, ki bo potreben za nadaljni development
- Popravilejni controllerji
This commit is contained in:
Kristjan Žagar
2022-07-08 15:54:52 +02:00
parent aa2946ee4a
commit c8e4f7da17
10 changed files with 472 additions and 254 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker-compose.yml
+10
View File
@@ -0,0 +1,10 @@
version: '3.1'
services:
flask-serv:
build: .
restart: always
image: flask-server
ports:
- 8080:8080
volumes: # when pushing to server, remove the first dot here in front of /mnt
- ./mnt/ssd/ds_ftp/classla_OS2022:/usr/src/app/classla_OS2022
rsdo5.json
+314 -248
View File
@@ -4,21 +4,30 @@
"title": "OpenAPI definition", "title": "OpenAPI definition",
"version": "v0" "version": "v0"
}, },
"servers": [{ "servers": [
"url": "http://localhost:8089", {
"description": "Generated server url" "url": "http://localhost:8089",
}], "description": "Generated server url"
}
],
"paths": { "paths": {
"/oznaciBesedilo": { "/oznaciBesedilo": {
"post": { "post": {
"tags": ["marktext"], "tags": [
"marktext"
],
"summary": "Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format", "summary": "Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format",
"operationId": "getText", "operationId": "getText",
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
"schema": { "schema": {
"type": "string" "type": "object",
"properties": {
"besedilo": {
"type": "string"
}
}
} }
} }
}, },
@@ -40,7 +49,9 @@
}, },
"/izlusci": { "/izlusci": {
"post": { "post": {
"tags": ["extract"], "tags": [
"extract"
],
"summary": "Izlusci terminološke kandidate iz seznama besedil v conllu obliki", "summary": "Izlusci terminološke kandidate iz seznama besedil v conllu obliki",
"operationId": "getCandidates", "operationId": "getCandidates",
"requestBody": { "requestBody": {
@@ -86,14 +97,18 @@
}, },
"/datotekaVBesedilo/ocr": { "/datotekaVBesedilo/ocr": {
"post": { "post": {
"tags": ["doc-2text"], "tags": [
"doc-2text"
],
"summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo s pomočjo ocr razpoznavanja", "summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo s pomočjo ocr razpoznavanja",
"operationId": "getTextOcr", "operationId": "getTextOcr",
"requestBody": { "requestBody": {
"content": { "content": {
"multipart/form-data": { "multipart/form-data": {
"schema": { "schema": {
"required": ["file"], "required": [
"file"
],
"type": "object", "type": "object",
"properties": { "properties": {
"file": { "file": {
@@ -121,13 +136,17 @@
}, },
"/datotekaVBesedilo/": { "/datotekaVBesedilo/": {
"post": { "post": {
"tags": ["doc-2text"], "tags": [
"doc-2text"
],
"summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo", "summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo",
"requestBody": { "requestBody": {
"content": { "content": {
"multipart/form-data": { "multipart/form-data": {
"schema": { "schema": {
"required": ["file"], "required": [
"file"
],
"type": "object", "type": "object",
"properties": { "properties": {
"file": { "file": {
@@ -155,52 +174,59 @@
}, },
"/oss/steviloBesedilPoIskanju": { "/oss/steviloBesedilPoIskanju": {
"get": { "get": {
"tags": ["oss"], "tags": [
"oss"
],
"summary": "Vrne število besedil glede na iskalne pogoje", "summary": "Vrne število besedil glede na iskalne pogoje",
"operationId": "getNumberTexts", "operationId": "getNumberTexts",
"parameters": [{ "parameters": [
"name": "leta", {
"in": "query", "name": "leta",
"required": true, "in": "query",
"schema": { "required": true,
"type": "array", "schema": {
"items": { "type": "array",
"type": "integer", "items": {
"format": "int64" "type": "integer",
"format": "int64"
}
}
},
{
"name": "vrste",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"name": "kljucnebesede",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"name": "cerifpodrocja",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
}
} }
} }
}, { ],
"name": "vrste",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}, {
"name": "kljucnebesede",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}, {
"name": "cerifpodrocja",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
}
}
}],
"responses": { "responses": {
"200": { "200": {
"description": "OK", "description": "OK",
@@ -218,52 +244,59 @@
}, },
"/oss/izlusciPoIskanju": { "/oss/izlusciPoIskanju": {
"get": { "get": {
"tags": ["oss"], "tags": [
"oss"
],
"summary": "Vrne terminloške kandidate glede na ", "summary": "Vrne terminloške kandidate glede na ",
"operationId": "getExtractedWords", "operationId": "getExtractedWords",
"parameters": [{ "parameters": [
"name": "leta", {
"in": "query", "name": "leta",
"required": true, "in": "query",
"schema": { "required": true,
"type": "array", "schema": {
"items": { "type": "array",
"type": "integer", "items": {
"format": "int64" "type": "integer",
"format": "int64"
}
}
},
{
"name": "vrste",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"name": "kljucnebesede",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"name": "cerifpodrocja",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
}
} }
} }
}, { ],
"name": "vrste",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}, {
"name": "kljucnebesede",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}, {
"name": "cerifpodrocja",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
}
}
}],
"responses": { "responses": {
"200": { "200": {
"description": "OK", "description": "OK",
@@ -283,52 +316,59 @@
}, },
"/oss/datotekePoIskanju": { "/oss/datotekePoIskanju": {
"get": { "get": {
"tags": ["oss"], "tags": [
"oss"
],
"summary": "Vrne seznam binarnih zapisov v originalnem formatu glede na iskalne pogoje", "summary": "Vrne seznam binarnih zapisov v originalnem formatu glede na iskalne pogoje",
"operationId": "getFiles", "operationId": "getFiles",
"parameters": [{ "parameters": [
"name": "leta", {
"in": "query", "name": "leta",
"required": true, "in": "query",
"schema": { "required": true,
"type": "array", "schema": {
"items": { "type": "array",
"type": "integer", "items": {
"format": "int64" "type": "integer",
"format": "int64"
}
}
},
{
"name": "vrste",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"name": "kljucnebesede",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"name": "cerifpodrocja",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
}
} }
} }
}, { ],
"name": "vrste",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}, {
"name": "kljucnebesede",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}, {
"name": "cerifpodrocja",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
}
}
}],
"responses": { "responses": {
"200": { "200": {
"description": "OK", "description": "OK",
@@ -352,18 +392,22 @@
}, },
"/oss/datotekaPoId": { "/oss/datotekaPoId": {
"get": { "get": {
"tags": ["oss"], "tags": [
"oss"
],
"summary": "Vrne binarni zapis v originalnem formatu po id-ju datoteke", "summary": "Vrne binarni zapis v originalnem formatu po id-ju datoteke",
"operationId": "getFile", "operationId": "getFile",
"parameters": [{ "parameters": [
"name": "file_id", {
"in": "query", "name": "file_id",
"required": true, "in": "query",
"schema": { "required": true,
"type": "integer", "schema": {
"format": "int64" "type": "integer",
"format": "int64"
}
} }
}], ],
"responses": { "responses": {
"200": { "200": {
"description": "OK", "description": "OK",
@@ -384,52 +428,59 @@
}, },
"/oss/conlluPoIskanju": { "/oss/conlluPoIskanju": {
"get": { "get": {
"tags": ["oss"], "tags": [
"oss"
],
"summary": "Vrne seznam CoNNL-U-jev glede na iskalne pogoje", "summary": "Vrne seznam CoNNL-U-jev glede na iskalne pogoje",
"operationId": "getConllus", "operationId": "getConllus",
"parameters": [{ "parameters": [
"name": "leta", {
"in": "query", "name": "leta",
"required": true, "in": "query",
"schema": { "required": true,
"type": "array", "schema": {
"items": { "type": "array",
"type": "integer", "items": {
"format": "int64" "type": "integer",
"format": "int64"
}
}
},
{
"name": "vrste",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"name": "kljucnebesede",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"name": "cerifpodrocja",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
}
} }
} }
}, { ],
"name": "vrste",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}, {
"name": "kljucnebesede",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}, {
"name": "cerifpodrocja",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
}
}
}],
"responses": { "responses": {
"200": { "200": {
"description": "OK", "description": "OK",
@@ -449,18 +500,22 @@
}, },
"/oss/conlluPoId": { "/oss/conlluPoId": {
"get": { "get": {
"tags": ["oss"], "tags": [
"oss"
],
"summary": "Vrne CoNNL-U po id-ju datoteke", "summary": "Vrne CoNNL-U po id-ju datoteke",
"operationId": "getConllu", "operationId": "getConllu",
"parameters": [{ "parameters": [
"name": "file_id", {
"in": "query", "name": "file_id",
"required": true, "in": "query",
"schema": { "required": true,
"type": "integer", "schema": {
"format": "int64" "type": "integer",
"format": "int64"
}
} }
}], ],
"responses": { "responses": {
"200": { "200": {
"description": "OK", "description": "OK",
@@ -477,17 +532,21 @@
}, },
"/oss/besediloPoId": { "/oss/besediloPoId": {
"get": { "get": {
"tags": ["oss"], "tags": [
"oss"
],
"summary": "Vrne besedilo po id-ju datoteke", "summary": "Vrne besedilo po id-ju datoteke",
"parameters": [{ "parameters": [
"name": "file_id", {
"in": "query", "name": "file_id",
"required": true, "in": "query",
"schema": { "required": true,
"type": "integer", "schema": {
"format": "int64" "type": "integer",
"format": "int64"
}
} }
}], ],
"responses": { "responses": {
"200": { "200": {
"description": "OK", "description": "OK",
@@ -504,52 +563,59 @@
}, },
"/oss/besedilaPoIskanju": { "/oss/besedilaPoIskanju": {
"get": { "get": {
"tags": ["oss"], "tags": [
"oss"
],
"summary": "Vrne seznam besedil glede na iskalne pogoje", "summary": "Vrne seznam besedil glede na iskalne pogoje",
"operationId": "getTexts", "operationId": "getTexts",
"parameters": [{ "parameters": [
"name": "leta", {
"in": "query", "name": "leta",
"required": true, "in": "query",
"schema": { "required": true,
"type": "array", "schema": {
"items": { "type": "array",
"type": "integer", "items": {
"format": "int64" "type": "integer",
"format": "int64"
}
}
},
{
"name": "vrste",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"name": "kljucnebesede",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"name": "cerifpodrocja",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
}
} }
} }
}, { ],
"name": "vrste",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}, {
"name": "kljucnebesede",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}, {
"name": "cerifpodrocja",
"in": "query",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "integer",
"format": "int64"
}
}
}],
"responses": { "responses": {
"200": { "200": {
"description": "OK", "description": "OK",
swagger_server/controllers/marktext_controller.py
+7 -3
View File
@@ -1,7 +1,9 @@
import connexion import connexion
import six import six
from swagger_server.models.oznaci_besedilo_body import OznaciBesediloBody # noqa: E501
from swagger_server import util from swagger_server import util
from swagger_server.classla import cl_utils
def get_text(body): # noqa: E501 def get_text(body): # noqa: E501
@@ -9,11 +11,13 @@ def get_text(body): # noqa: E501
# noqa: E501 # noqa: E501
:param body: :param body:
:type body: dict | bytes :type body: dict | bytes
:rtype: str :rtype: str
""" """
if connexion.request.is_json: if connexion.request.is_json:
body = str.from_dict(connexion.request.get_json()) # noqa: E501 body = OznaciBesediloBody.from_dict(connexion.request.get_json()) # noqa: E501
return 'do some magic!' conllu = cl_utils.raw_text_to_conllu(body.besedilo)
return conllu
swagger_server/controllers/oss_controller.py
+14 -1
View File
@@ -1,8 +1,11 @@
import connexion import connexion
import six import six
import os
from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat # noqa: E501 from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat # noqa: E501
from swagger_server import util from swagger_server import util
from flask import send_file
from swagger_server.db_utils import Ngrams_Manager
def get_conllu(file_id): # noqa: E501 def get_conllu(file_id): # noqa: E501
@@ -15,7 +18,17 @@ def get_conllu(file_id): # noqa: E501
:rtype: str :rtype: str
""" """
return 'do some magic!' file_path = util.get_conllu_path_by_id(file_id)
a = Ngrams_Manager.get_by_file_id(file_id)
print(a)
try:
return send_file(file_path, attachment_filename=f'{file_id}.conllu', as_attachment=True)
except Exception as e:
if "The system cannot find the file specified" in str(e):
return "The conllu with this ID doesn't exist.", 404
return str(e), 400
def get_conllus(leta, vrste, kljucnebesede, cerifpodrocja): # noqa: E501 def get_conllus(leta, vrste, kljucnebesede, cerifpodrocja): # noqa: E501
swagger_server/db_utils.py
+52
View File
@@ -0,0 +1,52 @@
import mariadb
import sys
# todo: DO NOT PUSH THIS TO GIT
database_info = {
'database': 'TEMPORARYVALJUSTFORTHECOMMITPURPOSE',
'host': 'TEMPORARYVALJUSTFORTHECOMMITPURPOSE',
'port': 'TEMPORARYVALJUSTFORTHECOMMITPURPOSE',
'user': 'TEMPORARYVALJUSTFORTHECOMMITPURPOSE',
'password': 'TEMPORARYVALJUSTFORTHECOMMITPURPOSE'
}
# Connect to MariaDB Platform
try:
conn = mariadb.connect(**database_info)
except mariadb.Error as e:
print(f"Error connecting to MariaDB Platform: {e}")
sys.exit(1)
# Get Cursor
cur = conn.cursor()
# class BaseModel(Model):
# class Meta:
# database = db
#
#
# class os2022_ngrams(BaseModel):
# file_id = IntegerField()
# sent_id = FloatField()
# ngram_len = IntegerField()
# frequency_g_t = IntegerField()
# gram_text = TextField()
# lemma_text = TextField()
# xpos_text = TextField()
# upos_text = TextField()
#
# db.connect()
class Ngrams_Manager:
@staticmethod
def get_by_file_id(file_id):
try:
# cur.execute(f'SELECT * from os2022_ngrams WHERE file_id = {file_id}')
# cur.execute(f'SELECT COUNT(*) FROM os2022_ngrams')
# return list(cur)
return 1
except Exception as e:
print(e, 'EXC')
return 0
swagger_server/models/__init__.py
+1
View File
@@ -6,4 +6,5 @@ from __future__ import absolute_import
from swagger_server.models.datoteka_v_besedilo_body import DatotekaVBesediloBody from swagger_server.models.datoteka_v_besedilo_body import DatotekaVBesediloBody
from swagger_server.models.datoteka_v_besedilo_ocr_body import DatotekaVBesediloOcrBody from swagger_server.models.datoteka_v_besedilo_ocr_body import DatotekaVBesediloOcrBody
from swagger_server.models.izlusci_body import IzlusciBody from swagger_server.models.izlusci_body import IzlusciBody
from swagger_server.models.oznaci_besedilo_body import OznaciBesediloBody
from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat
swagger_server/models/oznaci_besedilo_body.py
+62
View File
@@ -0,0 +1,62 @@
# coding: utf-8
from __future__ import absolute_import
from datetime import date, datetime # noqa: F401
from typing import List, Dict # noqa: F401
from swagger_server.models.base_model_ import Model
from swagger_server import util
class OznaciBesediloBody(Model):
"""NOTE: This class is auto generated by the swagger code generator program.
Do not edit the class manually.
"""
def __init__(self, besedilo: str=None): # noqa: E501
"""OznaciBesediloBody - a model defined in Swagger
:param besedilo: The besedilo of this OznaciBesediloBody. # noqa: E501
:type besedilo: str
"""
self.swagger_types = {
'besedilo': str
}
self.attribute_map = {
'besedilo': 'besedilo'
}
self._besedilo = besedilo
@classmethod
def from_dict(cls, dikt) -> 'OznaciBesediloBody':
"""Returns the dict as a model
:param dikt: A dict.
:type: dict
:return: The oznaciBesedilo_body of this OznaciBesediloBody. # noqa: E501
:rtype: OznaciBesediloBody
"""
return util.deserialize_model(dikt, cls)
@property
def besedilo(self) -> str:
"""Gets the besedilo of this OznaciBesediloBody.
:return: The besedilo of this OznaciBesediloBody.
:rtype: str
"""
return self._besedilo
@besedilo.setter
def besedilo(self, besedilo: str):
"""Sets the besedilo of this OznaciBesediloBody.
:param besedilo: The besedilo of this OznaciBesediloBody.
:type besedilo: str
"""
self._besedilo = besedilo
swagger_server/swagger/swagger.yaml
+6 -1
View File
@@ -17,7 +17,7 @@ paths:
content: content:
application/json: application/json:
schema: schema:
type: string $ref: '#/components/schemas/oznaciBesedilo_body'
required: true required: true
responses: responses:
"200": "200":
@@ -485,6 +485,11 @@ components:
kanonicnaoblika: kanonicnaoblika kanonicnaoblika: kanonicnaoblika
nosilnautez: 0.8008282 nosilnautez: 0.8008282
kandidat: kandidat kandidat: kandidat
oznaciBesedilo_body:
type: object
properties:
besedilo:
type: string
izlusci_body: izlusci_body:
type: object type: object
properties: properties:
swagger_server/test/test_marktext_controller.py
+2 -1
View File
@@ -5,6 +5,7 @@ from __future__ import absolute_import
from flask import json from flask import json
from six import BytesIO from six import BytesIO
from swagger_server.models.oznaci_besedilo_body import OznaciBesediloBody # noqa: E501
from swagger_server.test import BaseTestCase from swagger_server.test import BaseTestCase
@@ -16,7 +17,7 @@ class TestMarktextController(BaseTestCase):
Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
""" """
body = 'body_example' body = OznaciBesediloBody()
response = self.client.open( response = self.client.open(
'/oznaciBesedilo', '/oznaciBesedilo',
method='POST', method='POST',
swagger_server/util.py
+4
View File
@@ -140,3 +140,7 @@ def _deserialize_dict(data, boxed_type):
""" """
return {k: _deserialize(v, boxed_type) return {k: _deserialize(v, boxed_type)
for k, v in six.iteritems(data)} for k, v in six.iteritems(data)}
def get_conllu_path_by_id(file_id):
return f'..\\mnt\\ssd\\ds_ftp\\classla_OS2022\\conll\\rsdo_doc-{file_id}.plainText.conllu'