webapps/term_extractor
18
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- Popravljena swagger definicija in json (/oznaciBesedilo ni delal)

Browse Source 
- Dodani resourci za classlo
- Priprava util-ov za podatkovno bazo (se v zelo surovem stanju)
- Dodan docker-compose, ki bo potreben za nadaljni development
- Popravilejni controllerji
This commit is contained in:
Kristjan Žagar
2022-07-08 15:54:52 +02:00
parent aa2946ee4a
commit c8e4f7da17
10 changed files with 472 additions and 254 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker-compose.yml
+10
View File
@@ -0,0 +1,10 @@
version: '3.1'
services:
flask-serv:
build: .
restart: always
image: flask-server
ports:
- 8080:8080
volumes: # when pushing to server, remove the first dot here in front of /mnt
- ./mnt/ssd/ds_ftp/classla_OS2022:/usr/src/app/classla_OS2022
rsdo5.json
+113 -47
View File
@@ -4,23 +4,32 @@
"title": "OpenAPI definition",
"version": "v0"
},
"servers": [{
"servers": [
{
"url": "http://localhost:8089",
"description": "Generated server url"
}],
}
],
"paths": {
"/oznaciBesedilo": {
"post": {
"tags": ["marktext"],
"tags": [
"marktext"
],
"summary": "Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format",
"operationId": "getText",
"requestBody": {
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"besedilo": {
"type": "string"
}
}
}
}
},
"required": true
},
@@ -40,7 +49,9 @@
},
"/izlusci": {
"post": {
"tags": ["extract"],
"tags": [
"extract"
],
"summary": "Izlusci terminološke kandidate iz seznama besedil v conllu obliki",
"operationId": "getCandidates",
"requestBody": {
@@ -86,14 +97,18 @@
},
"/datotekaVBesedilo/ocr": {
"post": {
"tags": ["doc-2text"],
"tags": [
"doc-2text"
],
"summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo s pomočjo ocr razpoznavanja",
"operationId": "getTextOcr",
"requestBody": {
"content": {
"multipart/form-data": {
"schema": {
"required": ["file"],
"required": [
"file"
],
"type": "object",
"properties": {
"file": {
@@ -121,13 +136,17 @@
},
"/datotekaVBesedilo/": {
"post": {
"tags": ["doc-2text"],
"tags": [
"doc-2text"
],
"summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo",
"requestBody": {
"content": {
"multipart/form-data": {
"schema": {
"required": ["file"],
"required": [
"file"
],
"type": "object",
"properties": {
"file": {
@@ -155,10 +174,13 @@
},
"/oss/steviloBesedilPoIskanju": {
"get": {
"tags": ["oss"],
"tags": [
"oss"
],
"summary": "Vrne število besedil glede na iskalne pogoje",
"operationId": "getNumberTexts",
"parameters": [{
"parameters": [
{
"name": "leta",
"in": "query",
"required": true,
@@ -169,7 +191,8 @@
"format": "int64"
}
}
}, {
},
{
"name": "vrste",
"in": "query",
"required": true,
@@ -179,7 +202,8 @@
"type": "string"
}
}
}, {
},
{
"name": "kljucnebesede",
"in": "query",
"required": true,
@@ -189,7 +213,8 @@
"type": "string"
}
}
}, {
},
{
"name": "cerifpodrocja",
"in": "query",
"required": true,
@@ -200,7 +225,8 @@
"format": "int64"
}
}
}],
}
],
"responses": {
"200": {
"description": "OK",
@@ -218,10 +244,13 @@
},
"/oss/izlusciPoIskanju": {
"get": {
"tags": ["oss"],
"tags": [
"oss"
],
"summary": "Vrne terminloške kandidate glede na ",
"operationId": "getExtractedWords",
"parameters": [{
"parameters": [
{
"name": "leta",
"in": "query",
"required": true,
@@ -232,7 +261,8 @@
"format": "int64"
}
}
}, {
},
{
"name": "vrste",
"in": "query",
"required": true,
@@ -242,7 +272,8 @@
"type": "string"
}
}
}, {
},
{
"name": "kljucnebesede",
"in": "query",
"required": true,
@@ -252,7 +283,8 @@
"type": "string"
}
}
}, {
},
{
"name": "cerifpodrocja",
"in": "query",
"required": true,
@@ -263,7 +295,8 @@
"format": "int64"
}
}
}],
}
],
"responses": {
"200": {
"description": "OK",
@@ -283,10 +316,13 @@
},
"/oss/datotekePoIskanju": {
"get": {
"tags": ["oss"],
"tags": [
"oss"
],
"summary": "Vrne seznam binarnih zapisov v originalnem formatu glede na iskalne pogoje",
"operationId": "getFiles",
"parameters": [{
"parameters": [
{
"name": "leta",
"in": "query",
"required": true,
@@ -297,7 +333,8 @@
"format": "int64"
}
}
}, {
},
{
"name": "vrste",
"in": "query",
"required": true,
@@ -307,7 +344,8 @@
"type": "string"
}
}
}, {
},
{
"name": "kljucnebesede",
"in": "query",
"required": true,
@@ -317,7 +355,8 @@
"type": "string"
}
}
}, {
},
{
"name": "cerifpodrocja",
"in": "query",
"required": true,
@@ -328,7 +367,8 @@
"format": "int64"
}
}
}],
}
],
"responses": {
"200": {
"description": "OK",
@@ -352,10 +392,13 @@
},
"/oss/datotekaPoId": {
"get": {
"tags": ["oss"],
"tags": [
"oss"
],
"summary": "Vrne binarni zapis v originalnem formatu po id-ju datoteke",
"operationId": "getFile",
"parameters": [{
"parameters": [
{
"name": "file_id",
"in": "query",
"required": true,
@@ -363,7 +406,8 @@
"type": "integer",
"format": "int64"
}
}],
}
],
"responses": {
"200": {
"description": "OK",
@@ -384,10 +428,13 @@
},
"/oss/conlluPoIskanju": {
"get": {
"tags": ["oss"],
"tags": [
"oss"
],
"summary": "Vrne seznam CoNNL-U-jev glede na iskalne pogoje",
"operationId": "getConllus",
"parameters": [{
"parameters": [
{
"name": "leta",
"in": "query",
"required": true,
@@ -398,7 +445,8 @@
"format": "int64"
}
}
}, {
},
{
"name": "vrste",
"in": "query",
"required": true,
@@ -408,7 +456,8 @@
"type": "string"
}
}
}, {
},
{
"name": "kljucnebesede",
"in": "query",
"required": true,
@@ -418,7 +467,8 @@
"type": "string"
}
}
}, {
},
{
"name": "cerifpodrocja",
"in": "query",
"required": true,
@@ -429,7 +479,8 @@
"format": "int64"
}
}
}],
}
],
"responses": {
"200": {
"description": "OK",
@@ -449,10 +500,13 @@
},
"/oss/conlluPoId": {
"get": {
"tags": ["oss"],
"tags": [
"oss"
],
"summary": "Vrne CoNNL-U po id-ju datoteke",
"operationId": "getConllu",
"parameters": [{
"parameters": [
{
"name": "file_id",
"in": "query",
"required": true,
@@ -460,7 +514,8 @@
"type": "integer",
"format": "int64"
}
}],
}
],
"responses": {
"200": {
"description": "OK",
@@ -477,9 +532,12 @@
},
"/oss/besediloPoId": {
"get": {
"tags": ["oss"],
"tags": [
"oss"
],
"summary": "Vrne besedilo po id-ju datoteke",
"parameters": [{
"parameters": [
{
"name": "file_id",
"in": "query",
"required": true,
@@ -487,7 +545,8 @@
"type": "integer",
"format": "int64"
}
}],
}
],
"responses": {
"200": {
"description": "OK",
@@ -504,10 +563,13 @@
},
"/oss/besedilaPoIskanju": {
"get": {
"tags": ["oss"],
"tags": [
"oss"
],
"summary": "Vrne seznam besedil glede na iskalne pogoje",
"operationId": "getTexts",
"parameters": [{
"parameters": [
{
"name": "leta",
"in": "query",
"required": true,
@@ -518,7 +580,8 @@
"format": "int64"
}
}
}, {
},
{
"name": "vrste",
"in": "query",
"required": true,
@@ -528,7 +591,8 @@
"type": "string"
}
}
}, {
},
{
"name": "kljucnebesede",
"in": "query",
"required": true,
@@ -538,7 +602,8 @@
"type": "string"
}
}
}, {
},
{
"name": "cerifpodrocja",
"in": "query",
"required": true,
@@ -549,7 +614,8 @@
"format": "int64"
}
}
}],
}
],
"responses": {
"200": {
"description": "OK",
swagger_server/controllers/marktext_controller.py
+6 -2
View File
@@ -1,7 +1,9 @@
import connexion
import six
from swagger_server.models.oznaci_besedilo_body import OznaciBesediloBody # noqa: E501
from swagger_server import util
from swagger_server.classla import cl_utils
def get_text(body): # noqa: E501
@@ -15,5 +17,7 @@ def get_text(body): # noqa: E501
:rtype: str
"""
if connexion.request.is_json:
body = str.from_dict(connexion.request.get_json()) # noqa: E501
return 'do some magic!'
body = OznaciBesediloBody.from_dict(connexion.request.get_json()) # noqa: E501
conllu = cl_utils.raw_text_to_conllu(body.besedilo)
return conllu
swagger_server/controllers/oss_controller.py
+14 -1
View File
@@ -1,8 +1,11 @@
import connexion
import six
import os
from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat # noqa: E501
from swagger_server import util
from flask import send_file
from swagger_server.db_utils import Ngrams_Manager
def get_conllu(file_id): # noqa: E501
@@ -15,7 +18,17 @@ def get_conllu(file_id): # noqa: E501
:rtype: str
"""
return 'do some magic!'
file_path = util.get_conllu_path_by_id(file_id)
a = Ngrams_Manager.get_by_file_id(file_id)
print(a)
try:
return send_file(file_path, attachment_filename=f'{file_id}.conllu', as_attachment=True)
except Exception as e:
if "The system cannot find the file specified" in str(e):
return "The conllu with this ID doesn't exist.", 404
return str(e), 400
def get_conllus(leta, vrste, kljucnebesede, cerifpodrocja): # noqa: E501
swagger_server/db_utils.py
+52
View File
@@ -0,0 +1,52 @@
import mariadb
import sys
# todo: DO NOT PUSH THIS TO GIT
database_info = {
'database': 'TEMPORARYVALJUSTFORTHECOMMITPURPOSE',
'host': 'TEMPORARYVALJUSTFORTHECOMMITPURPOSE',
'port': 'TEMPORARYVALJUSTFORTHECOMMITPURPOSE',
'user': 'TEMPORARYVALJUSTFORTHECOMMITPURPOSE',
'password': 'TEMPORARYVALJUSTFORTHECOMMITPURPOSE'
}
# Connect to MariaDB Platform
try:
conn = mariadb.connect(**database_info)
except mariadb.Error as e:
print(f"Error connecting to MariaDB Platform: {e}")
sys.exit(1)
# Get Cursor
cur = conn.cursor()
# class BaseModel(Model):
# class Meta:
# database = db
#
#
# class os2022_ngrams(BaseModel):
# file_id = IntegerField()
# sent_id = FloatField()
# ngram_len = IntegerField()
# frequency_g_t = IntegerField()
# gram_text = TextField()
# lemma_text = TextField()
# xpos_text = TextField()
# upos_text = TextField()
#
# db.connect()
class Ngrams_Manager:
@staticmethod
def get_by_file_id(file_id):
try:
# cur.execute(f'SELECT * from os2022_ngrams WHERE file_id = {file_id}')
# cur.execute(f'SELECT COUNT(*) FROM os2022_ngrams')
# return list(cur)
return 1
except Exception as e:
print(e, 'EXC')
return 0
swagger_server/models/__init__.py
+1
View File
@@ -6,4 +6,5 @@ from __future__ import absolute_import
from swagger_server.models.datoteka_v_besedilo_body import DatotekaVBesediloBody
from swagger_server.models.datoteka_v_besedilo_ocr_body import DatotekaVBesediloOcrBody
from swagger_server.models.izlusci_body import IzlusciBody
from swagger_server.models.oznaci_besedilo_body import OznaciBesediloBody
from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat
swagger_server/models/oznaci_besedilo_body.py
+62
View File
@@ -0,0 +1,62 @@
# coding: utf-8
from __future__ import absolute_import
from datetime import date, datetime # noqa: F401
from typing import List, Dict # noqa: F401
from swagger_server.models.base_model_ import Model
from swagger_server import util
class OznaciBesediloBody(Model):
"""NOTE: This class is auto generated by the swagger code generator program.
Do not edit the class manually.
"""
def __init__(self, besedilo: str=None): # noqa: E501
"""OznaciBesediloBody - a model defined in Swagger
:param besedilo: The besedilo of this OznaciBesediloBody. # noqa: E501
:type besedilo: str
"""
self.swagger_types = {
'besedilo': str
}
self.attribute_map = {
'besedilo': 'besedilo'
}
self._besedilo = besedilo
@classmethod
def from_dict(cls, dikt) -> 'OznaciBesediloBody':
"""Returns the dict as a model
:param dikt: A dict.
:type: dict
:return: The oznaciBesedilo_body of this OznaciBesediloBody. # noqa: E501
:rtype: OznaciBesediloBody
"""
return util.deserialize_model(dikt, cls)
@property
def besedilo(self) -> str:
"""Gets the besedilo of this OznaciBesediloBody.
:return: The besedilo of this OznaciBesediloBody.
:rtype: str
"""
return self._besedilo
@besedilo.setter
def besedilo(self, besedilo: str):
"""Sets the besedilo of this OznaciBesediloBody.
:param besedilo: The besedilo of this OznaciBesediloBody.
:type besedilo: str
"""
self._besedilo = besedilo
swagger_server/swagger/swagger.yaml
+6 -1
View File
@@ -17,7 +17,7 @@ paths:
content:
application/json:
schema:
type: string
$ref: '#/components/schemas/oznaciBesedilo_body'
required: true
responses:
"200":
@@ -485,6 +485,11 @@ components:
kanonicnaoblika: kanonicnaoblika
nosilnautez: 0.8008282
kandidat: kandidat
oznaciBesedilo_body:
type: object
properties:
besedilo:
type: string
izlusci_body:
type: object
properties:
swagger_server/test/test_marktext_controller.py
+2 -1
View File
@@ -5,6 +5,7 @@ from __future__ import absolute_import
from flask import json
from six import BytesIO
from swagger_server.models.oznaci_besedilo_body import OznaciBesediloBody # noqa: E501
from swagger_server.test import BaseTestCase
@@ -16,7 +17,7 @@ class TestMarktextController(BaseTestCase):
Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format
"""
body = 'body_example'
body = OznaciBesediloBody()
response = self.client.open(
'/oznaciBesedilo',
method='POST',
swagger_server/util.py
+4
View File
@@ -140,3 +140,7 @@ def _deserialize_dict(data, boxed_type):
"""
return {k: _deserialize(v, boxed_type)
for k, v in six.iteritems(data)}
def get_conllu_path_by_id(file_id):
return f'..\\mnt\\ssd\\ds_ftp\\classla_OS2022\\conll\\rsdo_doc-{file_id}.plainText.conllu'