diff --git a/docker-compose.yml b/docker-compose.yml index a360e70..a11e7e1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: restart: always image: flask-server ports: - - "8081:8080" + - "8080:8080" volumes: # when pushing to server, remove the first dot here in front of /mnt - /mnt/ssd/ds_ftp/classla_OS2022:/usr/src/app/classla_OS2022 - ../classla/classla_resources:/root/classla_resources diff --git a/rsdo5.json b/rsdo5.json new file mode 100644 index 0000000..e0b176e --- /dev/null +++ b/rsdo5.json @@ -0,0 +1,1090 @@ +{ + "openapi": "3.0.1", + "info": { + "title": "OpenAPI definition", + "version": "v0" + }, + "servers": [ + { + "url": "http://localhost:8089", + "description": "Generated server url" + } + ], + "paths": { + "/job/{job_id}": { + "get": { + "tags": [ + "jobs" + ], + "summary": "Vrne status", + "operationId": "getJobStatus", + "parameters": [ + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JobResponse" + } + } + } + } + } + }, + "delete": { + "tags": [ + "jobs" + ], + "summary": "Izbriše job", + "operationId": "deleteJob", + "parameters": [ + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/oznaciBesediloAsync": { + "post": { + "tags": [ + "marktext-async" + ], + "summary": "Označi surovo (angl. raw) besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format", + "operationId": "getText", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "besedilo": { + "type": "string" + } + } + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/datotekaVBesediloAsync": { + "post": { + "tags": [ + "marktext-async" + ], + "summary": "Pretvori datoteko v besedilo, vrača tekst", + "operationId": "getTextFromDocAsync", + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "required": [ + "file" + ], + "type": "object", + "properties": { + "file": { + "type": "string", + "format": "binary" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/datotekaVConlluAsync": { + "post": { + "tags": [ + "marktext-async" + ], + "summary": "Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format", + "operationId": "getConlluFromFileAsync", + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "required": [ + "file" + ], + "type": "object", + "properties": { + "file": { + "type": "string", + "format": "binary" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/datotekaVConlluAsync/ocr": { + "post": { + "tags": [ + "marktext-async" + ], + "summary": "Pretvori datoteko v besedilo in označi s classlo/stanzo z uporabo slovenskih modelov ter vrne conll-u format", + "operationId": "getConlluFromFileOcrAsync", + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "required": [ + "file" + ], + "type": "object", + "properties": { + "file": { + "type": "string", + "format": "binary" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/datotekaVBesediloAsync/ocr": { + "post": { + "tags": [ + "marktext-async" + ], + "summary": "Pretvori datoteko v besedilo s pomočjo ocr razpoznavanja, vrača tekst", + "operationId": "getTextFromFileOcrAsync", + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "required": [ + "file" + ], + "type": "object", + "properties": { + "file": { + "type": "string", + "format": "binary" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/izlusciSync": { + "post": { + "tags": [ + "extract" + ], + "summary": "Izlusci terminološke kandidate iz seznama besedil v conllu obliki [sihrono, rezultat v sami zahtevi]", + "operationId": "getCandidates_sync", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "conllus": { + "type": "array", + "items": { + "type": "string" + } + }, + "prepovedaneBesede": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/TerminoloskiKandidat" + } + } + } + } + } + } + } + }, + "/izlusciAsync": { + "post": { + "tags": [ + "extract" + ], + "summary": "Izlusci terminološke kandidate iz seznama besedil v conllu obliki [asinhrono, ustvari novi job]", + "operationId": "getCandidates_async", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "conllus": { + "type": "array", + "items": { + "type": "string" + } + }, + "prepovedaneBesede": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/datotekaVBesediloSync": { + "post": { + "tags": [ + "marktext-sync" + ], + "summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vrača besedilo", + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "required": [ + "file" + ], + "type": "object", + "properties": { + "file": { + "type": "string", + "format": "binary" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/datotekaVBesediloSync/ocr": { + "post": { + "tags": [ + "marktext-sync" + ], + "summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo s pomočjo ocr razpoznavanja", + "operationId": "getTextOcr", + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "required": [ + "file" + ], + "type": "object", + "properties": { + "file": { + "type": "string", + "format": "binary" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/datotekaVConlluSync": { + "post": { + "tags": [ + "marktext-sync" + ], + "summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vrača conllu", + "operationId": "datotekaVBesediloInClassla", + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "required": [ + "file" + ], + "type": "object", + "properties": { + "file": { + "type": "string", + "format": "binary" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/datotekaVConlluSync/ocr": { + "post": { + "tags": [ + "marktext-sync" + ], + "summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v conllu s pomočjo ocr razpoznavanja", + "operationId": "getConlluOcr", + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "required": [ + "file" + ], + "type": "object", + "properties": { + "file": { + "type": "string", + "format": "binary" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/oss/steviloBesedilPoIskanju": { + "get": { + "tags": [ + "oss" + ], + "summary": "Vrne število besedil glede na iskalne pogoje", + "operationId": "getNumberTexts", + "parameters": [ + {"$ref": "#/components/parameters/leta"}, + {"$ref": "#/components/parameters/vrste"}, + {"$ref": "#/components/parameters/kljucneBesede"}, + {"$ref": "#/components/parameters/udk"} + ], + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "integer", + "format": "int64" + } + } + } + } + } + }, + "post": { + "tags": [ + "oss-post" + ], + "summary": "Vrne število besedil glede na iskalne pogoje", + "operationId": "getNumberTexts_post", + "parameters": [ + {"$ref": "#/components/parameters/leta"}, + {"$ref": "#/components/parameters/vrste"}, + {"$ref": "#/components/parameters/kljucneBesede"}, + {"$ref": "#/components/parameters/udk"} + ], + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "integer", + "format": "int64" + } + } + } + } + } + } + }, + "/oss/izlusciPoIskanju": { + "get": { + "tags": [ + "oss" + ], + "summary": "Vrne terminloške kandidate glede na ... (sync)", + "operationId": "getExtractedWords", + "parameters": [ + {"$ref": "#/components/parameters/leta"}, + {"$ref": "#/components/parameters/vrste"}, + {"$ref": "#/components/parameters/kljucneBesede"}, + {"$ref": "#/components/parameters/prepovedaneBesede"}, + {"$ref": "#/components/parameters/udk"} + ], + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/TerminoloskiKandidat" + } + } + } + } + } + } + }, + "post": { + "tags": [ + "oss-post" + ], + "summary": "Vrne terminloške kandidate glede na ... (sync)", + "operationId": "getExtractedWords_post", + "parameters": [ + {"$ref": "#/components/parameters/leta"}, + {"$ref": "#/components/parameters/vrste"}, + {"$ref": "#/components/parameters/kljucneBesede"}, + {"$ref": "#/components/parameters/prepovedaneBesede"}, + {"$ref": "#/components/parameters/udk"} + ], + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/TerminoloskiKandidat" + } + } + } + } + } + } + } + }, + "/oss/izlusciPoIskanjuAsync": { + "get": { + "tags": [ + "oss" + ], + "summary": "Vrne terminloške kandidate glede na ... (async)", + "operationId": "getExtractedWordsAsync", + "parameters": [ + {"$ref": "#/components/parameters/leta"}, + {"$ref": "#/components/parameters/vrste"}, + {"$ref": "#/components/parameters/kljucneBesede"}, + {"$ref": "#/components/parameters/prepovedaneBesede"}, + {"$ref": "#/components/parameters/udk"} + ], + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "string" + } + } + } + } + } + }, + "post": { + "tags": [ + "oss-post" + ], + "summary": "Vrne terminloške kandidate glede na ... (async)", + "operationId": "getExtractedWordsAsync_post", + "parameters": [ + {"$ref": "#/components/parameters/leta"}, + {"$ref": "#/components/parameters/vrste"}, + {"$ref": "#/components/parameters/kljucneBesede"}, + {"$ref": "#/components/parameters/prepovedaneBesede"}, + {"$ref": "#/components/parameters/udk"} + ], + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/oss/datotekePoIskanju": { + "get": { + "tags": [ + "oss" + ], + "summary": "Vrne seznam binarnih zapisov v originalnem formatu glede na iskalne pogoje", + "operationId": "getFiles", + "parameters": [ + {"$ref": "#/components/parameters/leta"}, + {"$ref": "#/components/parameters/vrste"}, + {"$ref": "#/components/parameters/kljucneBesede"}, + {"$ref": "#/components/parameters/udk"} + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/octet-stream": { + "schema": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string", + "format": "byte" + } + } + } + } + } + } + } + }, + "post": { + "tags": [ + "oss-post" + ], + "summary": "Vrne seznam binarnih zapisov v originalnem formatu glede na iskalne pogoje", + "operationId": "getFiles_post", + "parameters": [ + {"$ref": "#/components/parameters/leta"}, + {"$ref": "#/components/parameters/vrste"}, + {"$ref": "#/components/parameters/kljucneBesede"}, + {"$ref": "#/components/parameters/udk"} + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/octet-stream": { + "schema": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string", + "format": "byte" + } + } + } + } + } + } + } + } + }, + "/oss/datotekaPoId": { + "get": { + "tags": [ + "oss" + ], + "summary": "Vrne binarni zapis v originalnem formatu po id-ju datoteke", + "operationId": "getFile", + "parameters": [ + { + "name": "file_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/octet-stream": { + "schema": { + "type": "array", + "items": { + "type": "string", + "format": "byte" + } + } + } + } + } + } + } + }, + "/oss/conlluPoIskanju": { + "get": { + "tags": [ + "oss" + ], + "summary": "Vrne seznam CoNNL-U-jev glede na iskalne pogoje", + "operationId": "getConllus", + "parameters": [ + {"$ref": "#/components/parameters/leta"}, + {"$ref": "#/components/parameters/vrste"}, + {"$ref": "#/components/parameters/kljucneBesede"}, + {"$ref": "#/components/parameters/udk"} + ], + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + }, + "post": { + "tags": [ + "oss-post" + ], + "summary": "Vrne seznam CoNNL-U-jev glede na iskalne pogoje", + "operationId": "getConllus_post", + "parameters": [ + {"$ref": "#/components/parameters/leta"}, + {"$ref": "#/components/parameters/vrste"}, + {"$ref": "#/components/parameters/kljucneBesede"}, + {"$ref": "#/components/parameters/udk"} + ], + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + }, + "/oss/conlluPoId": { + "get": { + "tags": [ + "oss" + ], + "summary": "Vrne CoNNL-U po id-ju datoteke", + "operationId": "getConllu", + "parameters": [ + { + "name": "file_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/oss/besediloPoId": { + "get": { + "tags": [ + "oss" + ], + "summary": "Vrne besedilo po id-ju datoteke", + "parameters": [ + { + "name": "file_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/oss/besedilaPoIskanju": { + "get": { + "tags": [ + "oss" + ], + "summary": "Vrne seznam besedil glede na iskalne pogoje", + "operationId": "getTexts", + "parameters": [ + {"$ref": "#/components/parameters/leta"}, + {"$ref": "#/components/parameters/vrste"}, + {"$ref": "#/components/parameters/kljucneBesede"}, + {"$ref": "#/components/parameters/udk"} + ], + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + }, + "post": { + "tags": [ + "oss-post" + ], + "summary": "Vrne seznam besedil glede na iskalne pogoje", + "operationId": "getTexts_post", + "parameters": [ + {"$ref": "#/components/parameters/leta"}, + {"$ref": "#/components/parameters/vrste"}, + {"$ref": "#/components/parameters/kljucneBesede"}, + {"$ref": "#/components/parameters/udk"} + ], + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "components": { + "parameters": { + "leta": { + "name": "leta", + "in": "query", + "required": false, + "schema": { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + } + } + }, + "vrste": { + "name": "vrste", + "in": "query", + "required": false, + "schema": { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + } + } + }, + "kljucneBesede": { + "name": "kljucneBesede", + "in": "query", + "required": false, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "prepovedaneBesede": { + "name": "prepovedaneBesede", + "in": "query", + "required": false, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "udk": { + "name": "udk", + "in": "query", + "required": false, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "schemas": { + "TerminoloskiKandidat": { + "type": "object", + "properties": { + "kandidat": { + "type": "string" + }, + "kanonicnaoblika": { + "type": "string" + }, + "POSoznake": { + "type": "string" + }, + "nosilnautez": { + "type": "number", + "format": "float" + }, + "podporneutezi": { + "type": "array", + "items": { + "type": "number", + "format": "float" + } + }, + "pogostostpojavljanja": { + "type": "number", + "format": "int64" + } + } + }, + "JobResponse": { + "type": "object", + "properties": { + "job_status": { + "type": "string", + "enum": [ + "waiting in que", + "currently processing", + "finished processing (OK)", + "finished processing (ERROR)" + ] + }, + "finished_on": { + "type": "string", + "format": "date-time" + }, + "started_on": { + "type": "string", + "format": "date-time" + }, + "created_on": { + "type": "string", + "format": "date-time" + }, + "job_result": { + "type": "string" + } + }, + "required": [ + "finished_job" + ] + } + } + } +} \ No newline at end of file diff --git a/swagger_server/DB/.gitignore b/swagger_server/DB/.gitignore new file mode 100644 index 0000000..86d0cb2 --- /dev/null +++ b/swagger_server/DB/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore \ No newline at end of file diff --git a/swagger_server/__main__.py b/swagger_server/__main__.py index a00597f..12a62d9 100644 --- a/swagger_server/__main__.py +++ b/swagger_server/__main__.py @@ -7,6 +7,7 @@ from swagger_server import encoder def main(): app = connexion.App(__name__, specification_dir='./swagger/') app.app.json_encoder = encoder.JSONEncoder + app.app.config['JSON_SORT_KEYS'] = False app.add_api('swagger.yaml', arguments={'title': 'OpenAPI definition'}, pythonic_params=True) app.run(port=8080) diff --git a/swagger_server/controllers/oss_post_controller.py b/swagger_server/controllers/oss_post_controller.py new file mode 100644 index 0000000..f8f074d --- /dev/null +++ b/swagger_server/controllers/oss_post_controller.py @@ -0,0 +1,127 @@ +import connexion +import six + +from swagger_server.controllers.oss_controller import get_conllus, get_extracted_words, get_extracted_words_async, \ + get_files, get_number_texts, get_texts +from swagger_server.models.terminoloski_kandidat import TerminoloskiKandidat # noqa: E501 +from swagger_server import util + + +def get_conllus_post(leta=None, vrste=None, kljucne_besede=None, udk=None): # noqa: E501 + """Vrne seznam CoNNL-U-jev glede na iskalne pogoje + + # noqa: E501 + + :param leta: + :type leta: List[int] + :param vrste: + :type vrste: List[int] + :param kljucne_besede: + :type kljucne_besede: List[str] + :param udk: + :type udk: List[str] + + :rtype: List[str] + """ + return get_conllus(leta, vrste, kljucne_besede, udk) + + +def get_extracted_words_async_post(leta=None, vrste=None, kljucne_besede=None, prepovedane_besede=None, + udk=None,definicije=False): # noqa: E501 + """Vrne terminloške kandidate glede na ... (async) + + # noqa: E501 + + :param leta: + :type leta: List[int] + :param vrste: + :type vrste: List[int] + :param kljucne_besede: + :type kljucne_besede: List[str] + :param prepovedane_besede: + :type prepovedane_besede: List[str] + :param udk: + :type udk: List[str] + + :rtype: str + """ + return get_extracted_words_async(leta, vrste, kljucne_besede, prepovedane_besede, udk,,definicije) + + +def get_extracted_words_post(leta=None, vrste=None, kljucne_besede=None, prepovedane_besede=None, + udk=None,definicije=False): # noqa: E501 + """Vrne terminloške kandidate glede na ... (sync) + + # noqa: E501 + + :param leta: + :type leta: List[int] + :param vrste: + :type vrste: List[int] + :param kljucne_besede: + :type kljucne_besede: List[str] + :param prepovedane_besede: + :type prepovedane_besede: List[str] + :param udk: + :type udk: List[str] + + :rtype: List[TerminoloskiKandidat] + """ + return get_extracted_words(leta, vrste, kljucne_besede, prepovedane_besede, udk,definicije) + + +def get_files_post(leta=None, vrste=None, kljucne_besede=None, udk=None): # noqa: E501 + """Vrne seznam binarnih zapisov v originalnem formatu glede na iskalne pogoje + + # noqa: E501 + + :param leta: + :type leta: List[int] + :param vrste: + :type vrste: List[int] + :param kljucne_besede: + :type kljucne_besede: List[str] + :param udk: + :type udk: List[str] + + :rtype: List[List[bytearray]] + """ + return get_files(leta, vrste, kljucne_besede, udk) + + +def get_number_texts_post(leta=None, vrste=None, kljucne_besede=None, udk=None): # noqa: E501 + """Vrne število besedil glede na iskalne pogoje + + # noqa: E501 + + :param leta: + :type leta: List[int] + :param vrste: + :type vrste: List[int] + :param kljucne_besede: + :type kljucne_besede: List[str] + :param udk: + :type udk: List[str] + + :rtype: int + """ + return get_number_texts(leta, vrste, kljucne_besede, udk) + + +def get_texts_post(leta=None, vrste=None, kljucne_besede=None, udk=None): # noqa: E501 + """Vrne seznam besedil glede na iskalne pogoje + + # noqa: E501 + + :param leta: + :type leta: List[int] + :param vrste: + :type vrste: List[int] + :param kljucne_besede: + :type kljucne_besede: List[str] + :param udk: + :type udk: List[str] + + :rtype: List[str] + """ + return get_texts(leta, vrste, kljucne_besede, udk) diff --git a/swagger_server/models/job_response.py b/swagger_server/models/job_response.py index 2a0bfc0..5ea0c50 100644 --- a/swagger_server/models/job_response.py +++ b/swagger_server/models/job_response.py @@ -78,7 +78,7 @@ class JobResponse(Model): :param job_status: The job_status of this JobResponse. :type job_status: str """ - allowed_values = ["waiting in que", "currently processing", "finished processing"] # noqa: E501 + allowed_values = ["waiting in que", "currently processing", "finished processing (OK)", "finished processing (ERROR)"] # noqa: E501 if job_status not in allowed_values: raise ValueError( "Invalid value for `job_status` ({0}), must be one of {1}" diff --git a/swagger_server/requets_db/models/vrsta.py b/swagger_server/requets_db/models/vrsta.py index 01c4558..88527e7 100644 --- a/swagger_server/requets_db/models/vrsta.py +++ b/swagger_server/requets_db/models/vrsta.py @@ -7,22 +7,14 @@ import os from swagger_server.util import get_random_filename from werkzeug.utils import secure_filename -LOCAL_PATH = "requets_db/dbs" -SERVER_PATH = "swagger_server/requets_db/dbs" - -if not os.path.exists(LOCAL_PATH): - if os.path.exists('requets_db'): - os.makedirs(LOCAL_PATH, exist_ok=True) - elif os.path.exists('swagger_server/requets_db'): - os.makedirs(SERVER_PATH, exist_ok=True) - -DB = f'{LOCAL_PATH}/jobs.db' -if not os.path.exists(LOCAL_PATH): - DB = f'{SERVER_PATH}/jobs.db' -db = SqliteDatabase(DB, pragmas={ - # 'journal_mode': 'wal', - 'cache_size': -1 * 128 * 1024, # 128MB - 'foreign_keys': 1 +DB_Path = 'DB/jobs.db' +if not os.path.exists('DB'): + os.makedirs('DB', exist_ok=True) +db = SqliteDatabase(DB_Path, pragmas={ + 'journal_mode': 'wal', + 'cache_size': -1 * 256 * 1024, # 256MB + 'foreign_keys': 1, + 'charset': 'utf8' }) @@ -43,7 +35,7 @@ class Job(BaseModel): input_file = TextField(index=True, null=True) -db.drop_tables([Job]) # TODO: After pushing this, comment it and push again +# db.drop_tables([Job]) # If pushing this uncommented, comment it and push again db.create_tables([Job]) @@ -55,12 +47,13 @@ class JobManager: :possibilities: # 1 = pretvori datoteko v besedilo, 2 = oznaci besedilo, 12 = oboje # 3 = pretvori dat v besedilo OCR, 2 = oznaci besedilo, 32 = oboje - # 4 = izlusci async + # 4 = izlusci async glede na vhodne conlluje + # 5 = izlusci po iskanju async :return: Job object, Did already exist boolean """ try: - if job_type in [2, 4]: + if job_type in [2, 4, 5]: job, is_new = Job.get_or_create(job_type=job_type, job_input=job_input, input_size=len(job_input)) elif job_type in [1, 3, 12, 32]: tmp_file = "" @@ -72,7 +65,8 @@ class JobManager: pathlib.Path('tmp').mkdir(exist_ok=True) job_input: werkzeug.datastructures.FileStorage job_input.save(tmp_file) - job, is_new = Job.get_or_create(job_type=job_type, input_file=tmp_file, input_size=-1) + job, is_new = Job.get_or_create(job_type=job_type, input_file=tmp_file, + input_size=os.stat(tmp_file).st_size) return job, is_new except Exception as e: diff --git a/swagger_server/utils/cl_utils.py b/swagger_server/utils/cl_utils.py index d9a75e1..5020be2 100644 --- a/swagger_server/utils/cl_utils.py +++ b/swagger_server/utils/cl_utils.py @@ -10,8 +10,11 @@ nlp_loaded = True sent_extractor = re.compile(r"# sent_id = \d+\.\d+(.*?)\n\n", re.MULTILINE | re.DOTALL) + def raw_text_to_conllu(text): try: + if text == '' or text is None: + raise Exception("Text can not be empty (when trying to make conllu for it)") docall = nlpSlo(text) docallconllu = docall.to_conll()