webapps/term_extractor
18
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Dodana tika & format (v swagger definiciji se ni nic novega v tem commitu)

Browse Source
This commit is contained in:
Kikimanox
2022-08-31 14:49:47 +02:00
parent 5a87193490
commit c30b014a68
4 changed files with 316 additions and 294 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rsdo5.json
+1 -1
View File
@@ -179,7 +179,7 @@
"tags": [
"doc-2text"
],
"summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo",
"summary": "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... vraca besedilo",
"requestBody": {
"content": {
"multipart/form-data": {
swagger_server/controllers/doc2text_controller.py
+30 -15
View File
@@ -1,8 +1,10 @@
from tempfile import TemporaryFile
import connexion
import six
# import tika
import requests
import json
from swagger_server import util
import pandas as pd
import docx
@@ -10,6 +12,11 @@ import codecs
import xml.etree.ElementTree as ET
from PyPDF2 import PdfReader
tika_server = "http://tika:9998/tika"
tika_server = "http://rsdo.lhrs.feri.um.si:9998/tika"
def datoteka_v_besedilo_post(file=None): # noqa: E501
"""Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo
@@ -21,25 +28,33 @@ def datoteka_v_besedilo_post(file=None): # noqa: E501
:rtype: str
"""
#zakaj ? tika server se požene zraven v dodatnem containerju in se datoteka samo pošlje tja, nazaj pa se dobi čisto besedilo...
#tika ima rest api
if file is None:
return "No file provided", 400
tika_server="http://tika:9998/tika"
response=requests.post(tika_server, data=file)
return response.text, 200
try:
response = requests.put(tika_server, data=file)
return response.text, 200
except Exception as e:
return str(e), 500
def get_text_ocr(file=None): # noqa: E501
def get_text_ocr(file=None):
"""Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo s pomočjo ocr razpoznavanja
# noqa: E501
:param file:
:type file: strstr
:rtype: str
"""
if file is None:
return "No file provided", 400
tika_server="http://tika:9998/tika"
#za ocr se zraven samo doda header
response=requests.post(tika_server, data=file,headers={"X-Tika-PDFOcrStrategy":"ocr_only","X-Tika-OCRLanguage":"slv+eng"})
return response.text, 200
try:
response = requests.put(tika_server, data=file,
headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng"})
return response.text, 200
except Exception as e:
return str(e), 500
swagger_server/controllers/extract_controller.py
+24 -25
View File
@@ -19,53 +19,52 @@ def get_candidates(body): # noqa: E501
if connexion.request.is_json:
body = IzlusciBody.from_dict(connexion.request.get_json()) # noqa: E501
data={
terminoloski_kandidati:[
{
data = {
"terminoloski_kandidati": [
{
"POSoznake": "Ncmsn",
"kandidat": "vpliv",
"kanonicnaoblika": "vpliv",
"nosilnautez": 0.8008282,
"podporneutezi": [
6.0274563,
6.0274563
6.0274563,
6.0274563
],
"pogostostpojavljanja": [101,71]
},
{
"pogostostpojavljanja": [101, 71]
},
{
"POSoznake": "Agpnsg Ncnsg",
"kandidat": "bivalen okolje",
"kanonicnaoblika": "bivalno okolje",
"nosilnautez": 0.60254,
"podporneutezi": [
3.263,
2.134
3.263,
2.134
],
"pogostostpojavljanja": [27,11]
},
{
"pogostostpojavljanja": [27, 11]
},
{
"POSoznake": "Agpmsny Ncmsn Ncmpg",
"kandidat": "motorični status otrok",
"kanonicnaoblika": "motorični status otrok",
"nosilnautez": 0.3324,
"podporneutezi": [
1.221,
3.323
1.221,
3.323
],
"pogostostpojavljanja": [31,51]
},
{
"pogostostpojavljanja": [31, 51]
},
{
"POSoznake": "Agpnsn Ncnsn",
"kandidat": "diplomsko delo",
"kanonicnaoblika": "diplomsko delo",
"nosilnautez": 0.2008282,
"podporneutezi": [
0.883,
1.02
0.883,
1.02
],
"pogostostpojavljanja": [1241,111]
}
]
}
"pogostostpojavljanja": [1241, 111]
}
]
}
return data
swagger_server/swagger/swagger.yaml
+261 -253
View File
@@ -3,33 +3,33 @@ info:
title: OpenAPI definition
version: v0
servers:
- url: http://localhost:8089
description: Generated server url
- url: http://localhost:8089
description: Generated server url
paths:
/job/{job_id}:
get:
tags:
- jobs
- jobs
summary: Vrne status
operationId: get_job_status
parameters:
- name: job_id
in: path
required: true
style: simple
explode: false
schema:
type: integer
format: int64
- name: show_estimated_completion
in: query
description: Calculate estimate time remaining based on various factors (could
be inaccurate)
required: false
style: form
explode: true
schema:
type: boolean
- name: job_id
in: path
required: true
style: simple
explode: false
schema:
type: integer
format: int64
- name: show_estimated_completion
in: query
description: Calculate estimate time remaining based on various factors (could
be inaccurate)
required: false
style: form
explode: true
schema:
type: boolean
responses:
"200":
description: OK
@@ -41,7 +41,7 @@ paths:
/oznaciBesedilo:
post:
tags:
- marktext
- marktext
summary: Označi besedilo s classlo/stanzo z uporabo slovenskih modelov ter vrne
conll-u format
operationId: get_text
@@ -63,7 +63,7 @@ paths:
/izlusci:
post:
tags:
- extract
- extract
summary: Izlusci terminološke kandidate iz seznama besedil v conllu obliki
operationId: get_candidates
requestBody:
@@ -86,7 +86,7 @@ paths:
/datotekaVBesedilo/ocr:
post:
tags:
- doc-2text
- doc-2text
summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo\
\ s pomočjo ocr razpoznavanja"
operationId: get_text_ocr
@@ -107,7 +107,7 @@ paths:
/datotekaVBesedilo/:
post:
tags:
- doc-2text
- doc-2text
summary: "Pretvori datoteko formata pdf, doc, docx, ppt, xls,... v besedilo"
operationId: datoteka_v_besedilo_post
requestBody:
@@ -127,48 +127,48 @@ paths:
/oss/steviloBesedilPoIskanju:
get:
tags:
- oss
- oss
summary: Vrne število besedil glede na iskalne pogoje
operationId: get_number_texts
parameters:
- name: leta
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: vrste
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: kljucnebesede
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: cerifpodrocja
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: leta
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: vrste
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: kljucnebesede
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: cerifpodrocja
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
responses:
"200":
description: OK
@@ -182,48 +182,48 @@ paths:
/oss/izlusciPoIskanju:
get:
tags:
- oss
- oss
summary: 'Vrne terminloške kandidate glede na '
operationId: get_extracted_words
parameters:
- name: leta
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: vrste
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: kljucnebesede
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: cerifpodrocja
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: leta
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: vrste
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: kljucnebesede
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: cerifpodrocja
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
responses:
"200":
description: OK
@@ -238,49 +238,49 @@ paths:
/oss/datotekePoIskanju:
get:
tags:
- oss
- oss
summary: Vrne seznam binarnih zapisov v originalnem formatu glede na iskalne
pogoje
operationId: get_files
parameters:
- name: leta
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: vrste
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: kljucnebesede
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: cerifpodrocja
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: leta
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: vrste
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: kljucnebesede
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: cerifpodrocja
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
responses:
"200":
description: OK
@@ -298,18 +298,18 @@ paths:
/oss/datotekaPoId:
get:
tags:
- oss
- oss
summary: Vrne binarni zapis v originalnem formatu po id-ju datoteke
operationId: get_file
parameters:
- name: file_id
in: query
required: true
style: form
explode: true
schema:
type: integer
format: int64
- name: file_id
in: query
required: true
style: form
explode: true
schema:
type: integer
format: int64
responses:
"200":
description: OK
@@ -325,48 +325,48 @@ paths:
/oss/conlluPoIskanju:
get:
tags:
- oss
- oss
summary: Vrne seznam CoNNL-U-jev glede na iskalne pogoje
operationId: get_conllus
parameters:
- name: leta
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: vrste
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: kljucnebesede
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: cerifpodrocja
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: leta
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: vrste
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: kljucnebesede
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: cerifpodrocja
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
responses:
"200":
description: OK
@@ -381,18 +381,18 @@ paths:
/oss/conlluPoId:
get:
tags:
- oss
- oss
summary: Vrne CoNNL-U po id-ju datoteke
operationId: get_conllu
parameters:
- name: file_id
in: query
required: true
style: form
explode: true
schema:
type: integer
format: int64
- name: file_id
in: query
required: true
style: form
explode: true
schema:
type: integer
format: int64
responses:
"200":
description: OK
@@ -405,18 +405,18 @@ paths:
/oss/besediloPoId:
get:
tags:
- oss
- oss
summary: Vrne besedilo po id-ju datoteke
operationId: oss_besedilo_po_id_get
parameters:
- name: file_id
in: query
required: true
style: form
explode: true
schema:
type: integer
format: int64
- name: file_id
in: query
required: true
style: form
explode: true
schema:
type: integer
format: int64
responses:
"200":
description: OK
@@ -429,48 +429,48 @@ paths:
/oss/besedilaPoIskanju:
get:
tags:
- oss
- oss
summary: Vrne seznam besedil glede na iskalne pogoje
operationId: get_texts
parameters:
- name: leta
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: vrste
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: kljucnebesede
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: cerifpodrocja
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: leta
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
- name: vrste
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: kljucnebesede
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: string
- name: cerifpodrocja
in: query
required: true
style: form
explode: true
schema:
type: array
items:
type: integer
format: int64
responses:
"200":
description: OK
@@ -508,18 +508,18 @@ components:
format: int64
example:
podporneutezi:
- 6.0274563
- 6.0274563
- 6.0274563
- 6.0274563
pogostostpojavljanja:
- 1
- 1
- 1
- 1
POSoznake: POSoznake
kanonicnaoblika: kanonicnaoblika
nosilnautez: 0.8008282
kandidat: kandidat
JobResponse:
required:
- finished_job
- finished_job
type: object
properties:
finished_job:
@@ -555,7 +555,7 @@ components:
type: string
datotekaVBesedilo_ocr_body:
required:
- file
- file
type: object
properties:
file:
@@ -563,7 +563,15 @@ components:
format: binary
datotekaVBesedilo_body:
required:
- file
- file
type: object
properties:
file:
type: string
format: binary
datotekaVBesediloSync_body:
required:
- file
type: object
properties:
file: