Main brez modelov
This commit is contained in:
Marko Ferme
@@ -0,0 +1,47 @@
|
||||
import classla
|
||||
import time
|
||||
from swagger_server import util
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
nlp_loaded = False
|
||||
nlpSlo = classla.Pipeline('sl', processors='tokenize,ner,pos,lemma,depparse')
|
||||
nlp_loaded = True
|
||||
|
||||
sent_extractor = re.compile(r"# sent_id = \d+\.\d+(.*?)\n\n", re.MULTILINE | re.DOTALL)
|
||||
|
||||
def raw_text_to_conllu(text):
|
||||
try:
|
||||
docall = nlpSlo(text)
|
||||
docallconllu = docall.to_conll()
|
||||
|
||||
return docallconllu, 200
|
||||
except Exception as e:
|
||||
return e, 400
|
||||
|
||||
|
||||
def multipla_conllus_to_one_from_file_ids(list_file_ids):
|
||||
sent_cnt = 1
|
||||
ret = "# newpar id = 1\n"
|
||||
files = [f'{util.get_conllu_file_path_by_id(i)}' for i in list_file_ids]
|
||||
for file in files:
|
||||
txt = Path(file).read_text('utf-8')
|
||||
matches = sent_extractor.finditer(txt)
|
||||
for match in matches:
|
||||
ret += f'# sent_id = 1.{sent_cnt}{match.group(1)}\n\n'
|
||||
sent_cnt += 1
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def multipla_conllus_to_one_from_conllus_arr(list_conllus):
|
||||
sent_cnt = 1
|
||||
ret = "# newpar id = 1\n"
|
||||
for conllu in list_conllus:
|
||||
conllu = conllu.replace('\r\n', '\n')
|
||||
matches = sent_extractor.finditer(conllu)
|
||||
for match in matches:
|
||||
ret += f'# sent_id = 1.{sent_cnt}{match.group(1)}\n\n'
|
||||
sent_cnt += 1
|
||||
|
||||
return ret
|
||||
@@ -0,0 +1,225 @@
|
||||
import mariadb
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
import json
|
||||
|
||||
|
||||
database_info = {
|
||||
'database': os.getenv("MDB_DATABASE", "oss"),
|
||||
'host': os.getenv("MDB_HOST", "localhost"),
|
||||
'port': int(os.getenv("PORT", 3306)) ,
|
||||
'user': os.getenv("MDB_USER", "root"),
|
||||
'password': os.getenv("MDB_PASSWORD", "root"),
|
||||
}
|
||||
|
||||
canonapi_endpoint = "http://canonizer:5000/rest_api/canonize"
|
||||
|
||||
cur = None
|
||||
# Connect to MariaDB Platform
|
||||
|
||||
def get_files_by_udc(udc):
|
||||
ret = []
|
||||
|
||||
try:
|
||||
print(database_info)
|
||||
conn = mariadb.connect(**database_info)
|
||||
cur = conn.cursor()
|
||||
where_in = ','.join(['%s'] * len(udc))
|
||||
print(where_in)
|
||||
sql = "select distinct xml_id from metadata_udc where udk IN (%s)" % (where_in)
|
||||
print(sql)
|
||||
cur.execute(sql,udc)
|
||||
#cur.execute(f'SELECT COUNT(*) FROM os2022_ngrams')
|
||||
ret = list(cur)
|
||||
except mariadb.Error as e:
|
||||
print(f"Error connecting to MariaDB Platform: {e}")
|
||||
|
||||
|
||||
return ret
|
||||
|
||||
def vrni_oss_dokumente(leta, vrste, kljucnebesede, udk):
|
||||
ret = []
|
||||
|
||||
try:
|
||||
print(database_info)
|
||||
conn = mariadb.connect(**database_info)
|
||||
cur = conn.cursor()
|
||||
|
||||
|
||||
|
||||
sql = "select distinct document_id from metadata"
|
||||
where=""
|
||||
params=[]
|
||||
if (udk):
|
||||
where_in_udk = ','.join(['%s'] * len(udk))
|
||||
where=" udk IN (%s) " % (where_in_udk)
|
||||
params=udk
|
||||
|
||||
if (leta):
|
||||
|
||||
where_in_leta = ','.join(['%s'] * len(leta))
|
||||
if (where):
|
||||
where=where+ " AND "
|
||||
where=where + " leto IN (%s) " % (where_in_leta)
|
||||
params=params+leta
|
||||
|
||||
if (vrste):
|
||||
if (where):
|
||||
where=where+ " AND "
|
||||
where_in_vrste = ','.join(['%s'] * len(vrste))
|
||||
where=where + " tipologija IN (%s) " % (where_in_vrste)
|
||||
params=params+vrste
|
||||
|
||||
if (kljucnebesede):
|
||||
if (where):
|
||||
where=where+ " AND "
|
||||
where_in_kb = ','.join(['%s'] * len(kljucnebesede))
|
||||
where=where + " kljucnabeseda IN (%s) " % (where_in_kb)
|
||||
params=params+kljucnebesede
|
||||
|
||||
if(where):
|
||||
sql=sql+" where " + where + ";"
|
||||
|
||||
print(sql)
|
||||
print(params)
|
||||
|
||||
|
||||
|
||||
cur.execute(sql,params)
|
||||
|
||||
ret = list(cur)
|
||||
except mariadb.Error as e:
|
||||
print(f"Error connecting to MariaDB Platform: {e}")
|
||||
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, udk):
|
||||
ret = []
|
||||
|
||||
try:
|
||||
print(database_info)
|
||||
conn = mariadb.connect(**database_info)
|
||||
cur = conn.cursor()
|
||||
|
||||
|
||||
|
||||
sql = "select distinct document_id from metadata"
|
||||
where=""
|
||||
params=[]
|
||||
if (udk):
|
||||
where_in_udk = ','.join(['%s'] * len(udk))
|
||||
where=" udk IN (%s) " % (where_in_udk)
|
||||
params=udk
|
||||
|
||||
if (leta):
|
||||
|
||||
where_in_leta = ','.join(['%s'] * len(leta))
|
||||
if (where):
|
||||
where=where+ " AND "
|
||||
where=where + " leto IN (%s) " % (where_in_leta)
|
||||
params=params+leta
|
||||
|
||||
if (vrste):
|
||||
if (where):
|
||||
where=where+ " AND "
|
||||
where_in_vrste = ','.join(['%s'] * len(vrste))
|
||||
where=where + " tipologija IN (%s) " % (where_in_vrste)
|
||||
params=params+vrste
|
||||
|
||||
if (kljucnebesede):
|
||||
if (where):
|
||||
where=where+ " AND "
|
||||
where_in_kb = ','.join(['%s'] * len(kljucnebesede))
|
||||
where=where + " kljucnabeseda IN (%s) " % (where_in_kb)
|
||||
params=params+kljucnebesede
|
||||
|
||||
if(where):
|
||||
sql=sql+" where " + where
|
||||
|
||||
print(sql)
|
||||
print(params)
|
||||
|
||||
sqltk=f"""Select ngram,upos,avg(tfidf) as tfidf, sum(tf) as tf from (
|
||||
SELECT tf.ngram, tf.upos,(0.5+0.5*(tf.tf/d.maxtf))*log(152000/df.df)*(-1*log(1-((dff.df)/(1+df.df)))) as tfidf, tf.tf as tf
|
||||
FROM ngrams_upos_tf tf, documents d,
|
||||
(
|
||||
Select ngram, upos, count(*) as df from ngrams_upos_tf TF
|
||||
where document_id in
|
||||
({sql})
|
||||
group by TF.ngram, TF.upos
|
||||
) dff, ngrams_upos_df df
|
||||
where
|
||||
tf.document_id=d.document_id and
|
||||
df.ngram=tf.ngram AND df.upos=tf.upos and
|
||||
dff.ngram=tf.ngram AND dff.upos=tf.upos
|
||||
) X
|
||||
group by ngram,upos
|
||||
order by tfidf desc
|
||||
limit 1000;"""
|
||||
#
|
||||
print (sqltk)
|
||||
|
||||
cur.execute(sqltk,params)
|
||||
terms=cur.fetchall()
|
||||
#ret = list(cur)
|
||||
can = {'forms':[
|
||||
ngram
|
||||
for ngram in terms
|
||||
]
|
||||
}
|
||||
res = requests.post(ATEapi_endpoint, json=can)
|
||||
data = res.json().canonical_forms
|
||||
|
||||
ret = {'terminoloski_kandidati': [
|
||||
{
|
||||
'POSoznake': x.upos,
|
||||
'kandidat': x.ngram, # more to bit lemma al terms?
|
||||
'kanonicnaoblika': d,
|
||||
'ranking': x.tfidf,
|
||||
'podporneutezi': [
|
||||
0.0, # ????????
|
||||
0.0 # ??????
|
||||
],
|
||||
'pogostostpojavljanja': [tf, 0] # ???????
|
||||
}
|
||||
for d,x in zip(data,cur)
|
||||
]}
|
||||
|
||||
except mariadb.Error as e:
|
||||
print(f"Error connecting to MariaDB Platform: {e}")
|
||||
|
||||
|
||||
return ret
|
||||
|
||||
# class BaseModel(Model):
|
||||
# class Meta:
|
||||
# database = db
|
||||
#
|
||||
#
|
||||
# class os2022_ngrams(BaseModel):
|
||||
# file_id = IntegerField()
|
||||
# sent_id = FloatField()
|
||||
# ngram_len = IntegerField()
|
||||
# frequency_g_t = IntegerField()
|
||||
# gram_text = TextField()
|
||||
# lemma_text = TextField()
|
||||
# xpos_text = TextField()
|
||||
# upos_text = TextField()
|
||||
#
|
||||
# db.connect()
|
||||
|
||||
|
||||
#class Ngrams_Manager:
|
||||
#@staticmethod
|
||||
#def get_by_file_id(file_id):
|
||||
#try:
|
||||
# cur.execute(f'SELECT * from os2022_ngrams WHERE file_id = {file_id}')
|
||||
# cur.execute(f'SELECT COUNT(*) FROM os2022_ngrams')
|
||||
# return list(cur)
|
||||
# return 1
|
||||
#except Exception as e:
|
||||
#print(e, 'EXC')
|
||||
#return 0
|
||||
@@ -0,0 +1,117 @@
|
||||
import os.path
|
||||
|
||||
import pytesseract
|
||||
import requests
|
||||
import docx
|
||||
import xml.etree.ElementTree as ET
|
||||
from PyPDF2 import PdfReader
|
||||
from swagger_server.utils import cl_utils
|
||||
import cv2
|
||||
import numpy as np
|
||||
import magic
|
||||
import re
|
||||
#to še mora v env
|
||||
tika_server = "http://tika2:9999/tika"
|
||||
|
||||
# endpoint below to be used only for development purposes (don't need to run docker)
|
||||
# tika_server = "http://rsdo.lhrs.feri.um.si:9998/tika"
|
||||
|
||||
|
||||
def extract_text_prepResp(file, content_type=""):
|
||||
content_type = file.content_type
|
||||
if content_type is None:
|
||||
content_type = magic.from_file(file.stream.name, mime=True)
|
||||
|
||||
content = ""
|
||||
if tika_responding():
|
||||
try:
|
||||
response = requests.put(tika_server, data=file, headers={"Accept": "text/plain; charset=UTF-8"})
|
||||
content = response.text
|
||||
#preveri če je pretvorba uspešna
|
||||
|
||||
# original string
|
||||
res = re.findall(r'\w+', content)
|
||||
|
||||
#preveri, če imamo vsaj 10 besed in če je povprečna dolžina >3 in < 12
|
||||
#če to drži, idi v ocr
|
||||
reslen=map(lambda n:len(n),res)
|
||||
print(f"Število besed je {len(res)}")
|
||||
|
||||
if len(res)>0 :
|
||||
avglen=sum(reslen)/len(res)
|
||||
else:
|
||||
avglen=0
|
||||
|
||||
print(f"Povprečna dolžina besede je {avglen}")
|
||||
|
||||
if(len(res)<10 or avglen<4 or avglen>11):
|
||||
print("Besedilo je sumljivo, gremo v OCR in damo file na začetek!")
|
||||
file.seek(0)
|
||||
response = requests.put(tika_server, data=file, headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng",
|
||||
"Accept": "text/plain; charset=UTF-8"})
|
||||
content = response.text
|
||||
|
||||
#odstranim še vse prelome vrstic, ker imamo s tem probleme
|
||||
content=' '.join(content.splitlines())
|
||||
except:
|
||||
content = "ERROR - something went wrong when reading file with tika"
|
||||
|
||||
#if content == "":
|
||||
# if "openxmlformats-officedocument.wordprocessingml.document" in content_type:
|
||||
# content = '\n'.join([p.text for p in docx.Document(file).paragraphs])
|
||||
# elif "application/pdf" in content_type:
|
||||
# reader = PdfReader(file)
|
||||
# content = '\n'.join([p.extract_text() for p in reader.pages])
|
||||
# content = content
|
||||
# elif "text/xml" in content_type:
|
||||
# root = ET.parse(file).getroot()
|
||||
# plainText = root.findall('PlainText')
|
||||
# if len(plainText) == 0:
|
||||
# return "Didn't find anything in PlainText", 400
|
||||
# content = '\n'.join([pt.text for pt in plainText])
|
||||
# # elif "text/plain" in file.content_type:
|
||||
# else:
|
||||
# try:
|
||||
# content = file.read().decode('utf-8')
|
||||
# except:
|
||||
# content = "ERROR - something went wrong when reading file with not-tika method!"
|
||||
|
||||
return content, 200
|
||||
|
||||
|
||||
def ocr_text_prepResp(file):
|
||||
content = ""
|
||||
if tika_responding():
|
||||
try:
|
||||
response = requests.put(tika_server, data=file,
|
||||
headers={"X-Tika-PDFOcrStrategy": "ocr_only", "X-Tika-OCRLanguage": "slv+eng",
|
||||
"Accept": "text/plain; charset=UTF-8"})
|
||||
content = response.text
|
||||
except:
|
||||
content = "ERROR - something went wrong when reading file with tika (OCR)"
|
||||
|
||||
if content == "":
|
||||
try:
|
||||
win_p = "C:/Program Files/Tesseract-OCR/tesseract.exe"
|
||||
if os.path.exists(win_p):
|
||||
pytesseract.pytesseract.tesseract_cmd = win_p
|
||||
|
||||
# convert string data to numpy array
|
||||
file_bytes = np.fromstring(file.read(), np.uint8)
|
||||
# convert numpy array to image
|
||||
img = cv2.imdecode(file_bytes, cv2.IMREAD_COLOR)
|
||||
|
||||
conf = '-l eng+slv'
|
||||
content = pytesseract.image_to_string(img, config=conf)
|
||||
except:
|
||||
content = "ERROR - something went wrong when reading file with not-tika method! (OCR)"
|
||||
|
||||
return content, 200
|
||||
|
||||
|
||||
def tika_responding():
|
||||
try:
|
||||
ret = requests.get(tika_server)
|
||||
return ret.status_code == 200
|
||||
except:
|
||||
return False
|
||||
Reference in New Issue
Block a user