term_extractor/swagger_server/utils/db_utils.py

import mariadb
import os
import sys
import requests
import json
import time


database_info = {
    'database': os.getenv("MDB_DATABASE", "oss"),
    'host': os.getenv("MDB_HOST", "localhost"),
    'port': int(os.getenv("MDB_PORT", 3306)) ,
    'user': os.getenv("MDB_USER", "root"),
    'password': os.getenv("MDB_PASSWORD", "root"),
}

canonapi_endpoint = "http://canonizer:5000/rest_api/canonize"

cur = None
# Connect to MariaDB Platform

def get_files_by_udc(udc):
    ret = []

    try:
        print(database_info)
        conn = mariadb.connect(**database_info)
        cur = conn.cursor()
        where_in = ','.join(['%s'] * len(udc))
        print(where_in)
        sql = "select distinct xml_id from metadata_udc where udk IN (%s)" % (where_in)
        print(sql)
        cur.execute(sql,udc)
        #cur.execute(f'SELECT COUNT(*) FROM os2022_ngrams')
        ret = list(cur)
    except mariadb.Error as e:
        print(f"Error connecting to MariaDB Platform: {e}")
    

    return ret

def vrni_oss_dokumente(leta, vrste, kljucnebesede, udk):
    ret = []
  
    try:
        print(database_info)
        conn = mariadb.connect(**database_info)
        cur = conn.cursor()
        
        
        
        sql = "select distinct document_id from metadata"
        where=""
        params=[]
        if (udk):
            where_in_udk = ','.join(['%s'] * len(udk))
            where=" udk IN (%s) " % (where_in_udk)
            params=udk
        
        if (leta):

            where_in_leta = ','.join(['%s'] * len(leta))
            if (where):
                where=where+ " AND "
            where=where + " leto IN (%s) " % (where_in_leta)
            params=params+leta
        
        if (vrste):
            if (where):
                where=where+ " AND "
            where_in_vrste = ','.join(['%s'] * len(vrste))
            where=where + " tipologija IN (%s) " % (where_in_vrste)
            params=params+vrste

        if (kljucnebesede):
            if (where):
                where=where+ " AND "
            where_in_kb = ','.join(['%s'] * len(kljucnebesede))
            where=where + " kljucnabeseda IN (%s) " % (where_in_kb)
            params=params+kljucnebesede

        if(where):
            sql=sql+" where " + where + ";"
        
        print(sql)
        print(params)


        
        cur.execute(sql,params)
        
        ret = list(cur)
    except mariadb.Error as e:
        print(f"Error connecting to MariaDB Platform: {e}")
    

    return ret


def vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, prepovedane_besede, udk,definicije=False):
    ret = []
  
    try:
        print(database_info)
        conn = mariadb.connect(**database_info)
        cur = conn.cursor(dictionary=True)
        
        
        
        sql = "select distinct document_id from metadata"
        where=""
        params=[]
        if (udk):
            where_in_udk = ','.join(['%s'] * len(udk))
            where=" udk IN (%s) " % (where_in_udk)
            params=udk
        
        if (leta):

            where_in_leta = ','.join(['%s'] * len(leta))
            if (where):
                where=where+ " AND "
            where=where + " leto IN (%s) " % (where_in_leta)
            params=params+leta
        
        if (vrste):
            if (where):
                where=where+ " AND "
            where_in_vrste = ','.join(['%s'] * len(vrste))
            where=where + " tipologija IN (%s) " % (where_in_vrste)
            params=params+vrste

        if (kljucnebesede):
            if (where):
                where=where+ " AND "
            where_in_kb = ','.join(['%s'] * len(kljucnebesede))
            where=where + " kljucnabeseda IN (%s) " % (where_in_kb)
            params=params+kljucnebesede

        if(where):
            sql=sql+" where " + where
        
        print(sql)
        print(params)

        sqltk=f"""Select ngram,upos,convert(avg(tfidf),FLOAT) as tfidf, convert(sum(tf),INT) as tf from (
                    SELECT tf.ngram, tf.upos,(0.5+0.5*(tf.tf/d.maxtf))*log(152000/df.df)*(-1*log(1-((dff.df)/(1+df.df)))) as tfidf, tf.tf as tf
                    FROM ngrams_upos_tf tf, documents d,
                        (
                        Select ngram, upos, count(*) as df from ngrams_upos_tf TF
                        where document_id in
                    ({sql})
                    group by TF.ngram, TF.upos
                    ) dff, ngrams_upos_df df
                    where
                    tf.document_id=d.document_id and
                    df.ngram=tf.ngram AND df.upos=tf.upos and
                    dff.ngram=tf.ngram AND dff.upos=tf.upos
                    ) X
                    group by ngram,upos
                    order by tfidf desc
                    limit 1000;"""
                    #
        #sqltk=f"""select ngram,upos,convert(1.0,float) as tfidf,%s as tf from ngrams_upos_tf limit 10;"""

        print (sqltk)
        #še prepovedane besede ven
        start_time = time.time()
        cur.execute(sqltk,params)
        terms=cur.fetchall()
        print("Čas poizbedbe je %.2f sekund" % (time.time() - start_time))
        print (terms);
        #ret = list(cur)
        can = {'forms':[
            ngram["ngram"]
            for ngram in terms
            ]
        }
        print (can);
        res = requests.post(canonapi_endpoint, json=can)
        
        data = res.json()
        print (data);
        print (data.get("canonical_forms"));
        print (terms);
        print(zip(data.get("canonical_forms"),terms))




        ret = {'terminoloski_kandidati': [
            {
                'POSoznake': x.get("upos"),
                'kandidat': x.get("ngram"),  # more to bit lemma al terms?
                'definicija': None,
                'kanonicnaoblika': d,
                'ranking': x.get('tfidf'),
                'podporneutezi': [
                    0.0,  # ????????
                    0.0  # ??????
                ],
                'pogostostpojavljanja': [x.get('tf'), 0]  # ???????
            }
            for (d,x) in zip(data.get("canonical_forms"),terms) 
        ]}
    
        #if definicije
        #idi z variablo sql po id-je dokumentov, preberi conlluje iz diska
        #naredi en vlki conllu
        #pokliči metodo


    except mariadb.Error as e:
        print(f"Error connecting to MariaDB Platform: {e}")
    

    return ret

# class BaseModel(Model):
#     class Meta:
#         database = db
#
#
# class os2022_ngrams(BaseModel):
#     file_id = IntegerField()
#     sent_id = FloatField()
#     ngram_len = IntegerField()
#     frequency_g_t = IntegerField()
#     gram_text = TextField()
#     lemma_text = TextField()
#     xpos_text = TextField()
#     upos_text = TextField()
#
# db.connect()


#class Ngrams_Manager:
    #@staticmethod
    #def get_by_file_id(file_id):
        #try:
            # cur.execute(f'SELECT * from os2022_ngrams WHERE file_id = {file_id}')
            # cur.execute(f'SELECT COUNT(*) FROM os2022_ngrams')
            # return list(cur)
         #   return 1
        #except Exception as e:
            #print(e, 'EXC')
        #return 0