import mariadb import os import sys import requests import json database_info = { 'database': os.getenv("MDB_DATABASE", "oss"), 'host': os.getenv("MDB_HOST", "localhost"), 'port': int(os.getenv("PORT", 3306)) , 'user': os.getenv("MDB_USER", "root"), 'password': os.getenv("MDB_PASSWORD", "root"), } canonapi_endpoint = "http://canonizer:5000/rest_api/canonize" cur = None # Connect to MariaDB Platform def get_files_by_udc(udc): ret = [] try: print(database_info) conn = mariadb.connect(**database_info) cur = conn.cursor() where_in = ','.join(['%s'] * len(udc)) print(where_in) sql = "select distinct xml_id from metadata_udc where udk IN (%s)" % (where_in) print(sql) cur.execute(sql,udc) #cur.execute(f'SELECT COUNT(*) FROM os2022_ngrams') ret = list(cur) except mariadb.Error as e: print(f"Error connecting to MariaDB Platform: {e}") return ret def vrni_oss_dokumente(leta, vrste, kljucnebesede, udk): ret = [] try: print(database_info) conn = mariadb.connect(**database_info) cur = conn.cursor() sql = "select distinct document_id from metadata" where="" params=[] if (udk): where_in_udk = ','.join(['%s'] * len(udk)) where=" udk IN (%s) " % (where_in_udk) params=udk if (leta): where_in_leta = ','.join(['%s'] * len(leta)) if (where): where=where+ " AND " where=where + " leto IN (%s) " % (where_in_leta) params=params+leta if (vrste): if (where): where=where+ " AND " where_in_vrste = ','.join(['%s'] * len(vrste)) where=where + " tipologija IN (%s) " % (where_in_vrste) params=params+vrste if (kljucnebesede): if (where): where=where+ " AND " where_in_kb = ','.join(['%s'] * len(kljucnebesede)) where=where + " kljucnabeseda IN (%s) " % (where_in_kb) params=params+kljucnebesede if(where): sql=sql+" where " + where + ";" print(sql) print(params) cur.execute(sql,params) ret = list(cur) except mariadb.Error as e: print(f"Error connecting to MariaDB Platform: {e}") return ret def vrni_oss_terminoloske_kandidate(leta, vrste, kljucnebesede, udk): ret = [] try: print(database_info) conn = mariadb.connect(**database_info) cur = conn.cursor() sql = "select distinct document_id from metadata" where="" params=[] if (udk): where_in_udk = ','.join(['%s'] * len(udk)) where=" udk IN (%s) " % (where_in_udk) params=udk if (leta): where_in_leta = ','.join(['%s'] * len(leta)) if (where): where=where+ " AND " where=where + " leto IN (%s) " % (where_in_leta) params=params+leta if (vrste): if (where): where=where+ " AND " where_in_vrste = ','.join(['%s'] * len(vrste)) where=where + " tipologija IN (%s) " % (where_in_vrste) params=params+vrste if (kljucnebesede): if (where): where=where+ " AND " where_in_kb = ','.join(['%s'] * len(kljucnebesede)) where=where + " kljucnabeseda IN (%s) " % (where_in_kb) params=params+kljucnebesede if(where): sql=sql+" where " + where print(sql) print(params) sqltk=f"""Select ngram,upos,avg(tfidf) as tfidf, sum(tf) as tf from ( SELECT tf.ngram, tf.upos,(0.5+0.5*(tf.tf/d.maxtf))*log(152000/df.df)*(-1*log(1-((dff.df)/(1+df.df)))) as tfidf, tf.tf as tf FROM ngrams_upos_tf tf, documents d, ( Select ngram, upos, count(*) as df from ngrams_upos_tf TF where document_id in ({sql}) group by TF.ngram, TF.upos ) dff, ngrams_upos_df df where tf.document_id=d.document_id and df.ngram=tf.ngram AND df.upos=tf.upos and dff.ngram=tf.ngram AND dff.upos=tf.upos ) X group by ngram,upos order by tfidf desc limit 1000;""" # print (sqltk) cur.execute(sqltk,params) terms=cur.fetchall() #ret = list(cur) can = {'forms':[ ngram for ngram in terms ] } res = requests.post(ATEapi_endpoint, json=can) data = res.json().canonical_forms ret = {'terminoloski_kandidati': [ { 'POSoznake': x.upos, 'kandidat': x.ngram, # more to bit lemma al terms? 'kanonicnaoblika': d, 'ranking': x.tfidf, 'podporneutezi': [ 0.0, # ???????? 0.0 # ?????? ], 'pogostostpojavljanja': [tf, 0] # ??????? } for d,x in zip(data,cur) ]} except mariadb.Error as e: print(f"Error connecting to MariaDB Platform: {e}") return ret # class BaseModel(Model): # class Meta: # database = db # # # class os2022_ngrams(BaseModel): # file_id = IntegerField() # sent_id = FloatField() # ngram_len = IntegerField() # frequency_g_t = IntegerField() # gram_text = TextField() # lemma_text = TextField() # xpos_text = TextField() # upos_text = TextField() # # db.connect() #class Ngrams_Manager: #@staticmethod #def get_by_file_id(file_id): #try: # cur.execute(f'SELECT * from os2022_ngrams WHERE file_id = {file_id}') # cur.execute(f'SELECT COUNT(*) FROM os2022_ngrams') # return list(cur) # return 1 #except Exception as e: #print(e, 'EXC') #return 0