91 lines
3.2 KiB
Python
91 lines
3.2 KiB
Python
import os
|
|
import glob
|
|
import json
|
|
import numpy as np
|
|
import pandas as pd
|
|
import warnings
|
|
warnings.filterwarnings("ignore", category=FutureWarning)
|
|
|
|
def read_conll(f, lowercase=True, max_example=None):
|
|
examples = []
|
|
# with open(in_file) as f:
|
|
word, lemma, pos, msd, label = [], [], [], [], []
|
|
for line in f.readlines():
|
|
sp = line.decode('utf-8').strip().split('\t')
|
|
# print(sp)
|
|
if len(sp) == 10:
|
|
if '-' not in sp[0]:
|
|
word.append(sp[1].lower() if lowercase else sp[1])
|
|
lemma.append(sp[2].lower() if lowercase else sp[2])
|
|
pos.append(sp[3])
|
|
msd.append(sp[4])
|
|
label.append('n')
|
|
elif len(word) > 0:
|
|
examples.append({'word': word, 'lemma': lemma, 'pos': pos, 'msd':msd, 'labels': label})
|
|
word, lemma, pos, msd, label = [], [], [], [], []
|
|
if (max_example is not None) and (len(examples) == max_example):
|
|
break
|
|
if len(word) > 0:
|
|
examples.append({'word': word, 'lemma': lemma, 'pos': pos, 'msd':msd, 'labels': label})
|
|
return pd.DataFrame(examples)
|
|
|
|
|
|
def extract_terms_full(token_predictions,token_probabilities, val_texts, lemma_texts, pos_texts, msd_texts):
|
|
extracted_terms = list()
|
|
extracted_probs = list()
|
|
lemma_terms = list()
|
|
pos_terms = list()
|
|
msd_terms = list()
|
|
# go over all predictions
|
|
for i in range(len(token_predictions)):
|
|
pred = token_predictions[i]
|
|
prob = token_probabilities[i]
|
|
txt = val_texts[i]
|
|
lemma = lemma_texts[i]
|
|
pos = pos_texts[i]
|
|
msd = msd_texts[i]
|
|
for j in range(len(pred)):
|
|
if pred[j] == "B-T":
|
|
term = txt[j]
|
|
prob_term = str(round(prob[j][1],4))
|
|
lemma_term = lemma[j]
|
|
pos_term = pos[j]
|
|
msd_term = msd[j]
|
|
for k in range(j+1,len(pred)):
|
|
if pred[k]=="T":
|
|
term+=" "+txt[k]
|
|
prob_term+=" "+str(round(prob[k][2],4))
|
|
lemma_term+=" "+ lemma[k]
|
|
pos_term+=" "+ pos[k]
|
|
msd_term+=" "+ msd[k]
|
|
else: break
|
|
extracted_terms.append(term)
|
|
extracted_probs.append(prob_term)
|
|
lemma_terms.append(lemma_term)
|
|
pos_terms.append(pos_term)
|
|
msd_terms.append(msd_term)
|
|
return extracted_terms, extracted_probs, lemma_terms, pos_terms, msd_terms
|
|
|
|
def remap(tokenizer, texts, preds, probs):
|
|
text = texts.split(' ')
|
|
len_token = []
|
|
res_pred = []
|
|
final_pred = []
|
|
res_prob = []
|
|
final_prob = []
|
|
for i in range(len(text)):
|
|
temp = len(tokenizer(text[i], add_special_tokens=False)['input_ids'])
|
|
len_token.append(temp)
|
|
for val in len_token:
|
|
res_pred.append(val)
|
|
res_pred.extend([0]*(val-1))
|
|
res_prob.append(val)
|
|
res_prob.extend([0]*(val-1))
|
|
for p1, p2 in zip(res_pred, preds[1:len(preds)-1]):
|
|
if p1 != 0:
|
|
final_pred.append(p2)
|
|
for p1, p2 in zip(res_prob, probs[1:len(probs)-1]):
|
|
if p1 != 0:
|
|
final_prob.append(p2)
|
|
return text, final_pred, final_prob
|