Preperation for merging multiple conllus, also added ATE files

2022-10-18 10:07:01 +02:00
parent be84cdae1b
commit 5833dfafc8
25 changed files with 95006 additions and 21 deletions
@@ -0,0 +1,90 @@
+import os
+import glob
+import json
+import numpy as np
+import pandas as pd
+import warnings
+warnings.filterwarnings("ignore", category=FutureWarning)
+
+def read_conll(f, lowercase=True, max_example=None):
+    examples = []
+    # with open(in_file) as f:
+    word, lemma, pos, msd, label = [], [], [], [], []
+    for line in f.readlines():
+        sp = line.decode('utf-8').strip().split('\t')
+        # print(sp)
+        if len(sp) == 10:
+            if '-' not in sp[0]:
+                word.append(sp[1].lower() if lowercase else sp[1])
+                lemma.append(sp[2].lower() if lowercase else sp[2])
+                pos.append(sp[3])
+                msd.append(sp[4])
+                label.append('n')
+        elif len(word) > 0:
+            examples.append({'word': word, 'lemma': lemma, 'pos': pos, 'msd':msd,   'labels': label})
+            word, lemma, pos, msd, label = [], [], [], [], []
+            if (max_example is not None) and (len(examples) == max_example):
+                break
+    if len(word) > 0:
+        examples.append({'word': word, 'lemma': lemma, 'pos': pos,  'msd':msd, 'labels': label})
+    return pd.DataFrame(examples)
+
+
+def extract_terms_full(token_predictions,token_probabilities, val_texts, lemma_texts, pos_texts, msd_texts):
+    extracted_terms = list()
+    extracted_probs = list()
+    lemma_terms = list()
+    pos_terms = list()
+    msd_terms = list()
+    # go over all predictions
+    for i in range(len(token_predictions)):
+        pred = token_predictions[i]
+        prob = token_probabilities[i]
+        txt  = val_texts[i]
+        lemma  = lemma_texts[i]
+        pos  = pos_texts[i]
+        msd  = msd_texts[i]
+        for j in range(len(pred)):
+            if pred[j] == "B-T":
+                term = txt[j]
+                prob_term = str(round(prob[j][1],4))
+                lemma_term = lemma[j]
+                pos_term = pos[j]
+                msd_term = msd[j]
+                for k in range(j+1,len(pred)):
+                    if pred[k]=="T": 
+                        term+=" "+txt[k]
+                        prob_term+=" "+str(round(prob[k][2],4))
+                        lemma_term+=" "+ lemma[k]
+                        pos_term+=" "+ pos[k]
+                        msd_term+=" "+ msd[k]
+                    else: break
+                extracted_terms.append(term)
+                extracted_probs.append(prob_term)
+                lemma_terms.append(lemma_term)
+                pos_terms.append(pos_term)
+                msd_terms.append(msd_term)
+    return extracted_terms, extracted_probs, lemma_terms, pos_terms, msd_terms
+
+def remap(tokenizer, texts, preds, probs):
+    text = texts.split(' ')
+    len_token = []
+    res_pred = []
+    final_pred = []
+    res_prob = []
+    final_prob = []
+    for i in range(len(text)):
+        temp = len(tokenizer(text[i], add_special_tokens=False)['input_ids'])
+        len_token.append(temp)
+    for val in len_token:
+        res_pred.append(val)
+        res_pred.extend([0]*(val-1))    
+        res_prob.append(val)
+        res_prob.extend([0]*(val-1))  
+    for p1, p2 in zip(res_pred, preds[1:len(preds)-1]):
+        if p1 != 0:
+            final_pred.append(p2)
+    for p1, p2 in zip(res_prob, probs[1:len(probs)-1]):
+        if p1 != 0:
+            final_prob.append(p2)
+    return text, final_pred, final_prob