|
|
#!/usr/bin/env python
|
|
|
# Copyright 2019 CJVT
|
|
|
#
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
# you may not use this file except in compliance with the License.
|
|
|
# You may obtain a copy of the License at
|
|
|
#
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
#
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
# See the License for the specific language governing permissions and
|
|
|
# limitations under the License.
|
|
|
|
|
|
import argparse
|
|
|
import configparser
|
|
|
import copy
|
|
|
import csv
|
|
|
import hashlib
|
|
|
import os
|
|
|
import pickle
|
|
|
import re
|
|
|
import string
|
|
|
import time
|
|
|
import timeit
|
|
|
from multiprocessing import Pool
|
|
|
import gzip
|
|
|
|
|
|
def save_zipped_pickle(obj, filename, protocol=-1):
|
|
|
with gzip.open(filename, 'wb') as f:
|
|
|
pickle.dump(obj, f, protocol)
|
|
|
|
|
|
def load_zipped_pickle(filename):
|
|
|
with gzip.open(filename, 'rb') as f:
|
|
|
loaded_object = pickle.load(f)
|
|
|
return loaded_object
|
|
|
|
|
|
import pyconll
|
|
|
|
|
|
from Tree import Tree, create_output_string_form, create_output_string_deprel, create_output_string_lemma, create_output_string_upos, create_output_string_xpos, create_output_string_feats
|
|
|
|
|
|
# for separate searches of feats
|
|
|
# feats_detailed_list = [
|
|
|
# # lexical features
|
|
|
# 'PronType', 'NumType', 'Poss', 'Reflex', 'Foreign', 'Abbr',
|
|
|
#
|
|
|
# # Inflectional features (nominal)
|
|
|
# 'Gender', 'Animacy', 'NounClass', 'Number', 'Case', 'Definite', 'Degree',
|
|
|
#
|
|
|
# # Inflectional features (verbal)
|
|
|
# 'VerbForm', 'Mood', 'Tense', 'Aspect', 'Voice', 'Evident', 'Polarity', 'Person', 'Polite', 'Clusivity',
|
|
|
#
|
|
|
# # Other
|
|
|
# 'Variant', 'Number[psor]', 'Gender[psor]', 'NumForm'
|
|
|
# ]
|
|
|
|
|
|
# feats_detailed_list = []
|
|
|
|
|
|
# feats_detailed_dict = {key: {} for key in feats_detailed_list}
|
|
|
from generic import get_collocabilities
|
|
|
|
|
|
|
|
|
def decode_query(orig_query, dependency_type, feats_detailed_list):
|
|
|
new_query = False
|
|
|
|
|
|
# if command in bracelets remove them and treat command as new query
|
|
|
if orig_query[0] == '(' and orig_query[-1] == ')':
|
|
|
new_query = True
|
|
|
orig_query = orig_query[1:-1]
|
|
|
|
|
|
# if orig_query is '_' return {}
|
|
|
if dependency_type != '':
|
|
|
decoded_query = {'deprel': dependency_type}
|
|
|
else:
|
|
|
decoded_query = {}
|
|
|
|
|
|
if orig_query == '_':
|
|
|
return decoded_query
|
|
|
# if no spaces in query then this is query node and do this otherwise further split query
|
|
|
elif len(orig_query.split(' ')) == 1:
|
|
|
orig_query_split_parts = orig_query.split(' ')[0].split('&')
|
|
|
for orig_query_split_part in orig_query_split_parts:
|
|
|
orig_query_split = orig_query_split_part.split('=', 1)
|
|
|
if len(orig_query_split) > 1:
|
|
|
if orig_query_split[0] == 'L':
|
|
|
decoded_query['lemma'] = orig_query_split[1]
|
|
|
# return decoded_query
|
|
|
elif orig_query_split[0] == 'upos':
|
|
|
decoded_query['upos'] = orig_query_split[1]
|
|
|
# return decoded_query
|
|
|
elif orig_query_split[0] == 'xpos':
|
|
|
decoded_query['xpos'] = orig_query_split[1]
|
|
|
# return decoded_query
|
|
|
elif orig_query_split[0] == 'form':
|
|
|
decoded_query['form'] = orig_query_split[1]
|
|
|
# return decoded_query
|
|
|
elif orig_query_split[0] == 'feats':
|
|
|
decoded_query['feats'] = orig_query_split[1]
|
|
|
# return decoded_query
|
|
|
elif orig_query_split[0] in feats_detailed_list:
|
|
|
decoded_query['feats_detailed'] = {}
|
|
|
decoded_query['feats_detailed'][orig_query_split[0]] = orig_query_split[1]
|
|
|
return decoded_query
|
|
|
elif not new_query:
|
|
|
raise Exception('Not supported yet!')
|
|
|
else:
|
|
|
print('???')
|
|
|
elif not new_query:
|
|
|
decoded_query['form'] = orig_query_split_part
|
|
|
# return decoded_query
|
|
|
return decoded_query
|
|
|
|
|
|
# split over spaces if not inside braces
|
|
|
# PATTERN = re.compile(r'''((?:[^ ()]|\([^.]*\))+)''')
|
|
|
# all_orders = PATTERN.split(orig_query)
|
|
|
# PATTERN = re.compile(r"(?:[^ ()]|\([^.]*\))+")
|
|
|
# all_orders = re.findall(r"(?:[^ ()]|\([^ ]*\))+", orig_query)
|
|
|
all_orders = re.split(r"\s+(?=[^()]*(?:\(|$))", orig_query)
|
|
|
|
|
|
|
|
|
# all_orders = orig_query.split()
|
|
|
node_actions = all_orders[::2]
|
|
|
priority_actions = all_orders[1::2]
|
|
|
priority_actions_beginnings = [a[0] for a in priority_actions]
|
|
|
|
|
|
# find root index
|
|
|
try:
|
|
|
root_index = priority_actions_beginnings.index('>')
|
|
|
except ValueError:
|
|
|
root_index = len(priority_actions)
|
|
|
|
|
|
children = []
|
|
|
root = None
|
|
|
for i, node_action in enumerate(node_actions):
|
|
|
if i < root_index:
|
|
|
children.append(decode_query(node_action, priority_actions[i][1:], feats_detailed_list))
|
|
|
elif i > root_index:
|
|
|
children.append(decode_query(node_action, priority_actions[i - 1][1:], feats_detailed_list))
|
|
|
else:
|
|
|
root = decode_query(node_action, dependency_type, feats_detailed_list)
|
|
|
if children:
|
|
|
root["children"] = children
|
|
|
return root
|
|
|
|
|
|
|
|
|
def create_trees(config):
|
|
|
internal_saves = config.get('settings', 'internal_saves')
|
|
|
input_path = config.get('settings', 'input')
|
|
|
# internal_saves = filters['internal_saves']
|
|
|
# input_path = filters['input']
|
|
|
hash_object = hashlib.sha1(input_path.encode('utf-8'))
|
|
|
hex_dig = hash_object.hexdigest()
|
|
|
trees_read_outputfile = os.path.join(internal_saves, hex_dig)
|
|
|
|
|
|
if not os.path.exists(trees_read_outputfile):
|
|
|
|
|
|
train = pyconll.load_from_file(input_path)
|
|
|
|
|
|
form_dict, lemma_dict, upos_dict, xpos_dict, deprel_dict, feats_dict = {}, {}, {}, {}, {}, {}
|
|
|
|
|
|
all_trees = []
|
|
|
corpus_size = 0
|
|
|
feats_detailed_dict = {}
|
|
|
|
|
|
for sentence in train:
|
|
|
root = None
|
|
|
root_id = None
|
|
|
token_nodes = []
|
|
|
for token in sentence:
|
|
|
# token_feats = ''
|
|
|
# for k, v in token.feats.items():
|
|
|
# token_feats += k + next(iter(v)) + '|'
|
|
|
# token_feats = token_feats[:-1]
|
|
|
if not token.id.isdigit():
|
|
|
continue
|
|
|
|
|
|
# TODO check if 5th place is always there for feats
|
|
|
feats = token._fields[5]
|
|
|
node = Tree(int(token.id), token.form, token.lemma, token.upos, token.xpos, token.deprel, feats, token.feats, form_dict,
|
|
|
lemma_dict, upos_dict, xpos_dict, deprel_dict, feats_dict, feats_detailed_dict, token.head)
|
|
|
token_nodes.append(node)
|
|
|
if token.deprel == 'root':
|
|
|
root = node
|
|
|
|
|
|
corpus_size += 1
|
|
|
|
|
|
for token_id, token in enumerate(token_nodes):
|
|
|
if int(token.parent) == 0:
|
|
|
token.set_parent(None)
|
|
|
else:
|
|
|
parent_id = int(token.parent) - 1
|
|
|
# if token_id < parent_id:
|
|
|
# token_nodes[parent_id].add_l_child(token)
|
|
|
# elif token_id > parent_id:
|
|
|
# token_nodes[parent_id].add_r_child(token)
|
|
|
# else:
|
|
|
# raise Exception('Root element should not be here!')
|
|
|
if token_nodes[parent_id].children_split == -1 and token_id > parent_id:
|
|
|
token_nodes[parent_id].children_split = len(token_nodes[parent_id].children)
|
|
|
token_nodes[parent_id].add_child(token)
|
|
|
token.set_parent(token_nodes[parent_id])
|
|
|
|
|
|
for token in token_nodes:
|
|
|
if token.children_split == -1:
|
|
|
token.children_split = len(token.children)
|
|
|
|
|
|
if root == None:
|
|
|
raise Exception('No root element in sentence!')
|
|
|
all_trees.append(root)
|
|
|
|
|
|
save_zipped_pickle((all_trees, form_dict, lemma_dict, upos_dict, xpos_dict, deprel_dict, corpus_size, feats_detailed_dict), trees_read_outputfile, protocol=2)
|
|
|
# with open(trees_read_outputfile, 'wb') as output:
|
|
|
#
|
|
|
# pickle.dump((all_trees, form_dict, lemma_dict, upos_dict, xpos_dict, deprel_dict, corpus_size, feats_detailed_dict), output)
|
|
|
else:
|
|
|
print('Reading trees:')
|
|
|
print('Completed')
|
|
|
all_trees, form_dict, lemma_dict, upos_dict, xpos_dict, deprel_dict, corpus_size, feats_detailed_dict = load_zipped_pickle(trees_read_outputfile)
|
|
|
# with open(trees_read_outputfile, 'rb') as pkl_file:
|
|
|
# (all_trees, form_dict, lemma_dict, upos_dict, xpos_dict, deprel_dict, corpus_size, feats_detailed_dict) = pickle.load(pkl_file)
|
|
|
|
|
|
return all_trees, form_dict, lemma_dict, upos_dict, xpos_dict, deprel_dict, corpus_size, feats_detailed_dict
|
|
|
|
|
|
|
|
|
# def order_independent_queries(query_tree):
|
|
|
# all_children = query_tree['l_children'] + query_tree['r_children']
|
|
|
# if all_children > 0:
|
|
|
#
|
|
|
# else:
|
|
|
# return query_tree
|
|
|
# pass
|
|
|
|
|
|
def printable_answers(query):
|
|
|
# all_orders = re.findall(r"(?:[^ ()]|\([^ ]*\))+", query)
|
|
|
all_orders = re.split(r"\s+(?=[^()]*(?:\(|$))", query)
|
|
|
|
|
|
# all_orders = orig_query.split()
|
|
|
node_actions = all_orders[::2]
|
|
|
# priority_actions = all_orders[1::2]
|
|
|
|
|
|
if len(node_actions) > 1:
|
|
|
res = []
|
|
|
# for node_action in node_actions[:-1]:
|
|
|
# res.extend(printable_answers(node_action[1:-1]))
|
|
|
# res.extend([node_actions[-1]])
|
|
|
for node_action in node_actions:
|
|
|
# if command in bracelets remove them and treat command as new query
|
|
|
# TODO FIX BRACELETS IN A BETTER WAY
|
|
|
if not node_action:
|
|
|
res.extend(['('])
|
|
|
elif node_action[0] == '(' and node_action[-1] == ')':
|
|
|
res.extend(printable_answers(node_action[1:-1]))
|
|
|
else:
|
|
|
res.extend([node_action])
|
|
|
return res
|
|
|
else:
|
|
|
return [query]
|
|
|
|
|
|
|
|
|
def tree_calculations(input_data):
|
|
|
tree, query_tree, create_output_string_funct, filters = input_data
|
|
|
_, subtrees = tree.get_subtrees(query_tree, [], create_output_string_funct, filters)
|
|
|
return subtrees
|
|
|
|
|
|
def get_unigrams(input_data):
|
|
|
tree, query_tree, create_output_string_funct, filters = input_data
|
|
|
unigrams = tree.get_unigrams(create_output_string_funct, filters)
|
|
|
return unigrams
|
|
|
|
|
|
|
|
|
def tree_calculations_chunks(input_data):
|
|
|
trees, query_tree, create_output_string_funct, filters = input_data
|
|
|
|
|
|
result_dict = {}
|
|
|
for tree in trees:
|
|
|
_, subtrees = tree.get_subtrees(query_tree, [], create_output_string_funct, filters)
|
|
|
|
|
|
for query_results in subtrees:
|
|
|
for r in query_results:
|
|
|
if r in result_dict:
|
|
|
result_dict[r] += 1
|
|
|
else:
|
|
|
result_dict[r] = 1
|
|
|
return result_dict
|
|
|
|
|
|
|
|
|
def chunkify(a, n):
|
|
|
k, m = divmod(len(a), n)
|
|
|
return (a[i * k + min(i, m):(i + 1) * k + min(i + 1, m)] for i in range(n))
|
|
|
|
|
|
|
|
|
def add_node(tree):
|
|
|
if 'children' in tree:
|
|
|
tree['children'].append({})
|
|
|
else:
|
|
|
tree['children'] = [{}]
|
|
|
|
|
|
|
|
|
# walk over all nodes in tree and add a node to each possible node
|
|
|
def tree_grow(orig_tree):
|
|
|
new_trees = []
|
|
|
new_tree = copy.deepcopy(orig_tree)
|
|
|
add_node(new_tree)
|
|
|
new_trees.append(new_tree)
|
|
|
if 'children' in orig_tree:
|
|
|
children = []
|
|
|
for child_tree in orig_tree['children']:
|
|
|
children.append(tree_grow(child_tree))
|
|
|
for i, child in enumerate(children):
|
|
|
for child_res in child:
|
|
|
new_tree = copy.deepcopy(orig_tree)
|
|
|
new_tree['children'][i] = child_res
|
|
|
new_trees.append(new_tree)
|
|
|
|
|
|
return new_trees
|
|
|
|
|
|
|
|
|
def compare_trees(tree1, tree2):
|
|
|
if tree1 == {} and tree2 == {}:
|
|
|
return True
|
|
|
|
|
|
if 'children' not in tree1 or 'children' not in tree2 or len(tree1['children']) != len(tree2['children']):
|
|
|
return False
|
|
|
|
|
|
children2_connections = []
|
|
|
|
|
|
for child1_i, child1 in enumerate(tree1['children']):
|
|
|
child_duplicated = False
|
|
|
for child2_i, child2 in enumerate(tree2['children']):
|
|
|
if child2_i in children2_connections:
|
|
|
pass
|
|
|
if compare_trees(child1, child2):
|
|
|
children2_connections.append(child2_i)
|
|
|
child_duplicated = True
|
|
|
break
|
|
|
if not child_duplicated:
|
|
|
return False
|
|
|
|
|
|
return True
|
|
|
|
|
|
def create_ngrams_query_trees(n, trees):
|
|
|
for i in range(n - 1):
|
|
|
new_trees = []
|
|
|
for tree in trees:
|
|
|
# append new_tree only if it is not already inside
|
|
|
for new_tree in tree_grow(tree):
|
|
|
duplicate = False
|
|
|
for confirmed_new_tree in new_trees:
|
|
|
if compare_trees(new_tree, confirmed_new_tree):
|
|
|
duplicate = True
|
|
|
break
|
|
|
if not duplicate:
|
|
|
new_trees.append(new_tree)
|
|
|
|
|
|
trees = new_trees
|
|
|
# delete_duplicates(trees)
|
|
|
# print('here')
|
|
|
# tree_grow(tree)
|
|
|
# tree_grow(tree)
|
|
|
# tree['children'] = [{}]
|
|
|
return trees
|
|
|
|
|
|
def count_trees(cpu_cores, all_trees, query_tree, create_output_string_functs, filters, unigrams_dict, result_dict):
|
|
|
with Pool(cpu_cores) as p:
|
|
|
# 1.25 s (16 cores)
|
|
|
# chunked_trees = list(chunkify(all_trees, cpu_cores))
|
|
|
# if cpu_cores > 1:
|
|
|
# part_results = p.map(tree_calculations_chunks,
|
|
|
# [(tree, query_tree, create_output_string_funct, filters) for tree in chunked_trees])
|
|
|
#
|
|
|
# for part_result in part_results:
|
|
|
# for r_k, r_v in part_result.items():
|
|
|
# if r_k in result_dict:
|
|
|
# result_dict[r_k] += r_v
|
|
|
# else:
|
|
|
# result_dict[r_k] = r_v
|
|
|
# 1.02 s (16 cores)
|
|
|
if cpu_cores > 1:
|
|
|
# input_data = (tree, query_tree, create_output_string_functs, filters)
|
|
|
all_unigrams = p.map(get_unigrams, [(tree, query_tree, create_output_string_functs, filters) for tree in all_trees])
|
|
|
for unigrams in all_unigrams:
|
|
|
for unigram in unigrams:
|
|
|
if unigram in unigrams_dict:
|
|
|
unigrams_dict[unigram] += 1
|
|
|
else:
|
|
|
unigrams_dict[unigram] = 1
|
|
|
|
|
|
all_subtrees = p.map(tree_calculations, [(tree, query_tree, create_output_string_functs, filters) for tree in all_trees])
|
|
|
|
|
|
# for subtrees in all_subtrees:
|
|
|
for tree_i, subtrees in enumerate(all_subtrees):
|
|
|
|
|
|
for query_results in subtrees:
|
|
|
for r in query_results:
|
|
|
# if r.key == '(ne <advmod more >xcomp (se <expl izogniti) >punct .)':
|
|
|
# print('HERE')
|
|
|
# print(tree_i)
|
|
|
if filters['node_order']:
|
|
|
key = r.get_key() + r.order
|
|
|
else:
|
|
|
key = r.get_key()
|
|
|
# if r == '(" < , < je < velik) < tem':
|
|
|
# print(tree_i)
|
|
|
# if r in result_dict:
|
|
|
# result_dict[r] += 1
|
|
|
# else:
|
|
|
# result_dict[r] = 1
|
|
|
if key in result_dict:
|
|
|
result_dict[key]['number'] += 1
|
|
|
else:
|
|
|
result_dict[key] = {'object': r, 'number': 1}
|
|
|
|
|
|
# 3.65 s (1 core)
|
|
|
else:
|
|
|
# for tree_i, tree in enumerate(all_trees[-5:]):
|
|
|
for tree_i, tree in enumerate(all_trees):
|
|
|
# for tree_i, tree in enumerate(all_trees[852:]):
|
|
|
# for tree_i, tree in enumerate(all_trees[1689:]):
|
|
|
# for tree_i, tree in enumerate(all_trees[1:3]):
|
|
|
input_data = (tree, query_tree, create_output_string_functs, filters)
|
|
|
if filters['association_measures']:
|
|
|
unigrams = get_unigrams(input_data)
|
|
|
for unigram in unigrams:
|
|
|
if unigram in unigrams_dict:
|
|
|
unigrams_dict[unigram] += 1
|
|
|
else:
|
|
|
unigrams_dict[unigram] = 1
|
|
|
# for tree_i, tree in enumerate(all_trees[1:]):
|
|
|
# text = Če pa ostane odrasel otrok doma, se starši le težko sprijaznijo s tem, da je "velik", otrok pa ima ves čas občutek, da se njegovi starši po nepotrebnem vtikajo v njegovo življenje.
|
|
|
# for tree_i, tree in enumerate(all_trees[5170:]):
|
|
|
# for tree in all_trees:
|
|
|
subtrees = tree_calculations(input_data)
|
|
|
for query_results in subtrees:
|
|
|
for r in query_results:
|
|
|
if filters['node_order']:
|
|
|
key = r.get_key() + r.order
|
|
|
else:
|
|
|
key = r.get_key()
|
|
|
# if r == '(" < , < je < velik) < tem':
|
|
|
# print(tree_i)
|
|
|
if key in result_dict:
|
|
|
result_dict[key]['number'] += 1
|
|
|
else:
|
|
|
result_dict[key] = {'object': r, 'number': 1}
|
|
|
|
|
|
def read_filters(config, feats_detailed_list):
|
|
|
tree_size_range = config.get('settings', 'tree_size', fallback='0').split('-')
|
|
|
tree_size_range = [int(r) for r in tree_size_range]
|
|
|
|
|
|
if tree_size_range[0] > 1:
|
|
|
if len(tree_size_range) == 1:
|
|
|
query_tree = create_ngrams_query_trees(tree_size_range[0], [{}])
|
|
|
elif len(tree_size_range) == 2:
|
|
|
query_tree = []
|
|
|
for i in range(tree_size_range[0], tree_size_range[1] + 1):
|
|
|
query_tree.extend(create_ngrams_query_trees(i, [{}]))
|
|
|
else:
|
|
|
query_tree = [decode_query('(' + config.get('settings', 'query') + ')', '', feats_detailed_list)]
|
|
|
# order_independent_queries(query_tree)
|
|
|
|
|
|
# set filters
|
|
|
node_types = config.get('settings', 'node_type').split('+')
|
|
|
create_output_string_functs = []
|
|
|
for node_type in node_types:
|
|
|
assert node_type in ['deprel', 'lemma', 'upos', 'xpos', 'form', 'feats'], '"node_type" is not set up correctly'
|
|
|
cpu_cores = config.getint('settings', 'cpu_cores')
|
|
|
if node_type == 'deprel':
|
|
|
create_output_string_funct = create_output_string_deprel
|
|
|
elif node_type == 'lemma':
|
|
|
create_output_string_funct = create_output_string_lemma
|
|
|
elif node_type == 'upos':
|
|
|
create_output_string_funct = create_output_string_upos
|
|
|
elif node_type == 'xpos':
|
|
|
create_output_string_funct = create_output_string_xpos
|
|
|
elif node_type == 'feats':
|
|
|
create_output_string_funct = create_output_string_feats
|
|
|
else:
|
|
|
create_output_string_funct = create_output_string_form
|
|
|
create_output_string_functs.append(create_output_string_funct)
|
|
|
|
|
|
result_dict = {}
|
|
|
unigrams_dict = {}
|
|
|
filters = {}
|
|
|
filters['internal_saves'] = config.get('settings', 'internal_saves')
|
|
|
filters['input'] = config.get('settings', 'input')
|
|
|
filters['node_order'] = config.get('settings', 'node_order') == 'fixed'
|
|
|
# filters['caching'] = config.getboolean('settings', 'caching')
|
|
|
filters['dependency_type'] = config.get('settings', 'dependency_type') == 'labeled'
|
|
|
if config.has_option('settings', 'label_whitelist'):
|
|
|
filters['label_whitelist'] = config.get('settings', 'label_whitelist').split('|')
|
|
|
else:
|
|
|
filters['label_whitelist'] = []
|
|
|
|
|
|
if config.has_option('settings', 'root_whitelist'):
|
|
|
# test
|
|
|
filters['root_whitelist'] = []
|
|
|
|
|
|
for option in config.get('settings', 'root_whitelist').split('|'):
|
|
|
attribute_dict = {}
|
|
|
for attribute in option.split('&'):
|
|
|
value = attribute.split('=')
|
|
|
# assert value[0] in ['deprel', 'lemma', 'upos', 'xpos', 'form',
|
|
|
# 'feats'], '"root_whitelist" is not set up correctly'
|
|
|
attribute_dict[value[0]] = value[1]
|
|
|
filters['root_whitelist'].append(attribute_dict)
|
|
|
# filters['root_whitelist'] = [{'upos': 'NOUN', 'Case': 'Nom'}, {'upos': 'ADJ', 'Degree': 'Sup'}]
|
|
|
else:
|
|
|
filters['root_whitelist'] = []
|
|
|
|
|
|
filters['complete_tree_type'] = config.get('settings', 'tree_type') == 'complete'
|
|
|
filters['association_measures'] = config.getboolean('settings', 'association_measures')
|
|
|
filters['nodes_number'] = config.getboolean('settings', 'nodes_number')
|
|
|
filters['frequency_threshold'] = config.getfloat('settings', 'frequency_threshold', fallback=0)
|
|
|
filters['lines_threshold'] = config.getint('settings', 'lines_threshold', fallback=0)
|
|
|
filters['print_root'] = config.getboolean('settings', 'print_root')
|
|
|
|
|
|
return filters, query_tree, create_output_string_functs, cpu_cores, unigrams_dict, result_dict, tree_size_range, node_types
|
|
|
|
|
|
def main():
|
|
|
parser = argparse.ArgumentParser()
|
|
|
|
|
|
## Required parameters
|
|
|
parser.add_argument("--config_file",
|
|
|
default=None,
|
|
|
type=str,
|
|
|
required=True,
|
|
|
help="The input config file.")
|
|
|
args = parser.parse_args()
|
|
|
|
|
|
config = configparser.ConfigParser()
|
|
|
config.read(args.config_file)
|
|
|
|
|
|
|
|
|
|
|
|
# a = args.config_file
|
|
|
# config.read('config.ini')
|
|
|
# create queries
|
|
|
|
|
|
# 261 - 9 grams
|
|
|
# 647 - 10 grams
|
|
|
# 1622 - 11 grams
|
|
|
# 4126 - 12 grams
|
|
|
# 10598 - 13 grams
|
|
|
(all_trees, form_dict, lemma_dict, upos_dict, xpos_dict, deprel_dict, corpus_size,
|
|
|
feats_detailed_list) = create_trees(config)
|
|
|
|
|
|
filters, query_tree, create_output_string_functs, cpu_cores, unigrams_dict, result_dict, tree_size_range, node_types = read_filters(config, feats_detailed_list)
|
|
|
|
|
|
# if config.getint('settings', 'tree_size') == 2:
|
|
|
# tree_size = 2
|
|
|
# query_tree = [{"children": [{}]}]
|
|
|
# elif config.getint('settings', 'tree_size') == 3:
|
|
|
# tree_size = 3
|
|
|
# query_tree = [{"children": [{}, {}]}, {"children": [{"children": [{}]}]}]
|
|
|
# elif config.getint('settings', 'tree_size') == 4:
|
|
|
# tree_size = 4
|
|
|
# query_tree = [{"children": [{}, {}, {}]}, {"children": [{"children": [{}, {}]}]}, {"children": [{"children": [{}]}, {}]}, {"children": [{"children": [{"children": [{}]}]}]}]
|
|
|
# elif config.getint('settings', 'tree_size') == 5:
|
|
|
# tree_size = 5
|
|
|
# query_tree = [{"children": [{}, {}, {}, {}]}, {"children": [{"children": [{}]}, {}, {}]}, {"children": [{"children": [{}, {}]}, {}]}, {"children": [{"children": [{}]}, {"children": [{}]}]},
|
|
|
# {"children": [{"children": [{"children": [{}]}]}, {}]}, {"children": [{"children": [{"children": [{}]}, {}]}]}, {"children": [{"children": [{"children": [{}, {}]}]}]},
|
|
|
# {"children": [{"children": [{"children": [{"children": [{}]}]}]}]}, {'children': [{'children': [{}, {}, {}]}]}]
|
|
|
|
|
|
|
|
|
|
|
|
# for tree in all_trees[2:]:
|
|
|
# for tree in all_trees[1205:]:
|
|
|
start_exe_time = time.time()
|
|
|
count_trees(cpu_cores, all_trees, query_tree, create_output_string_functs, filters, unigrams_dict, result_dict)
|
|
|
|
|
|
print("Execution time:")
|
|
|
print("--- %s seconds ---" % (time.time() - start_exe_time))
|
|
|
# test 1 layer queries
|
|
|
# # tree.r_children = []
|
|
|
# # tree.children[1].children = []
|
|
|
# # query = [{'children': [{}]}, {'children': [{}]}]
|
|
|
# # query = [{"children": [{}, {}]}, {"children": [{}]}, {"children": [{}, {}, {}]}]
|
|
|
# query = [{"children": [{'form': 'je'}, {}]}, {"children": [{'form': 'je'}]}, {"children": [{'form': 'je'}, {}, {}]}]
|
|
|
# # query = [{'q1':'', "children": [{'a1':''}, {'a2':''}]}, {'q2':'', "children": [{'b1':''}]}, {'q3':'', "children": [{'c1':''}, {'c2':''}, {'c3':''}]}]
|
|
|
# _, _, subtrees = tree.get_subtrees(query, [], create_output_string_funct)
|
|
|
# # _, subtrees = tree.get_subtrees([{'q1':'', "children": [{'a1':''}, {'a2':''}], "children": []}, {'q2':'', "children": [{'b1':''}], "children": []}, {'q3':'', "children": [{'c1':''}, {'c2':''}, {'c3':''}], "children": []}], [])
|
|
|
# print('HERE!')
|
|
|
|
|
|
# test 2 layer queries
|
|
|
# tree.r_children = [Tree('je', '', '', '', '', form_dict, lemma_dict, upos_dict, xpos_dict, deprel_dict, None)]
|
|
|
# tree.l_children[1].l_children = []
|
|
|
# new_tree = Tree('bil', '', '', '', '', form_dict, lemma_dict, upos_dict, xpos_dict, deprel_dict, None)
|
|
|
# new_tree.l_children = [tree]
|
|
|
# _, subtrees = new_tree.get_subtrees(
|
|
|
# [{"l_children":[{"l_children": [{'a1': ''}, {'a2': ''}, {'a3': ''}, {'a4': ''}]}]}], [])
|
|
|
# # _, subtrees = new_tree.get_subtrees(
|
|
|
# # [{"l_children":[{"l_children": [{'a1': ''}, {'a2': ''}, {'a3': ''}, {'a4': ''}], "r_children": []}], "r_children": []}], [])
|
|
|
sorted_list = sorted(result_dict.items(), key=lambda x: x[1]['number'], reverse=True)
|
|
|
|
|
|
with open(config.get('settings', 'output'), "w", newline="") as f:
|
|
|
# header - use every second space as a split
|
|
|
writer = csv.writer(f, delimiter='\t')
|
|
|
if tree_size_range[-1]:
|
|
|
len_words = tree_size_range[-1]
|
|
|
else:
|
|
|
len_words = int(len(config.get('settings', 'query').split(" "))/2 + 1)
|
|
|
header = ["Structure"] + ["Node " + string.ascii_uppercase[i] + "-" + node_type for i in range(len_words) for node_type in node_types] + ['Absolute frequency']
|
|
|
header += ['Relative frequency']
|
|
|
if filters['node_order']:
|
|
|
header += ['Order']
|
|
|
header += ['Free structure']
|
|
|
if filters['nodes_number']:
|
|
|
header += ['Number of nodes']
|
|
|
if filters['print_root']:
|
|
|
header += ['Root node']
|
|
|
if filters['association_measures']:
|
|
|
header += ['MI', 'MI3', 'Dice', 'logDice', 't-score', 'simple-LL']
|
|
|
# header = [" ".join(words[i:i + span]) for i in range(0, len(words), span)] + ['Absolute frequency']
|
|
|
writer.writerow(header)
|
|
|
|
|
|
if filters['lines_threshold']:
|
|
|
sorted_list = sorted_list[:filters['lines_threshold']]
|
|
|
|
|
|
# body
|
|
|
for k, v in sorted_list:
|
|
|
v['object'].get_array()
|
|
|
relative_frequency = v['number'] * 1000000.0 / corpus_size
|
|
|
if filters['frequency_threshold'] and filters['frequency_threshold'] > v['number']:
|
|
|
break
|
|
|
words_only = [word_att for word in v['object'].array for word_att in word] + ['' for i in range((tree_size_range[-1] - len(v['object'].array)) * len(v['object'].array[0]))]
|
|
|
# words_only = printable_answers(k)
|
|
|
row = [v['object'].get_key()[1:-1]] + words_only + [str(v['number'])]
|
|
|
row += ['%.4f' % relative_frequency]
|
|
|
if filters['node_order']:
|
|
|
row += [v['object'].order]
|
|
|
row += [v['object'].get_key_sorted()[1:-1]]
|
|
|
if filters['nodes_number']:
|
|
|
row += ['%d' % len(v['object'].array)]
|
|
|
if filters['print_root']:
|
|
|
row += [v['object'].node.name]
|
|
|
if filters['association_measures']:
|
|
|
row += get_collocabilities(v, unigrams_dict, corpus_size)
|
|
|
writer.writerow(row)
|
|
|
|
|
|
return "Done"
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
start_time = time.time()
|
|
|
main()
|
|
|
print("Total:")
|
|
|
print("--- %s seconds ---" % (time.time() - start_time))
|
