HUGE refactor, creating lots of modules, no code changes though!

This commit is contained in:
Ozbolt Menegatti 2019-06-15 18:55:35 +02:00
parent 43c6c9151b
commit 90dbbca5d5
18 changed files with 1708 additions and 1544 deletions

86
src/codes_tagset.py Normal file
View File

@ -0,0 +1,86 @@
CODES = {
"Noun": "N",
"Verb": "V",
"Adjective": "A",
"Adverb": "R",
"Pronoun": "P",
"Numeral": "M",
"Preposition": "S",
"Conjunction": "C",
"Particle": "Q",
"Interjection": "I",
"Abbreviation": "Y",
"Residual": "X",
'common': 'c',
'proper': 'p',
'masculine': 'm',
'feminine': 'f',
'neuter': 'n',
"singular": "s",
"dual": "d",
"plural": "p",
"nominative": "n",
"genitive": "g",
"dative": "d",
"accusative": "a",
"locative": "l",
"instrumental": "i",
"no": "n",
"yes": "y",
"main": "m",
"auxiliary": "a",
"perfective": "e",
"progressive": "p",
"biaspectual": "b",
"infinitive": "n",
"supine": "u",
"participle": "p",
"present": "r",
"future": "f",
"conditional": "c",
"imperative": "m",
"first": "1",
"second": "2",
"third": "3",
"general": "g",
"possessive": "s",
"positive": "p",
"comparative": "c",
"superlative": "s",
"personal": "p",
"demonstrative": "d",
"relative": "r",
"reflexive": "x",
"interrogative": "q",
"indefinite": "i",
"negative": "z",
"bound": "b",
"digit": "d",
"roman": "r",
"letter": "l",
"cardinal": "c",
"ordinal": "o",
"pronominal": "p",
"special": "s",
"coordinating": "c",
"subordinating": "s",
"foreign": "f",
"typo": "t",
"program": "p",
}
TAGSET = {
"N": ['type', 'gender', 'number', 'case', 'animate'],
"V": ['type', 'aspect', 'vform', 'person', 'number', 'gender', 'negative'],
"A": ['type', 'degree', 'gender', 'number', 'case', 'definiteness'],
"R": ['type', 'degree'],
"P": ['type', 'person', 'gender', 'number', 'case', 'owner_number', 'owned_gender', 'clitic'],
"M": ['form', 'type', 'gender', 'number', 'case', 'definiteness'],
"S": ['case'],
"C": ['type'],
"Q": [],
"I": [],
"Y": [],
"X": ['type']
}

166
src/component.py Normal file
View File

@ -0,0 +1,166 @@
from enum import Enum
import logging
from restriction import Restriction
from order import Order
from representation_assigner import RepresentationAssigner
class ComponentStatus(Enum):
Optional = 0
Required = 1
Forbidden = 2
class ComponentType(Enum):
Other = 0
Core = 2
Core2w = 3
class Component:
def __init__(self, info):
idx = info['cid']
name = info['name'] if 'name' in info else None
typ = ComponentType.Core if info['type'] == "core" else ComponentType.Other
if 'status' not in info:
status = ComponentStatus.Required
elif info['status'] == 'forbidden':
status = ComponentStatus.Forbidden
elif info['status'] == 'obligatory':
status = ComponentStatus.Required
elif info['status'] == 'optional':
status = ComponentStatus.Optional
else:
raise NotImplementedError("strange status: {}".format(info['status']))
self.status = status
self.name = name
self.idx = idx
self.restrictions = []
self.next_element = []
self.representation = []
self.selection = {}
self.type = typ
self.iter_ctr = 0
def add_next(self, next_component, link_label, order):
self.next_element.append((next_component, link_label, Order.new(order)))
def set_restriction(self, restrictions_tag):
if restrictions_tag is None:
self.restrictions = [Restriction(None)]
elif restrictions_tag.tag == "restriction":
self.restrictions = [Restriction(restrictions_tag)]
elif restrictions_tag.tag == "restriction_or":
self.restrictions = [Restriction(el) for el in restrictions_tag]
else:
raise RuntimeError("Unreachable")
def set_representation(self, representation):
for rep in representation:
crend = RepresentationAssigner()
for feature in rep:
crend.add_feature(feature.attrib)
self.representation.append(crend)
def find_next(self, deps, comps, restrs, reprs):
to_ret = []
for d in deps:
if d[0] == self.idx:
_, idx, dep_label, order = d
next_component = Component(comps[idx])
next_component.set_restriction(restrs[idx])
next_component.set_representation(reprs[idx])
to_ret.append(next_component)
self.add_next(next_component, dep_label, order)
others = next_component.find_next(deps, comps, restrs, reprs)
to_ret.extend(others)
return to_ret
def name_str(self):
return "_" if self.name is None else self.name
def match(self, word):
m1 = self._match_self(word)
if m1 is None:
return None
mn = self._match_next(word)
if mn is None:
return None
to_ret = [m1]
for cmatch in mn:
# if good match but nothing to add, just continue
if len(cmatch) == 0:
continue
# if more than one match found for particular component
elif len(cmatch) > 1:
# if more than one match in multiple components, NOPE!
if len(to_ret) > 1:
logging.warning("Strange multiple match: {}".format(
str([w.id for w in cmatch[0].values()])))
for tr in to_ret:
tr.update(cmatch[0])
continue
# yeah, so we have found more than one match, =>
# more than one element in to_ret
to_ret = [{**dict(to_ret[0]), **m} for m in cmatch]
else:
for tr in to_ret:
tr.update(cmatch[0])
return to_ret
def _match_self(self, word):
# matching
for restr in self.restrictions:
if restr.match(word): # match either
return {self.idx: word}
def _match_next(self, word):
# matches for every component in links from this component
to_ret = []
# need to get all links that match
for next, link, order in self.next_element:
next_links = word.get_links(link)
to_ret.append([])
# good flag
good = next.status != ComponentStatus.Required
for next_word in next_links:
if not order.match(word, next_word):
continue
match = next.match(next_word)
if match is not None:
# special treatement for forbidden
if next.status == ComponentStatus.Forbidden:
good = False
break
else:
assert type(match) is list
to_ret[-1].extend(match)
good = True
# if none matched, nothing found!
if not good:
return None
return to_ret

190
src/formatter.py Normal file
View File

@ -0,0 +1,190 @@
from math import log2
import re
from component import ComponentType
class Formatter:
def __init__(self, colocation_ids, word_renderer):
self.colocation_ids = colocation_ids
self.word_renderer = word_renderer
self.additional_init()
def header_repeat(self):
raise NotImplementedError("Header repeat formatter not implemented")
def header_right(self):
raise NotImplementedError("Header right formatter not implemented")
def content_repeat(self, words, representations, idx, sidx):
raise NotImplementedError("Content repeat formatter not implemented")
def content_right(self, freq):
raise NotImplementedError("Content right formatter not implemented")
def group(self):
raise NotImplementedError("Group for formatter not implemented")
def additional_init(self):
pass
def length(self):
return len(self.header_repeat())
def set_structure(self, structure):
pass
def new_match(self, match):
pass
class OutNoStatFormatter(Formatter):
def additional_init(self):
self.representation = ""
def header_repeat(self):
return ["Lemma", "Representative_form", "RF_scenario"]
def header_right(self):
return ["Joint_representative_form", "Frequency"]
def content_repeat(self, words, representations, idx, _sidx):
word = words[idx]
if idx not in representations:
return [word.lemma, "", ""]
rep = representations[idx]
if rep is None:
self.representation += " " + word.lemma
return [word.lemma, word.lemma, "lemma_fallback"]
else:
self.representation += " " + rep
return [word.lemma, rep, "ok"]
def content_right(self, freq):
rep = re.sub(' +', ' ', self.representation.strip())
result = [rep, str(freq)]
self.representation = ""
return result
def group(self):
return True
class AllFormatter(Formatter):
def header_repeat(self):
return ["Token_ID", "Word_form", "Lemma", "Msd"]
def header_right(self):
return []
def content_repeat(self, words, _representations, idx, _sidx):
word = words[idx]
return [word.id, word.text, word.lemma, word.msd]
def content_right(self, _freq):
return []
def group(self):
return False
class StatsFormatter(Formatter):
def additional_init(self):
self.stats = None
self.jppb = None
self.corew = None
@staticmethod
def stat_str(num):
return "{:.5f}".format(num) if isinstance(num, float) else str(num)
def set_structure(self, structure):
jppb = []
corew = []
for component in structure.components:
if component.type == ComponentType.Core2w:
jppb.append(component.idx)
if component.type != ComponentType.Other:
corew.append(component.idx)
assert(len(jppb) == 2)
self.jppb = tuple(jppb)
self.corew = tuple(corew)
def new_match(self, match):
self.stats = {"freq": {}}
for cid in self.corew:
if cid not in match.matches[0]:
freq = 0
else:
word = match.matches[0][cid]
freq = self.word_renderer.num_words[(word.lemma, word.msd[0])]
self.stats["freq"][cid] = freq
fx = self.stats["freq"][self.jppb[0]]
fy = self.stats["freq"][self.jppb[1]]
freq = len(match)
N = self.word_renderer.num_all_words()
self.stats['d12'] = freq / fx - (fy - freq) / (N - fx)
self.stats['d21'] = freq / fy - (fx - freq) / (N - fy)
self.stats['df'] = match.distinct_forms()
self.stats['freq_all'] = freq
def header_repeat(self):
return ["Distribution"]
def header_right(self):
return ["Delta_p12", "Delta_p21", "LogDice_core", "LogDice_all", "Distinct_forms"]
def content_repeat(self, words, representations, idx, sidx):
# not a core word
if idx not in self.corew:
return [""] * self.length()
word = words[idx]
key = (sidx, idx, word.lemma)
distribution = self.colocation_ids.dispersions[key]
return [self.stat_str(distribution)]
def content_right(self, freq):
fx = self.stats["freq"][self.jppb[0]]
fy = self.stats["freq"][self.jppb[1]]
freq = self.stats['freq_all']
logdice_core = 14 + log2(2 * freq / (fx + fy))
fi = [self.stats["freq"][idx] for idx in self.corew]
fi = [f for f in fi if f > 0]
logdice_all = 14 + log2(len(fi) * freq / sum(fi))
return [self.stat_str(x) for x in (
self.stats["d12"], self.stats["d21"], logdice_core, logdice_all, self.stats['df']
)]
def group(self):
return True
class OutFormatter(Formatter):
def additional_init(self):
self.f1 = OutNoStatFormatter(self.colocation_ids, self.word_renderer)
self.f2 = StatsFormatter(self.colocation_ids, self.word_renderer)
def header_repeat(self):
return self.f1.header_repeat() + self.f2.header_repeat()
def header_right(self):
return self.f1.header_right() + self.f2.header_right()
def content_repeat(self, words, representations, idx, sidx):
cr1 = self.f1.content_repeat(words, representations, idx, sidx)
cr2 = self.f2.content_repeat(words, representations, idx, sidx)
return cr1 + cr2
def content_right(self, freq):
return self.f1.content_right(freq) + self.f2.content_right(freq)
def group(self):
return self.f1.group() and self.f2.group()
def set_structure(self, structure):
self.f2.set_structure(structure)
def new_match(self, match):
self.f2.new_match(match)

26
src/lemma_features.py Normal file
View File

@ -0,0 +1,26 @@
from restriction import MorphologyRegex
def get_lemma_features(et):
lf = et.find('lemma_features')
if lf is None:
return {}
result = {}
for pos in lf.iter('POS'):
rgx_list = MorphologyRegex(pos).rgx
rgx_str = ""
for position in rgx_list:
if position == ".":
rgx_str += " "
elif len(position) == 1:
rgx_str += position
elif len(position) == 3 and position[0] == "[" and position[2] == "]":
rgx_str += position[1]
else:
raise RuntimeError("Strange rgx for lemma_feature...")
assert rgx_str[0].isupper()
result[rgx_str[0]] = rgx_str.strip().replace(' ', '-')
return result

20
src/match.py Normal file
View File

@ -0,0 +1,20 @@
class StructureMatch:
def __init__(self, match_id, structure):
self.match_id = match_id
self.structure = structure
self.matches = []
self.representations = {}
def distinct_forms(self):
dm = set()
keys = list(self.matches[0].keys())
for words in self.matches:
dm.add(" ".join(words[k].text for k in keys))
return len(dm)
def append(self, match):
self.matches.append(match)
def __len__(self):
return len(self.matches)

46
src/match_store.py Normal file
View File

@ -0,0 +1,46 @@
from collections import defaultdict
from match import StructureMatch
from representation_assigner import RepresentationAssigner
try:
from tqdm import tqdm
except ImportError:
tqdm = lambda x: x
class MatchStore:
def __init__(self, args):
self.data = {}
self.min_frequency = args.min_freq
self.dispersions = {}
def _add_match(self, key, structure, match):
if key not in self.data:
self.data[key] = StructureMatch(str(len(self.data) + 1), structure)
self.data[key].append(match)
def get(self, key, n):
return self.data[key][n]
def add_matches(self, matches):
for structure, nms in matches.items():
for nm in nms:
self._add_match(nm[1], structure, nm[0])
def get_matches_for(self, structure):
for _cid_tup, sm in self.data.items():
if sm.structure != structure:
continue
yield sm
def set_representations(self, word_renderer):
for _1, sm in tqdm(self.data.items()):
RepresentationAssigner.set_representations(sm, word_renderer)
def determine_colocation_dispersions(self):
dispersions = defaultdict(int)
for (structure_id, *word_tups) in self.data.keys():
for component_id, lemma in word_tups:
dispersions[(structure_id, component_id, lemma)] += 1
self.dispersions = dict(dispersions)

34
src/order.py Normal file
View File

@ -0,0 +1,34 @@
from enum import Enum
class Order(Enum):
FromTo = 0
ToFrom = 1
Any = 2
@staticmethod
def new(order):
if order is None:
return Order.Any
elif order == "to-from":
return Order.ToFrom
elif order == "from-to":
return Order.FromTo
else:
raise NotImplementedError("What kind of ordering is: {}".format(order))
def match(self, from_w, to_w):
if self is Order.Any:
return True
fi = from_w.int_id
ti = to_w.int_id
if self is Order.FromTo:
return fi < ti
elif self is Order.ToFrom:
return ti < fi
else:
raise NotImplementedError("Should not be here: Order match")

167
src/representation.py Normal file
View File

@ -0,0 +1,167 @@
import logging
from collections import Counter
from codes_tagset import TAGSET, CODES
from word import WordMsdOnly
class ComponentRepresentation:
def __init__(self, data, word_renderer):
self.data = data
self.word_renderer = word_renderer
self.words = []
self.rendition_text = None
self.agreement = []
def get_agreement(self):
return []
def add_word(self, word):
self.words.append(word)
def render(self):
if self.rendition_text is None:
self.rendition_text = self._render()
def _render(self):
raise NotImplementedError("Not implemented for class: {}".format(type(self)))
class LemmaCR(ComponentRepresentation):
def _render(self):
return self.words[0].lemma if len(self.words) > 0 else None
class LexisCR(ComponentRepresentation):
def _render(self):
return self.data['lexis']
class WordFormAllCR(ComponentRepresentation):
def _render(self):
if len(self.words) == 0:
return None
else:
forms = [w.text.lower() for w in self.words]
return "/".join(set(forms))
class WordFormAnyCR(ComponentRepresentation):
def _render(self):
text_forms = {}
msd_lemma_txt_triplets = Counter([(w.msd, w.lemma, w.text) for w in self.words])
for (msd, lemma, text), _n in reversed(msd_lemma_txt_triplets.most_common()):
text_forms[(msd, lemma)] = text
words_counter = []
for word in self.words:
words_counter.append((word.msd, word.lemma))
sorted_words = sorted(set(words_counter), key=lambda x: -words_counter.count(x))
for word_msd, word_lemma in sorted_words:
for agr in self.agreement:
if not agr.match(word_msd):
break
else:
for agr in self.agreement:
agr.confirm_match()
if word_lemma is None:
return None
else:
return text_forms[(word_msd, word_lemma)]
class WordFormMsdCR(WordFormAnyCR):
def __init__(self, *args):
super().__init__(*args)
self.lemma = None
self.msd = None
def check_msd(self, word_msd):
if 'msd' not in self.data:
return True
selectors = self.data['msd']
for key, value in selectors.items():
t = word_msd[0]
v = TAGSET[t].index(key.lower())
f1 = word_msd[v + 1]
f2 = CODES[value]
if '-' not in [f1, f2] and f1 != f2:
return False
return True
def add_word(self, word):
if self.lemma is None:
self.lemma = word.lemma
self.msd = word.msd
if self.check_msd(word.msd):
super().add_word(word)
def _render(self):
msd = self.word_renderer.get_lemma_msd(self.lemma, self.msd)
self.words.append(WordMsdOnly(msd))
return super()._render()
class WordFormAgreementCR(WordFormMsdCR):
def __init__(self, data, word_renderer):
super().__init__(data, word_renderer)
self.rendition_candidate = None
def get_agreement(self):
return self.data['other']
def match(self, word_msd):
existing = [(w.msd, w.text) for w in self.words]
lemma_available_words = self.word_renderer.available_words(self.lemma, existing)
for candidate_msd, candidate_text in lemma_available_words:
if self.msd[0] != candidate_msd[0]:
continue
if WordFormAgreementCR.check_agreement(word_msd, candidate_msd, self.data['agreement']):
if self.check_msd(candidate_msd):
self.rendition_candidate = candidate_text
return True
return False
def confirm_match(self):
self.rendition_text = self.rendition_candidate
@staticmethod
def check_agreement(msd1, msd2, agreements):
for agr_case in agreements:
t1 = msd1[0]
# if not in msd, some strange msd was tries, skipping...
if agr_case not in TAGSET[t1]:
logging.warning("Cannot do agreement: {} for msd {} not found!"
.format(agr_case, msd1))
return False
v1 = TAGSET[t1].index(agr_case)
# if none specified: nedolocnik, always agrees
if v1 + 1 >= len(msd1):
continue
# first is uppercase, not in TAGSET
m1 = msd1[v1 + 1]
# REPEAT (not DRY!)
t2 = msd2[0]
if agr_case not in TAGSET[t2]:
logging.warning("Cannot do agreement: {} for msd {} not found!"
.format(agr_case, msd2))
return False
v2 = TAGSET[t2].index(agr_case)
if v2 + 1 >= len(msd2):
continue
m2 = msd2[v2 + 1]
# match!
if '-' not in [m1, m2] and m1 != m2:
return False
return True
def render(self):
pass

View File

@ -0,0 +1,82 @@
from representation import ComponentRepresentation, LemmaCR, LexisCR, WordFormAgreementCR, WordFormAnyCR, WordFormMsdCR, WordFormAllCR
class RepresentationAssigner:
def __init__(self):
self.more = {}
self.representation_factory = ComponentRepresentation
def add_feature(self, feature):
if 'rendition' in feature:
if feature['rendition'] == "lemma":
self.representation_factory = LemmaCR
elif feature['rendition'] == "word_form":
# just by default, changes with selection
self.representation_factory = WordFormAnyCR
elif feature['rendition'] == "lexis":
self.representation_factory = LexisCR
self.more['lexis'] = feature['string']
else:
raise NotImplementedError("Representation rendition: {}".format(feature))
elif 'selection' in feature:
if feature['selection'] == "msd":
# could already be agreement
if self.representation_factory != WordFormAgreementCR:
self.representation_factory = WordFormMsdCR
self.more['msd'] = {k: v for k, v in feature.items() if k != 'selection'}
elif feature['selection'] == "all":
self.representation_factory = WordFormAllCR
elif feature['selection'] == 'agreement':
assert feature['head'][:4] == 'cid_'
assert feature['msd'] is not None
self.representation_factory = WordFormAgreementCR
self.more['agreement'] = feature['msd'].split('+')
self.more['other'] = feature['head'][4:]
else:
raise NotImplementedError("Representation selection: {}".format(feature))
def cr_instance(self, word_renderer):
return self.representation_factory(self.more, word_renderer)
@staticmethod
def set_representations(match, word_renderer):
representations = {}
for c in match.structure.components:
representations[c.idx] = []
for rep in c.representation:
representations[c.idx].append(rep.cr_instance(word_renderer))
for cid, reps in representations.items():
for rep in reps:
for agr in rep.get_agreement():
if len(representations[agr]) != 1:
n = len(representations[agr])
raise NotImplementedError(
"Structure {}: ".format(match.structure.id) +
"component {} has agreement".format(cid) +
" with component {}".format(agr) +
", however there are {} (!= 1) representations".format(n) +
" of component {}!".format(agr))
representations[agr][0].agreement.append(rep)
for words in match.matches:
# first pass, check everything but agreements
for w_id, w in words.items():
component = match.structure.get_component(w_id)
component_representations = representations[component.idx]
for representation in component_representations:
representation.add_word(w)
for cid, reps in representations.items():
for rep in reps:
rep.render()
for cid, reps in representations.items():
reps = [rep.rendition_text for rep in reps]
if reps == []:
pass
elif all(r is None for r in reps):
match.representations[cid] = None
else:
match.representations[cid] = " ".join(("" if r is None else r) for r in reps)

133
src/restriction.py Normal file
View File

@ -0,0 +1,133 @@
import re
from enum import Enum
from codes_tagset import CODES, TAGSET
class RestrictionType(Enum):
Morphology = 0
Lexis = 1
MatchAll = 2
def determine_ppb(rgx):
if rgx[0] in ("A", "N", "R"):
return 0
elif rgx[0] == "V":
if len(rgx) == 1:
return 2
elif 'a' in rgx[1]:
return 3
elif 'm' in rgx[1]:
return 1
else:
return 2
else:
return 4
class MorphologyRegex:
def __init__(self, restriction):
self.min_msd_length = 1
restr_dict = {}
for feature in restriction:
feature_dict = dict(feature.items())
match_type = True
if "filter" in feature_dict:
assert feature_dict['filter'] == "negative"
match_type = False
del feature_dict['filter']
assert len(feature_dict) == 1
key, value = next(iter(feature_dict.items()))
restr_dict[key] = (value, match_type)
assert 'POS' in restr_dict
category = restr_dict['POS'][0].capitalize()
cat_code = CODES[category]
rgx = [cat_code] + ['.'] * 10
del restr_dict['POS']
for attribute, (value, typ) in restr_dict.items():
index = TAGSET[cat_code].index(attribute.lower())
assert index >= 0
if '|' in value:
match = "".join(CODES[val] for val in value.split('|'))
else:
match = CODES[value]
match = "[{}{}]".format("" if typ else "^", match)
rgx[index + 1] = match
if typ:
self.min_msd_length = max(index + 1, self.min_msd_length)
# strip rgx
for i in reversed(range(len(rgx))):
if rgx[i] == '.':
rgx = rgx[:-1]
else:
break
self.re_objects = [re.compile(r) for r in rgx]
self.rgx = rgx
def __call__(self, text):
if len(text) <= self.min_msd_length:
return False
for c, r in zip(text, self.re_objects):
if not r.match(c):
return False
return True
class LexisRegex:
def __init__(self, restriction):
restr_dict = {}
for feature in restriction:
restr_dict.update(feature.items())
assert "lemma" in restr_dict
self.match_list = restr_dict['lemma'].split('|')
def __call__(self, text):
return text in self.match_list
class Restriction:
def __init__(self, restriction_tag):
self.ppb = 4 # polnopomenska beseda (0-4)
if restriction_tag is None:
self.type = RestrictionType.MatchAll
self.matcher = None
self.present = None
return
restriction_type = restriction_tag.get('type')
if restriction_type == "morphology":
self.type = RestrictionType.Morphology
self.matcher = MorphologyRegex(list(restriction_tag))
self.ppb = determine_ppb(self.matcher.rgx)
elif restriction_type == "lexis":
self.type = RestrictionType.Lexis
self.matcher = LexisRegex(list(restriction_tag))
else:
raise NotImplementedError()
def match(self, word):
if self.type == RestrictionType.Morphology:
match_to = word.msd
elif self.type == RestrictionType.Lexis:
match_to = word.lemma
elif self.type == RestrictionType.MatchAll:
return True
else:
raise RuntimeError("Unreachable!")
return self.matcher(match_to)

110
src/syntactic_structure.py Normal file
View File

@ -0,0 +1,110 @@
from xml.etree import ElementTree
import logging
from component import Component, ComponentType
from lemma_features import get_lemma_features
class SyntacticStructure:
def __init__(self):
self.id = None
self.lbs = None
self.components = []
@staticmethod
def from_xml(xml):
st = SyntacticStructure()
st.id = xml.get('id')
st.lbs = xml.get('LBS')
assert len(list(xml)) == 1
system = next(iter(xml))
assert system.get('type') == 'JOS'
components, dependencies, definitions = list(system)
deps = [(dep.get('from'), dep.get('to'), dep.get('label'), dep.get('order'))
for dep in dependencies]
comps = {comp.get('cid'): dict(comp.items()) for comp in components}
restrs, forms = {}, {}
for comp in definitions:
n = comp.get('cid')
restrs[n] = None
forms[n] = []
for el in comp:
if el.tag.startswith("restriction"):
assert restrs[n] is None
restrs[n] = el
elif el.tag.startswith("representation"):
st.add_representation(n, el, forms)
else:
raise NotImplementedError("Unknown definition: {} in structure {}"
.format(el.tag, st.id))
fake_root_component = Component({'cid': '#', 'type': 'other'})
st.components = fake_root_component.find_next(deps, comps, restrs, forms)
st.determine_core2w()
return st
def determine_core2w(self):
ppb_components = []
for c in self.components:
if c.type != ComponentType.Core:
continue
ppb = 4
for r in c.restrictions:
ppb = min(r.ppb, ppb)
ppb_components.append((c, ppb))
ppb_components = sorted(ppb_components, key=lambda c: c[1])
if len(ppb_components) > 2 and ppb_components[1][1] == ppb_components[2][1]:
raise RuntimeError("Cannot determine 2 'jedrna polnopomenska beseda' for", self.id)
for c, _ in ppb_components[:2]:
c.type = ComponentType.Core2w
def add_representation(self, n, rep_el, forms):
assert rep_el.tag == "representation"
to_add = []
for el in rep_el:
assert el.tag == "feature"
if 'rendition' in el.attrib or 'selection' in el.attrib:
to_add.append(el)
else:
logging.warning("Strange representation feature in structure {}. Skipping"
.format(self.id))
continue
forms[n].append(to_add)
def get_component(self, idx):
for c in self.components:
if c.idx == idx:
return c
raise RuntimeError("Unknown component id: {}".format(idx))
def match(self, word):
matches = self.components[0].match(word)
return [] if matches is None else matches
def build_structures(filename):
max_num_components = -1
with open(filename, 'r') as fp:
et = ElementTree.XML(fp.read())
structures = []
for structure in et.iter('syntactic_structure'):
to_append = SyntacticStructure.from_xml(structure)
if to_append is None:
continue
structures.append(to_append)
max_num_components = max(max_num_components, len(to_append.components))
lemma_features = get_lemma_features(et)
return structures, lemma_features, max_num_components

217
src/wani.py Normal file
View File

@ -0,0 +1,217 @@
from xml.etree import ElementTree
import re
import sys
import logging
import argparse
import pickle
import time
import subprocess
import concurrent.futures
import tempfile
try:
from tqdm import tqdm
except ImportError:
tqdm = lambda x: x
from word import Word
from syntactic_structure import build_structures
from match_store import MatchStore
from word_stats import WordStats
from writer import Writer
def is_root_id(id_):
return len(id_.split('.')) == 3
def load_files(args):
filenames = args.input
skip_id_check = args.skip_id_check
do_msd_translate = not args.no_msd_translate
for n, fname in enumerate(filenames):
if args.count_files:
status = " :: {} / {}".format(n, len(filenames))
else:
status = ""
yield load_tei_file(fname, skip_id_check, do_msd_translate, args.pc_tag, status)
def load_tei_file(filename, skip_id_check, do_msd_translate, pc_tag, status):
logging.info("LOADING FILE: {}{}".format(filename, status))
with open(filename, 'r') as fp:
xmlstring = re.sub(' xmlns="[^"]+"', '', fp.read(), count=1)
xmlstring = xmlstring.replace(' xml:', ' ')
et = ElementTree.XML(xmlstring)
words = {}
for w in et.iter("w"):
words[w.get('id')] = Word(w, do_msd_translate)
for pc in et.iter(pc_tag):
words[pc.get('id')] = Word.pc_word(pc, do_msd_translate)
for l in et.iter("link"):
if 'dep' in l.keys():
ana = l.get('afun')
lfrom = l.get('from')
dest = l.get('dep')
else:
ana = l.get('ana')
if ana[:4] != 'syn:': # dont bother...
continue
ana = ana[4:]
lfrom, dest = l.get('target').replace('#', '').split()
if lfrom in words:
if not skip_id_check and is_root_id(lfrom):
logging.error("NOO: {}".format(lfrom))
sys.exit(1)
if dest in words:
next_word = words[dest]
words[lfrom].add_link(ana, next_word)
else:
logging.error("Unknown id: {}".format(dest))
sys.exit(1)
else:
# strange errors, just skip...
pass
return list(words.values())
def match_file(words, structures):
matches = {s: [] for s in structures}
for s in tqdm(structures):
for w in words:
mhere = s.match(w)
for match in mhere:
colocation_id = [(idx, w.lemma) for idx, w in match.items()]
colocation_id = [s.id] + list(sorted(colocation_id, key=lambda x: x[0]))
colocation_id = tuple(colocation_id)
matches[s].append((match, colocation_id))
return matches
def main(structures_file, args):
structures, lemma_msds, max_num_components = build_structures(structures_file)
match_store = MatchStore(args)
word_stats = WordStats(lemma_msds)
if args.parallel:
num_parallel = int(args.parallel)
# make temporary directory to hold temporary files
with tempfile.TemporaryDirectory() as tmpdirname:
cmd = sys.argv
for inpt in args.input:
if inpt in cmd:
cmd.remove(inpt)
# remove "--parallel X"
pidx = cmd.index('--parallel')
del cmd[pidx]
del cmd[pidx]
def func(n):
cmdn = [sys.executable] + cmd + [args.input[n],
"--match-to-file", "{}/{}.p".format(tmpdirname, n)]
subprocess.check_call(cmdn)
return n
# use ThreadPoolExecuter to run subprocesses in parallel using py threads
with concurrent.futures.ThreadPoolExecutor(max_workers=num_parallel) as executor:
# fancy interface to wait for threads to finish
for id_input in executor.map(func, [i for i, _ in enumerate(args.input)]):
with open("{}/{}.p".format(tmpdirname, id_input), "rb") as fp:
words, matches = pickle.load(fp)
match_store.add_matches(matches)
word_stats.add_words(words)
else:
for words in load_files(args):
matches = match_file(words, structures)
# just save to temporary file, used for children of a parallel process
# MUST NOT have more than one file
if args.match_to_file is not None:
with open(args.match_to_file, "wb") as fp:
pickle.dump((words, matches), fp)
return
else:
match_store.add_matches(matches)
word_stats.add_words(words)
# get word renders for lemma/msd
word_stats.generate_renders()
match_store.determine_colocation_dispersions()
# figure out representations!
if args.out or args.out_no_stat:
match_store.set_representations(word_stats)
Writer.make_output_writer(args, max_num_components, match_store, word_stats).write_out(
structures, match_store)
Writer.make_output_no_stat_writer(args, max_num_components, match_store, word_stats).write_out(
structures, match_store)
Writer.make_all_writer(args, max_num_components, match_store, word_stats).write_out(
structures, match_store)
Writer.make_stats_writer(args, max_num_components, match_store, word_stats).write_out(
structures, match_store)
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='Extract structures from a parsed corpus.')
parser.add_argument('structures',
help='Structures definitions in xml file')
parser.add_argument('input',
help='input xml file in `ssj500k form`, can list more than one', nargs='+')
parser.add_argument('--out',
help='Classic output file')
parser.add_argument('--out-no-stat',
help='Output file, but without statistical columns')
parser.add_argument('--all',
help='Additional output file, writes more data')
parser.add_argument('--stats',
help='Output file for statistics')
parser.add_argument('--no-msd-translate',
help='MSDs are translated from slovene to english by default',
action='store_true')
parser.add_argument('--skip-id-check',
help='Skips checks for ids of <w> and <pc>, if they are in correct format',
action='store_true')
parser.add_argument('--min_freq', help='Minimal frequency in output',
type=int, default=0, const=1, nargs='?')
parser.add_argument('--verbose', help='Enable verbose output to stderr',
choices=["warning", "info", "debug"], default="info",
const="info", nargs='?')
parser.add_argument('--count-files',
help="Count files: more verbose output", action='store_true')
parser.add_argument('--multiple-output',
help='Generate one output for each syntactic structure',
action='store_true')
parser.add_argument('--sort-by',
help="Sort by a this column (index)", type=int, default=-1)
parser.add_argument('--sort-reversed',
help="Sort in reversed ored", action='store_true')
parser.add_argument('--pc-tag',
help='Tag for separators, usually pc or c', default="pc")
parser.add_argument('--parallel',
help='Run in multiple processes, should speed things up')
parser.add_argument('--match-to-file', help='Do not use!')
args = parser.parse_args()
logging.basicConfig(stream=sys.stderr, level=args.verbose.upper())
start = time.time()
main(args.structures, args)
logging.info("TIME: {}".format(time.time() - start))

61
src/word.py Normal file
View File

@ -0,0 +1,61 @@
from collections import defaultdict
import logging
from msd_translate import MSD_TRANSLATE
class WordMsdOnly:
def __init__(self, msd):
self.msd = msd
self.lemma = None
self.text = None
def most_frequent_text(self, _):
return None
class Word:
def __init__(self, xml, do_msd_translate):
self.lemma = xml.get('lemma')
self.msd = Word.get_msd(xml)
self.msd = MSD_TRANSLATE[self.msd] if do_msd_translate else self.msd
self.id = xml.get('id')
self.text = xml.text
self.links = defaultdict(list)
last_num = self.id.split('.')[-1]
if last_num[0] not in '0123456789':
last_num = last_num[1:]
self.int_id = int(last_num)
assert None not in (self.id, self.lemma, self.msd)
@staticmethod
def get_msd(comp):
d = dict(comp.items())
if 'msd' in d:
return d['msd']
elif 'ana' in d:
return d['ana'][4:]
else:
logging.error(d)
raise NotImplementedError("MSD?")
@staticmethod
def pc_word(pc, do_msd_translate):
pc.set('lemma', pc.text)
pc.set('msd', "N" if do_msd_translate else "U")
return Word(pc, do_msd_translate)
def add_link(self, link, to):
self.links[link].append(to)
def get_links(self, link):
if link not in self.links and "|" in link:
for l in link.split('|'):
self.links[link].extend(self.links[l])
return self.links[link]
def most_frequent_text(self, word_renderer):
return word_renderer.render(self.lemma, self.msd)

100
src/word_stats.py Normal file
View File

@ -0,0 +1,100 @@
from collections import defaultdict, Counter
class WordStats:
def __init__(self, lemma_features):
self.all_words = []
self.rendered_words = {}
self.frequent_words = {}
self.num_words = {}
self.lemma_msd = {}
self.lemma_features = lemma_features
self.memoized_msd_merges = {}
def add_words(self, words):
self.all_words.extend(words)
def num_all_words(self):
return len(self.all_words)
def generate_renders(self):
num_words = defaultdict(int)
data = defaultdict(lambda: defaultdict(list))
for w in self.all_words:
data[w.lemma][w.msd].append(w.text)
for lemma, ld in data.items():
self.rendered_words[lemma] = {}
freq_words = defaultdict(int)
common_msd = "*" * 10
for msd, texts in ld.items():
# TODO: this should be out of generate_renders...
num_words[(lemma, msd[0])] += len(texts)
rep = max(set(texts), key=texts.count)
self.rendered_words[lemma][msd] = (rep, len(texts))
for txt in texts:
freq_words[(msd, txt)] += 1
common_msd = self.merge_msd(common_msd, msd)
self.lemma_msd[lemma] = common_msd
self.frequent_words[lemma] = []
for (msd, txt), n in sorted(freq_words.items(), key=lambda x: -x[1]):
self.frequent_words[lemma].append((msd, txt, n))
lf = self.lemma_features
for lemma in self.lemma_msd:
cmsd = self.lemma_msd[lemma]
if cmsd[0] in lf:
self.lemma_msd[lemma] = "".join(
l1 if l1 != "-" else l2 for l1, l2 in zip(lf[cmsd[0]], cmsd)
)
self.num_words = dict(num_words)
def merge_msd(self, common_msd, new_msd):
key = (common_msd, new_msd)
if key in self.memoized_msd_merges:
return self.memoized_msd_merges[key]
def merge_letter(l1, l2):
if l1 == "*":
return l2
elif l1 != l2:
return "-"
else:
return l1
value = "".join(merge_letter(l1, l2) for l1, l2 in zip(common_msd, new_msd))
self.memoized_msd_merges[key] = value
return value
def render(self, lemma, msd):
if lemma in self.rendered_words:
if msd in self.rendered_words[lemma]:
return self.rendered_words[lemma][msd][0]
def available_words(self, lemma, existing_texts):
counted_texts = Counter(existing_texts)
for (msd, text), _n in counted_texts.most_common():
yield (msd, text)
if lemma in self.frequent_words:
for msd, text, _ in self.frequent_words[lemma]:
if (msd, text) not in counted_texts:
yield (msd, text)
def get_lemma_msd(self, lemma, word_msd):
# should be here, since we collect every lemmas
lemma_msd = self.lemma_msd[lemma]
if lemma_msd[0] == '-':
if word_msd[0] in self.lemma_features:
return self.lemma_features[word_msd[0]]
else:
return '-'
else:
return lemma_msd

137
src/writer.py Normal file
View File

@ -0,0 +1,137 @@
import logging
from formatter import OutFormatter, OutNoStatFormatter, AllFormatter, StatsFormatter
class Writer:
@staticmethod
def other_params(args):
return (args.multiple_output, int(args.sort_by), args.sort_reversed)
@staticmethod
def make_output_writer(args, num_components, colocation_ids, word_renderer):
params = Writer.other_params(args)
return Writer(args.out, num_components, OutFormatter(colocation_ids, word_renderer), params)
@staticmethod
def make_output_no_stat_writer(args, num_components, colocation_ids, word_renderer):
params = Writer.other_params(args)
return Writer(args.out_no_stat, num_components, OutNoStatFormatter(colocation_ids, word_renderer), params)
@staticmethod
def make_all_writer(args, num_components, colocation_ids, word_renderer):
return Writer(args.all, num_components, AllFormatter(colocation_ids, word_renderer), None)
@staticmethod
def make_stats_writer(args, num_components, colocation_ids, word_renderer):
params = Writer.other_params(args)
return Writer(args.stats, num_components, StatsFormatter(colocation_ids, word_renderer), params)
def __init__(self, file_out, num_components, formatter, params):
if params is None:
self.multiple_output = False
self.sort_by = -1
self.sort_order = None
else:
self.multiple_output = params[0]
self.sort_by = params[1]
self.sort_order = params[2]
self.num_components = num_components
self.output_file = file_out
self.formatter = formatter
def header(self):
repeating_cols = self.formatter.header_repeat()
cols = ["C{}_{}".format(i + 1, thd) for i in range(self.num_components)
for thd in repeating_cols]
cols = ["Structure_ID"] + cols + ["Colocation_ID"]
cols += self.formatter.header_right()
return cols
def sorted_rows(self, rows):
if self.sort_by < 0 or len(rows) < 2:
return rows
if len(rows[0]) <= self.sort_by:
logging.warning("Cannot sort by column #{}: Not enough columns!".format(len(rows[0])))
return rows
try:
int(rows[0][self.sort_by])
def key(row):
return int(row[self.sort_by])
except ValueError:
def key(row):
return row[self.sort_by].lower()
return sorted(rows, key=key, reverse=self.sort_order)
def write_header(self, file_handler):
file_handler.write(", ".join(self.header()) + "\n")
def write_out_worker(self, file_handler, structure, colocation_ids):
rows = []
components = structure.components
for match in colocation_ids.get_matches_for(structure):
self.formatter.new_match(match)
for words in match.matches:
to_write = []
for idx, _comp in enumerate(components):
idx = str(idx + 1)
if idx not in words:
to_write.extend([""] * self.formatter.length())
else:
to_write.extend(self.formatter.content_repeat(words, match.representations, idx, structure.id))
# make them equal size
to_write.extend([""] * (self.num_components * self.formatter.length() - len(to_write)))
# structure_id and colocation_id
to_write = [structure.id] + to_write + [match.match_id]
# header_right
to_write.extend(self.formatter.content_right(len(match)))
rows.append(to_write)
if self.formatter.group():
break
if rows != []:
rows = self.sorted_rows(rows)
file_handler.write("\n".join([", ".join(row) for row in rows]) + "\n")
file_handler.flush()
def write_out(self, structures, colocation_ids):
if self.output_file is None:
return
def fp_close(fp_):
fp_.close()
def fp_open(snum=None):
if snum is None:
return open(self.output_file, "w")
else:
return open("{}.{}".format(self.output_file, snum), "w")
if not self.multiple_output:
fp = fp_open()
self.write_header(fp)
for s in structures:
if self.multiple_output:
fp = fp_open(s.id)
self.write_header(fp)
self.formatter.set_structure(s)
self.write_out_worker(fp, s, colocation_ids)
if self.multiple_output:
fp_close(fp)
if not self.multiple_output:
fp_close(fp)

133
src/writerpy Normal file
View File

@ -0,0 +1,133 @@
class Writer:
@staticmethod
def other_params(args):
return (args.multiple_output, int(args.sort_by), args.sort_reversed)
@staticmethod
def make_output_writer(args, colocation_ids, word_renderer):
params = Writer.other_params(args)
return Writer(args.out, OutFormatter(colocation_ids, word_renderer), params)
@staticmethod
def make_output_no_stat_writer(args, colocation_ids, word_renderer):
params = Writer.other_params(args)
return Writer(args.out_no_stat, OutNoStatFormatter(colocation_ids, word_renderer), params)
@staticmethod
def make_all_writer(args, colocation_ids, word_renderer):
return Writer(args.all, AllFormatter(colocation_ids, word_renderer), None)
@staticmethod
def make_stats_writer(args, colocation_ids, word_renderer):
params = Writer.other_params(args)
return Writer(args.stats, StatsFormatter(colocation_ids, word_renderer), params)
def __init__(self, file_out, formatter, params):
if params is None:
self.multiple_output = False
self.sort_by = -1
self.sort_order = None
else:
self.multiple_output = params[0]
self.sort_by = params[1]
self.sort_order = params[2]
self.output_file = file_out
self.formatter = formatter
def header(self):
repeating_cols = self.formatter.header_repeat()
cols = ["C{}_{}".format(i + 1, thd) for i in range(MAX_NUM_COMPONENTS)
for thd in repeating_cols]
cols = ["Structure_ID"] + cols + ["Colocation_ID"]
cols += self.formatter.header_right()
return cols
def sorted_rows(self, rows):
if self.sort_by < 0 or len(rows) < 2:
return rows
if len(rows[0]) <= self.sort_by:
logging.warning("Cannot sort by column #{}: Not enough columns!".format(len(rows[0])))
return rows
try:
int(rows[0][self.sort_by])
def key(row):
return int(row[self.sort_by])
except ValueError:
def key(row):
return row[self.sort_by].lower()
return sorted(rows, key=key, reverse=self.sort_order)
def write_header(self, file_handler):
file_handler.write(", ".join(self.header()) + "\n")
def write_out_worker(self, file_handler, structure, colocation_ids):
rows = []
components = structure.components
for match in colocation_ids.get_matches_for(structure):
self.formatter.new_match(match)
for words in match.matches:
to_write = []
for idx, _comp in enumerate(components):
idx = str(idx + 1)
if idx not in words:
to_write.extend([""] * self.formatter.length())
else:
to_write.extend(self.formatter.content_repeat(words, match.representations, idx, structure.id))
# make them equal size
to_write.extend([""] * (MAX_NUM_COMPONENTS * self.formatter.length() - len(to_write)))
# structure_id and colocation_id
to_write = [structure.id] + to_write + [match.match_id]
# header_right
to_write.extend(self.formatter.content_right(len(match)))
rows.append(to_write)
if self.formatter.group():
break
if rows != []:
rows = self.sorted_rows(rows)
file_handler.write("\n".join([", ".join(row) for row in rows]) + "\n")
file_handler.flush()
def write_out(self, structures, colocation_ids):
if self.output_file is None:
return
def fp_close(fp_):
if fp_ != sys.stdout:
fp_.close()
def fp_open(snum=None):
if snum is None:
return open(self.output_file, "w")
else:
return open("{}.{}".format(self.output_file, snum), "w")
if not self.multiple_output:
fp = fp_open()
self.write_header(fp)
for s in structures:
if self.multiple_output:
fp = fp_open(s.id)
self.write_header(fp)
self.formatter.set_structure(s)
self.write_out_worker(fp, s, colocation_ids)
if self.multiple_output:
fp_close(fp)
if not self.multiple_output:
fp_close(fp)

1544
wani.py

File diff suppressed because it is too large Load Diff