ozbolt/luscenje_struktur
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

White reset at paragraphs not sentences + progress bar updates on paragraphs not sentences.

Browse Source
This commit is contained in:
Luka 2021-01-26 14:57:42 +01:00
parent 552f2e4bd0
commit f1366548b6
1 changed files with 61 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

125
luscenje_struktur/loader.py
View File

@ -197,76 +197,73 @@ def file_sentence_generator(et, args):
previous_pc = False
words = {}
sentences = list(et.iter('s'))
for sentence in progress(sentences, "load-text"):
# create fake root word
words[sentence.get('id')] = Word.fake_root_word(sentence.get('id'))
last_word_id = None
paragraphs = list(et.iter('p'))
for paragraph in progress(paragraphs, "load-text"):
previous_glue = ''
sentences = list(paragraph.iter('s'))
for sentence in sentences:
# create fake root word
words[sentence.get('id')] = Word.fake_root_word(sentence.get('id'))
last_word_id = None
if args.new_tei:
for w in sentence.iter():
if w.tag == 'w':
words[w.get('id')] = Word.from_xml(w, do_msd_translate)
if use_punctuations:
previous_glue = '' if 'join' in w.attrib and w.get('join') == 'right' else ' '
elif w.tag == pc_tag:
words[w.get('id')] = Word.pc_word(w, do_msd_translate)
if use_punctuations:
words[w.get('id')].previous_glue = previous_glue
words[w.get('id')].glue = '' if 'join' in w.attrib and w.get('join') == 'right' else ' '
previous_glue = '' if 'join' in w.attrib and w.get('join') == 'right' else ' '
else:
for w in sentence.iter():
if w.tag == 'w':
words[w.get('id')] = Word.from_xml(w, do_msd_translate)
if use_punctuations:
previous_glue = ''
last_word_id = None
elif w.tag == pc_tag:
words[w.get('id')] = Word.pc_word(w, do_msd_translate)
if use_punctuations:
last_word_id = w.get('id')
words[w.get('id')].previous_glue = previous_glue
previous_glue = ''
elif use_punctuations and w.tag == 'c':
# always save previous glue
previous_glue = w.text
if last_word_id:
words[last_word_id].glue += w.text
# for w in sentence.iter("w"):
# words[w.get('id')] = Word.from_xml(w, do_msd_translate)
# for pc in sentence.iter(pc_tag):
# words[pc.get('id')] = Word.pc_word(pc, do_msd_translate)
for l in sentence.iter("link"):
if 'dep' in l.keys():
ana = l.get('afun')
lfrom = l.get('from')
dest = l.get('dep')
if args.new_tei:
for w in sentence.iter():
if w.tag == 'w':
words[w.get('id')] = Word.from_xml(w, do_msd_translate)
if use_punctuations:
previous_glue = '' if 'join' in w.attrib and w.get('join') == 'right' else ' '
elif w.tag == pc_tag:
words[w.get('id')] = Word.pc_word(w, do_msd_translate)
if use_punctuations:
words[w.get('id')].previous_glue = previous_glue
words[w.get('id')].glue = '' if 'join' in w.attrib and w.get('join') == 'right' else ' '
previous_glue = '' if 'join' in w.attrib and w.get('join') == 'right' else ' '
else:
ana = l.get('ana')
if ana[:8] != 'jos-syn:': # dont bother...
continue
ana = ana[8:]
lfrom, dest = l.get('target').replace('#', '').split()
for w in sentence.iter():
if w.tag == 'w':
words[w.get('id')] = Word.from_xml(w, do_msd_translate)
if use_punctuations:
previous_glue = ''
last_word_id = None
elif w.tag == pc_tag:
words[w.get('id')] = Word.pc_word(w, do_msd_translate)
if use_punctuations:
last_word_id = w.get('id')
words[w.get('id')].previous_glue = previous_glue
previous_glue = ''
elif use_punctuations and w.tag == 'c':
# always save previous glue
previous_glue = w.text
if last_word_id:
words[last_word_id].glue += w.text
if lfrom in words:
if not skip_id_check and is_root_id(lfrom):
logging.error("Id {} is not fine, you might want to try with tag --skip-id-check".format(lfrom))
sys.exit(1)
if dest in words:
next_word = words[dest]
words[lfrom].add_link(ana, next_word)
for l in sentence.iter("link"):
if 'dep' in l.keys():
ana = l.get('afun')
lfrom = l.get('from')
dest = l.get('dep')
else:
logging.error("Unknown id: {}".format(dest))
sys.exit(1)
ana = l.get('ana')
if ana[:8] != 'jos-syn:': # dont bother...
continue
ana = ana[8:]
lfrom, dest = l.get('target').replace('#', '').split()
else:
# strange errors, just skip...
pass
if lfrom in words:
if not skip_id_check and is_root_id(lfrom):
logging.error("Id {} is not fine, you might want to try with tag --skip-id-check".format(lfrom))
sys.exit(1)
if dest in words:
next_word = words[dest]
words[lfrom].add_link(ana, next_word)
else:
logging.error("Unknown id: {}".format(dest))
sys.exit(1)
else:
# strange errors, just skip...
pass
return list(words.values())