New progress bar

This commit is contained in:
2019-06-17 17:30:51 +02:00
parent 3552f14b81
commit 70b05e8637
7 changed files with 70 additions and 21 deletions

View File

@@ -3,8 +3,7 @@ import logging
import re
import sys
from tqdm import tqdm
from progress_bar import progress
from word import Word
@@ -23,7 +22,6 @@ def load_files(args):
def load_xml(filename):
logging.info("LOADING XML: {}".format(filename))
with open(filename, 'r') as fp:
content = fp.read()
@@ -35,7 +33,7 @@ def load_xml(filename):
def file_sentence_generator(et, skip_id_check, do_msd_translate, pc_tag):
words = {}
sentences = list(et.iter('s'))
for sentence in tqdm(sentences, desc="load-text"):
for sentence in progress(sentences, "load-text", infile=True):
for w in sentence.iter("w"):
words[w.get('id')] = Word(w, do_msd_translate)
for pc in sentence.iter(pc_tag):