2019-02-03 21:54:26 +00:00
|
|
|
from parser import parser
|
|
|
|
|
import os
|
|
|
|
|
from os.path import join
|
|
|
|
|
|
|
|
|
|
SSJ500K_2_1 = 27829 # number of sentences
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
2019-02-12 01:48:34 +00:00
|
|
|
# make sure you sanitize every input into unicode
|
2019-02-03 21:54:26 +00:00
|
|
|
|
2019-02-12 01:48:34 +00:00
|
|
|
print("parsing ssj")
|
|
|
|
|
ssj_file = "../data/ssj500k-sl.sample.xml"
|
|
|
|
|
ssj_dict = parser.parse_tei(ssj_file)
|
|
|
|
|
# assert (len(ssj_dict) == 27829), "Parsed wrong number of sentences."
|
|
|
|
|
print("end parsing ssj")
|
2019-02-03 21:54:26 +00:00
|
|
|
|
2019-02-12 01:48:34 +00:00
|
|
|
print("parsing kres")
|
|
|
|
|
# kres_file = "../data/kres_example/F0019343.xml.parsed.xml"
|
|
|
|
|
kres_dir = "../data/kres_example/"
|
|
|
|
|
for kres_file in os.listdir(kres_dir):
|
|
|
|
|
res_dict = parser.parse_tei(join(kres_dir, kres_file))
|
|
|
|
|
for _, sentence in res_dict.items():
|
|
|
|
|
parser.to_conll09(sentence)
|
|
|
|
|
print("end parsing kres")
|
