lkrsnik/stress_asignment
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[MAJOR REFACTOR] Added accent classification (from scratch) and deleted unnecessary y output (output where no accent should be employed) X-es in case of both syllabled inputs have also been changed in similar manner.

Browse Source
This commit is contained in:
lkrsnik 2017-07-27 18:20:18 +02:00
parent 83584a0c6f
commit 18348b78fc
2 changed files with 146 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
.idea/workspace.xml
View File

@ -2,10 +2,8 @@
<project version="4">
<component name="ChangeListManager">
<list default="true" id="8a8ba9af-e1a4-433a-9968-475192610776" name="Default" comment="">
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/.idea/dictionaries/luka.xml" afterPath="$PROJECT_DIR$/.idea/dictionaries/luka.xml" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/.idea/workspace.xml" afterPath="$PROJECT_DIR$/.idea/workspace.xml" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/cnn/word_accetuation/cnn_dictionary/cnn.ipynb" afterPath="$PROJECT_DIR$/cnn/word_accetuation/cnn_dictionary/cnn.ipynb" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/cnn/word_accetuation/syllabled_letters/cnn.ipynb" afterPath="$PROJECT_DIR$/cnn/word_accetuation/syllabled_letters/cnn.ipynb" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/cnn/word_accetuation/syllables/cnn.ipynb" afterPath="$PROJECT_DIR$/cnn/word_accetuation/syllables/cnn.ipynb" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/prepare_data.py" afterPath="$PROJECT_DIR$/prepare_data.py" />
</list>
@ -37,22 +35,25 @@
<file leaf-file-name="prepare_data.py" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/prepare_data.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="568">
<caret line="482" column="74" lean-forward="true" selection-start-line="482" selection-start-column="74" selection-end-line="482" selection-end-column="74" />
<state relative-caret-position="244">
<caret line="252" column="38" lean-forward="true" selection-start-line="252" selection-start-column="38" selection-end-line="252" selection-end-column="38" />
<folding>
<element signature="e#24#63#0" expanded="true" />
<element signature="e#5524#5637#0" expanded="false" />
<element signature="e#5684#6970#0" expanded="false" />
<element signature="e#7131#8538#0" expanded="false" />
<element signature="e#8626#8921#0" expanded="false" />
<element signature="e#13363#13665#0" expanded="false" />
<element signature="e#13722#14551#0" expanded="false" />
<element signature="e#14615#14961#0" expanded="false" />
<element signature="e#16836#17749#0" expanded="false" />
<element signature="e#18179#18375#0" expanded="false" />
<element signature="e#18436#18627#0" expanded="false" />
<element signature="e#18694#19341#0" expanded="false" />
<element signature="e#19440#21738#0" expanded="false" />
<element signature="e#5658#5771#0" expanded="false" />
<element signature="e#5818#7106#0" expanded="false" />
<element signature="e#7267#8674#0" expanded="false" />
<element signature="e#8762#9057#0" expanded="false" />
<element signature="e#13496#13798#0" expanded="false" />
<element signature="e#13855#14684#0" expanded="false" />
<element signature="e#14748#15094#0" expanded="false" />
<element signature="e#16969#17882#0" expanded="false" />
<element signature="e#18312#18508#0" expanded="false" />
<element signature="e#18569#18760#0" expanded="false" />
<element signature="e#18827#19474#0" expanded="false" />
<element signature="e#19573#21871#0" expanded="false" />
<element signature="e#22137#22836#0" expanded="false" />
<element signature="e#29631#29772#0" expanded="false" />
<element signature="e#29922#32067#0" expanded="false" />
</folding>
</state>
</provider>
@ -149,16 +150,6 @@
</component>
<component name="FindInProjectRecents">
<findStrings>
<find>_create_X_features</find>
<find>raise</find>
<find>create_syllables_dictionary</find>
<find>decode_</find>
<find>create_x_features</find>
<find>generate_x_and_y</find>
<find>create_syllables</find>
<find>split_consonants</find>
<find>get_unresonant_silent_consonants</find>
<find>dict_occurances_in_dataset_rate</find>
<find>count_vowels</find>
<find>shuffle_full_vowel_inputs</find>
<find>generate_presentable_y</find>
@ -179,6 +170,16 @@
<find>_create_syllable_letters_translator</find>
<find>_get_unresonant_silent_consonants</find>
<find>el[0]</find>
<find>max_num_vowels</find>
<find>index</find>
<find>accentuated</find>
<find>create_syll</find>
<find>shuffle_all_inputs</find>
<find>accented</find>
<find>_accented</find>
<find>size</find>
<find>decode_x</find>
<find>self._input_type ==</find>
</findStrings>
</component>
<component name="Git.Settings">
@ -532,7 +533,7 @@
</component>
<component name="XDebuggerManager">
<breakpoint-manager>
<option name="time" value="5" />
<option name="time" value="6" />
</breakpoint-manager>
<watches-manager />
</component>
@ -831,22 +832,25 @@
</entry>
<entry file="file://$PROJECT_DIR$/prepare_data.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="568">
<caret line="482" column="74" lean-forward="true" selection-start-line="482" selection-start-column="74" selection-end-line="482" selection-end-column="74" />
<state relative-caret-position="244">
<caret line="252" column="38" lean-forward="true" selection-start-line="252" selection-start-column="38" selection-end-line="252" selection-end-column="38" />
<folding>
<element signature="e#24#63#0" expanded="true" />
<element signature="e#5524#5637#0" expanded="false" />
<element signature="e#5684#6970#0" expanded="false" />
<element signature="e#7131#8538#0" expanded="false" />
<element signature="e#8626#8921#0" expanded="false" />
<element signature="e#13363#13665#0" expanded="false" />
<element signature="e#13722#14551#0" expanded="false" />
<element signature="e#14615#14961#0" expanded="false" />
<element signature="e#16836#17749#0" expanded="false" />
<element signature="e#18179#18375#0" expanded="false" />
<element signature="e#18436#18627#0" expanded="false" />
<element signature="e#18694#19341#0" expanded="false" />
<element signature="e#19440#21738#0" expanded="false" />
<element signature="e#5658#5771#0" expanded="false" />
<element signature="e#5818#7106#0" expanded="false" />
<element signature="e#7267#8674#0" expanded="false" />
<element signature="e#8762#9057#0" expanded="false" />
<element signature="e#13496#13798#0" expanded="false" />
<element signature="e#13855#14684#0" expanded="false" />
<element signature="e#14748#15094#0" expanded="false" />
<element signature="e#16969#17882#0" expanded="false" />
<element signature="e#18312#18508#0" expanded="false" />
<element signature="e#18569#18760#0" expanded="false" />
<element signature="e#18827#19474#0" expanded="false" />
<element signature="e#19573#21871#0" expanded="false" />
<element signature="e#22137#22836#0" expanded="false" />
<element signature="e#29631#29772#0" expanded="false" />
<element signature="e#29922#32067#0" expanded="false" />
</folding>
</state>
</provider>

108
prepare_data.py
View File

@ -11,13 +11,14 @@ import os.path
class Data:
def __init__(self, input_type, allow_shuffle_vector_generation=False, save_generated_data=True, shuffle_all_inputs=True,
additional_letter_attributes=True, reverse_inputs=True):
additional_letter_attributes=True, reverse_inputs=True, accent_classification=False):
self._input_type = input_type
self._save_generated_data = save_generated_data
self._allow_shuffle_vector_generation = allow_shuffle_vector_generation
self._shuffle_all_inputs = shuffle_all_inputs
self._additional_letter_attributes = additional_letter_attributes
self._reverse_inputs = reverse_inputs
self._accent_classification = accent_classification
self.x_train = None
self.x_other_features_train = None
@ -30,14 +31,14 @@ class Data:
self.y_validate = None
def generate_data(self, train_inputs_name, test_inputs_name, validate_inputs_name, test_and_validation_size=0.1,
content_name='SlovarIJS_BESEDE_utf8.lex',
force_override=False, content_name='SlovarIJS_BESEDE_utf8.lex',
content_shuffle_vector='content_shuffle_vector', shuffle_vector='shuffle_vector',
inputs_location='../../internal_representations/inputs/', content_location='../../../data/'):
content_path = '{}{}'.format(content_location, content_name)
train_path = '{}{}.h5'.format(inputs_location, train_inputs_name)
test_path = '{}{}.h5'.format(inputs_location, test_inputs_name)
validate_path = '{}{}.h5'.format(inputs_location, validate_inputs_name)
if os.path.exists(train_path) and os.path.exists(test_path) and os.path.exists(validate_path):
if not force_override and os.path.exists(train_path) and os.path.exists(test_path) and os.path.exists(validate_path):
print('LOADING DATA...')
self.x_train, self.x_other_features_train, self.y_train = self._load_inputs(train_path)
self.x_test, self.x_other_features_test, self.y_test = self._load_inputs(test_path)
@ -62,7 +63,7 @@ class Data:
print('CONTENT READ SUCCESSFULLY')
print('CREATING DICTIONARY...')
dictionary, max_word, max_num_vowels, vowels, accented_vowels = self._create_dict(content)
if self._input_type == 's' or self._input_type == 'ls':
if self._input_type == 's' or self._input_type == 'sl':
dictionary = self._create_syllables_dictionary(content, vowels)
print('DICTIONARY CREATION SUCCESSFUL!')
# test_and_validation_size = 0.1
@ -125,7 +126,7 @@ class Data:
break
line += 1
dictionary_input = sorted(dictionary_input)
max_num_vowels += 1
# max_num_vowels += 1
return dictionary_input, max_word, max_num_vowels, vowels, accented_vowels
# split content so that there is no overfitting
@ -230,23 +231,22 @@ class Data:
word = word[::-1]
j = 0
word_accentuations = []
# word_accentuations = []
num_vowels = 0
for c in list(word):
index = 0
if self._is_vowel(word, j, vowels):
num_vowels += 1
for d in accentuated_vowels:
if c == d:
word_accentuations.append(num_vowels)
if not self._accent_classification:
y[i][num_vowels] = 1
else:
y[i][num_vowels] = index
# word_accentuations.append(num_vowels)
break
index += 1
if self._is_vowel(word, j, vowels):
num_vowels += 1
j += 1
if len(word_accentuations) > 0:
for word_accentuation in word_accentuations:
y[i][word_accentuation] = 1
else:
y[i][0] = 1
i += 1
return y
@ -255,10 +255,10 @@ class Data:
shuffle_vector_location):
if self._input_type == 'l':
x = self._x_letter_input(content, dictionary, max_word, vowels)
elif self._input_type == 's' or self._input_type == 'ls':
elif self._input_type == 's' or self._input_type == 'sl':
x = self._x_syllable_input(content, dictionary, max_num_vowels, vowels)
else:
raise ValueError('No input_type provided. It could be \'l\', \'s\' or \'ls\'.')
raise ValueError('No input_type provided. It could be \'l\', \'s\' or \'sl\'.')
y = self._y_output(content, max_num_vowels, vowels, accentuated_vowels)
print('CREATING OTHER FEATURES...')
@ -476,26 +476,61 @@ class Data:
def _generator_instance(self, orig_x, orig_x_additional, orig_y, batch_size, content_path):
if self._input_type == 'l':
return self._letter_generator(orig_x, orig_x_additional, orig_y, batch_size)
content = self._read_content(content_path)
dictionary, max_word, max_num_vowels, vowels, accented_vowels = self._create_dict(content)
return self._letter_generator(orig_x, orig_x_additional, orig_y, batch_size, accented_vowels)
elif self._input_type == 's':
content = self._read_content(content_path)
dictionary, max_word, max_num_vowels, vowels, accented_vowels = self._create_dict(content)
syllable_dictionary = self._create_syllables_dictionary(content, vowels)
eye = np.eye(len(syllable_dictionary), dtype=int)
return self._syllable_generator(orig_x, orig_x_additional, orig_y, batch_size, eye)
return self._syllable_generator(orig_x, orig_x_additional, orig_y, batch_size, eye, accented_vowels)
elif self._input_type == 'sl':
content = self._read_content(content_path)
dictionary, max_word, max_num_vowels, vowels, accented_vowels = self._create_dict(content)
syllable_dictionary = self._create_syllables_dictionary(content, vowels)
max_syllable = self._get_max_syllable(syllable_dictionary)
syllable_letters_translator = self._create_syllable_letters_translator(max_syllable, syllable_dictionary, dictionary, vowels)
return self._syllable_generator(orig_x, orig_x_additional, orig_y, batch_size, syllable_letters_translator)
return self._syllable_generator(orig_x, orig_x_additional, orig_y, batch_size, syllable_letters_translator, accented_vowels)
# generator for inputs for tracking of data fitting
def _letter_generator(self, orig_x, orig_x_additional, orig_y, batch_size):
def _letter_generator(self, orig_x, orig_x_additional, orig_y, batch_size, accented_vowels):
size = orig_x.shape[0]
while 1:
loc = 0
if self._accent_classification:
eye = np.eye(len(accented_vowels), dtype=int)
eye_input_accent = np.eye(len(orig_y[0]), dtype=int)
input_x_stack = []
input_x_other_features_stack = []
input_y_stack = []
while loc < size:
while len(input_x_stack) < batch_size and loc < size:
accent_loc = 0
for accent in orig_y[loc]:
if accent > 0:
new_orig_x_additional = orig_x_additional[loc]
new_orig_x_additional = np.concatenate((new_orig_x_additional, eye_input_accent[accent_loc]))
input_x_stack.append(orig_x[loc])
input_x_other_features_stack.append(new_orig_x_additional)
input_y_stack.append(eye[int(accent)])
accent_loc += 1
loc += 1
if len(input_x_stack) > batch_size:
yield ([np.array(input_x_stack[:batch_size]),
np.array(input_x_other_features_stack[:batch_size])], np.array(input_y_stack)[:batch_size])
input_x_stack = input_x_stack[batch_size:]
input_x_other_features_stack = input_x_other_features_stack[batch_size:]
input_y_stack = input_y_stack[batch_size:]
else:
# print('BBB')
# print(np.array(input_stack))
# yield (np.array(input_stack))
yield ([np.array(input_x_stack), np.array(input_x_other_features_stack)], np.array(input_y_stack))
input_x_stack = []
input_x_other_features_stack = []
input_y_stack = []
else:
while loc < size:
if loc + batch_size >= size:
yield ([orig_x[loc:size], orig_x_additional[loc:size]], orig_y[loc:size])
@ -504,10 +539,41 @@ class Data:
loc += batch_size
# generator for inputs for tracking of data fitting
def _syllable_generator(self, orig_x, orig_x_additional, orig_y, batch_size, translator):
def _syllable_generator(self, orig_x, orig_x_additional, orig_y, batch_size, translator, accented_vowels):
size = orig_x.shape[0]
while 1:
loc = 0
if self._accent_classification:
eye = np.eye(len(accented_vowels), dtype=int)
eye_input_accent = np.eye(len(orig_y[0]), dtype=int)
input_x_stack = []
input_x_other_features_stack = []
input_y_stack = []
while loc < size:
while len(input_x_stack) < batch_size and loc < size:
accent_loc = 0
for accent in orig_y[loc]:
if accent > 0:
new_orig_x_additional = orig_x_additional[loc]
new_orig_x_additional = np.concatenate((new_orig_x_additional, eye_input_accent[accent_loc]))
input_x_stack.append(orig_x[loc])
input_x_other_features_stack.append(new_orig_x_additional)
input_y_stack.append(eye[int(accent)])
accent_loc += 1
loc += 1
if len(input_x_stack) > batch_size:
gen_orig_x = translator[np.array(input_x_stack[:batch_size])]
yield ([gen_orig_x, np.array(input_x_other_features_stack[:batch_size])], np.array(input_y_stack)[:batch_size])
input_x_stack = input_x_stack[batch_size:]
input_x_other_features_stack = input_x_other_features_stack[batch_size:]
input_y_stack = input_y_stack[batch_size:]
else:
gen_orig_x = translator[np.array(input_x_stack)]
yield ([gen_orig_x, np.array(input_x_other_features_stack)], np.array(input_y_stack))
input_x_stack = []
input_x_other_features_stack = []
input_y_stack = []
else:
while loc < size:
if loc + batch_size >= size:
gen_orig_x = translator[orig_x[loc:size]]