lkrsnik/stress_asignment
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added fake_epoch for observation

Browse Source
This commit is contained in:
lkrsnik 2017-07-07 12:45:47 +02:00
parent 592ad9b26b
commit 669aa6bbfd
2 changed files with 28 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
.idea/workspace.xml
View File

@ -2,7 +2,9 @@
<project version="4"> <project version="4">
<component name="ChangeListManager"> <component name="ChangeListManager">
<list default="true" id="8a8ba9af-e1a4-433a-9968-475192610776" name="Default" comment=""> <list default="true" id="8a8ba9af-e1a4-433a-9968-475192610776" name="Default" comment="">
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/.idea/workspace.xml" afterPath="$PROJECT_DIR$/.idea/workspace.xml" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/cnn/word_accetuation/cnn_dictionary/cnn.ipynb" afterPath="$PROJECT_DIR$/cnn/word_accetuation/cnn_dictionary/cnn.ipynb" /> <change type="MODIFICATION" beforePath="$PROJECT_DIR$/cnn/word_accetuation/cnn_dictionary/cnn.ipynb" afterPath="$PROJECT_DIR$/cnn/word_accetuation/cnn_dictionary/cnn.ipynb" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/prepare_data.py" afterPath="$PROJECT_DIR$/prepare_data.py" />
</list> </list>
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" /> <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="TRACKING_ENABLED" value="true" /> <option name="TRACKING_ENABLED" value="true" />
@ -32,8 +34,8 @@
<file leaf-file-name="prepare_data.py" pinned="false" current-in-tab="true"> <file leaf-file-name="prepare_data.py" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/prepare_data.py"> <entry file="file://$PROJECT_DIR$/prepare_data.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="504"> <state relative-caret-position="406">
<caret line="315" column="28" lean-forward="false" selection-start-line="315" selection-start-column="28" selection-end-line="315" selection-end-column="28" /> <caret line="461" column="22" lean-forward="false" selection-start-line="461" selection-start-column="22" selection-end-line="461" selection-end-column="22" />
<folding> <folding>
<element signature="e#24#63#0" expanded="true" /> <element signature="e#24#63#0" expanded="true" />
</folding> </folding>
@ -135,6 +137,7 @@
<find>split_number</find> <find>split_number</find>
<find>StringIO</find> <find>StringIO</find>
<find>shuffle_inputs</find> <find>shuffle_inputs</find>
<find>generator</find>
</findStrings> </findStrings>
</component> </component>
<component name="Git.Settings"> <component name="Git.Settings">
@ -178,7 +181,7 @@
<foldersAlwaysOnTop value="true" /> <foldersAlwaysOnTop value="true" />
</navigator> </navigator>
<panes> <panes>
<pane id="Scope" /> <pane id="Scratches" />
<pane id="ProjectPane"> <pane id="ProjectPane">
<subPane> <subPane>
<PATH> <PATH>
@ -205,49 +208,9 @@
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT> </PATH_ELEMENT>
</PATH> </PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="accetuation" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="accetuation" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="cnn" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="word_accetuation" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="accetuation" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="accetuation" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="cnn" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="word_accetuation" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="cnn_dictionary" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
</subPane> </subPane>
</pane> </pane>
<pane id="Scratches" /> <pane id="Scope" />
</panes> </panes>
</component> </component>
<component name="PropertiesComponent"> <component name="PropertiesComponent">
@ -860,8 +823,8 @@
<entry file="file://$PROJECT_DIR$/cnn/word_accetuation/cnn_dictionary/cnn_test_on_other_attributes.ipynb" /> <entry file="file://$PROJECT_DIR$/cnn/word_accetuation/cnn_dictionary/cnn_test_on_other_attributes.ipynb" />
<entry file="file://$PROJECT_DIR$/prepare_data.py"> <entry file="file://$PROJECT_DIR$/prepare_data.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="504"> <state relative-caret-position="406">
<caret line="315" column="28" lean-forward="false" selection-start-line="315" selection-start-column="28" selection-end-line="315" selection-end-column="28" /> <caret line="461" column="22" lean-forward="false" selection-start-line="461" selection-start-column="22" selection-end-line="461" selection-end-column="22" />
<folding> <folding>
<element signature="e#24#63#0" expanded="true" /> <element signature="e#24#63#0" expanded="true" />
</folding> </folding>
@ -928,12 +891,6 @@
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/cnn/word_accetuation/cnn_dictionary/character_based_ffnn_keras.ipynb"> <entry file="file://$PROJECT_DIR$/cnn/word_accetuation/cnn_dictionary/character_based_ffnn_keras.ipynb" />
<provider selected="true" editor-type-id="ipnb-editor">
<state>
<selected id="-1" />
</state>
</provider>
</entry>
</component> </component>
</project> </project>

18
prepare_data.py
View File

@ -7,6 +7,7 @@ import h5py
import gc import gc
import math import math
# functions for saving, loading and shuffling whole arrays to ram # functions for saving, loading and shuffling whole arrays to ram
def save_inputs(file_name, X, y, other_features=[]): def save_inputs(file_name, X, y, other_features=[]):
h5f = h5py.File(file_name, 'w') h5f = h5py.File(file_name, 'w')
@ -18,6 +19,7 @@ def save_inputs(file_name, X, y, other_features=[]):
h5f.create_dataset(k, data=v) h5f.create_dataset(k, data=v)
h5f.close() h5f.close()
def load_inputs(file_name, other_features=False): def load_inputs(file_name, other_features=False):
h5f = h5py.File(file_name,'r') h5f = h5py.File(file_name,'r')
X = h5f['X'][:] X = h5f['X'][:]
@ -42,6 +44,7 @@ def shuffle_inputs(X, y, X_pure=[]):
else: else:
return X, y return X, y
# functions for saving and loading partial arrays to ram # functions for saving and loading partial arrays to ram
def create_and_save_inputs(file_name, part, X, y, X_pure): def create_and_save_inputs(file_name, part, X, y, X_pure):
# X, y, X_pure = generate_full_vowel_matrix_inputs() # X, y, X_pure = generate_full_vowel_matrix_inputs()
@ -51,6 +54,7 @@ def create_and_save_inputs(file_name, part, X, y, X_pure):
h5f.create_dataset(k,data=v) h5f.create_dataset(k,data=v)
h5f.close() h5f.close()
def load_extended_inputs(file_name, obtain_range): def load_extended_inputs(file_name, obtain_range):
h5f = h5py.File(file_name,'r') h5f = h5py.File(file_name,'r')
X = h5f['X'][obtain_range[0]:obtain_range[1]] X = h5f['X'][obtain_range[0]:obtain_range[1]]
@ -70,6 +74,7 @@ def create_and_save_shuffle_vector(file_name, shuffle_vector):
h5f.create_dataset(k,data=v) h5f.create_dataset(k,data=v)
h5f.close() h5f.close()
def load_shuffle_vector(file_name): def load_shuffle_vector(file_name):
h5f = h5py.File(file_name,'r') h5f = h5py.File(file_name,'r')
shuffle_vector = h5f['shuffle_vector'][[179859, 385513, 893430]] shuffle_vector = h5f['shuffle_vector'][[179859, 385513, 893430]]
@ -452,6 +457,19 @@ def generate_X_and_y_RAM_efficient(name, split_number):
h5f.close() h5f.close()
# generator for inputs for tracking of data fitting
def generate_fake_epoch(orig_X, orig_X_additional, orig_y, batch_size):
size = orig_X.shape[0]
while 1:
loc = 0
while loc < size:
if loc + batch_size >= size:
yield([orig_X[loc:size], orig_X_additional[loc:size]], orig_y[loc:size])
else:
yield([orig_X[loc:loc + batch_size], orig_X_additional[loc:loc + batch_size]], orig_y[loc:loc + batch_size])
loc += batch_size
# generator for inputs # generator for inputs
def generate_arrays_from_file(path, batch_size): def generate_arrays_from_file(path, batch_size):
h5f = h5py.File(path, 'r') h5f = h5py.File(path, 'r')