Added multiple results and error analysis

master
lkrsnik 7 years ago
parent 7180550b0d
commit dbd0b90f92

@ -2,16 +2,27 @@
<project version="4">
<component name="ChangeListManager">
<list default="true" id="8a8ba9af-e1a4-433a-9968-475192610776" name="Default" comment="">
<change type="DELETED" beforePath="$PROJECT_DIR$/cnn/accent_classification/letters/60_epoch.h5" afterPath="" />
<change type="DELETED" beforePath="$PROJECT_DIR$/cnn/accent_classification/letters/60_epoch_history.pkl" afterPath="" />
<change type="DELETED" beforePath="$PROJECT_DIR$/cnn/word_accetuation/cnn_dictionary/test_with_additional_letter_attributes/40_epoch.h5" afterPath="" />
<change type="DELETED" beforePath="$PROJECT_DIR$/cnn/word_accetuation/cnn_dictionary/test_with_additional_letter_attributes/40_epoch_history.pkl" afterPath="" />
<change type="DELETED" beforePath="$PROJECT_DIR$/cnn/word_accetuation/cnn_dictionary/test_with_additional_letter_attributes/cnn.ipynb" afterPath="" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/cnn/word_accetuation/syllabled_letters/v2_5_3/40_epoch.h5" afterPath="$PROJECT_DIR$/cnn/word_accetuation/syllabled_letters/v2_5_3/40_epoch.h5" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/cnn/word_accetuation/syllabled_letters/v2_5_3/40_epoch_history.pkl" afterPath="$PROJECT_DIR$/cnn/word_accetuation/syllabled_letters/v2_5_3/40_epoch_history.pkl" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/.idea/workspace.xml" afterPath="$PROJECT_DIR$/.idea/workspace.xml" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/cnn/accent_classification/letters/cnn.ipynb" afterPath="$PROJECT_DIR$/cnn/accent_classification/letters/cnn.ipynb" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/cnn/word_accetuation/cnn_dictionary/cnn.ipynb" afterPath="$PROJECT_DIR$/cnn/word_accetuation/cnn_dictionary/cnn.ipynb" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/cnn/word_accetuation/cnn_dictionary/connected_text_accetuation.ipynb" afterPath="$PROJECT_DIR$/cnn/word_accetuation/cnn_dictionary/connected_text_accetuation.ipynb" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/cnn/word_accetuation/cnn_dictionary/results_presentation.ipynb" afterPath="$PROJECT_DIR$/cnn/word_accetuation/cnn_dictionary/results_presentation.ipynb" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/cnn/word_accetuation/syllabled_letters/cnn.ipynb" afterPath="$PROJECT_DIR$/cnn/word_accetuation/syllabled_letters/cnn.ipynb" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/cnn/word_accetuation/syllables/cnn.ipynb" afterPath="$PROJECT_DIR$/cnn/word_accetuation/syllables/cnn.ipynb" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/cnn/word_accetuation/syllabled_letters/v2_5_3/gmlog/description" afterPath="$PROJECT_DIR$/cnn/word_accetuation/syllabled_letters/v2_5_3/gmlog/description" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/cnn/word_accetuation/syllabled_letters/v2_5_3/gmlog/diag" afterPath="$PROJECT_DIR$/cnn/word_accetuation/syllabled_letters/v2_5_3/gmlog/diag" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/cnn/word_accetuation/syllabled_letters/v2_5_3/gmlog/errors" afterPath="$PROJECT_DIR$/cnn/word_accetuation/syllabled_letters/v2_5_3/gmlog/errors" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/cnn/word_accetuation/syllabled_letters/v2_5_3/gmlog/grami" afterPath="$PROJECT_DIR$/cnn/word_accetuation/syllabled_letters/v2_5_3/gmlog/grami" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/cnn/word_accetuation/syllabled_letters/v2_5_3/gmlog/local" afterPath="$PROJECT_DIR$/cnn/word_accetuation/syllabled_letters/v2_5_3/gmlog/local" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/cnn/word_accetuation/syllabled_letters/v2_5_3/gmlog/xml" afterPath="$PROJECT_DIR$/cnn/word_accetuation/syllabled_letters/v2_5_3/gmlog/xml" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/cnn/word_accetuation/syllabled_letters/v2_5_3/notes" afterPath="$PROJECT_DIR$/cnn/word_accetuation/syllabled_letters/v2_5_3/notes" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/cnn/word_accetuation/syllabled_letters/v2_5_3/out.txt" afterPath="$PROJECT_DIR$/cnn/word_accetuation/syllabled_letters/v2_5_3/out.txt" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/cnn/word_accetuation/syllabled_letters/v2_5_3/workbench.py" afterPath="$PROJECT_DIR$/cnn/word_accetuation/syllabled_letters/v2_5_3/workbench.py" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/prepare_data.py" afterPath="$PROJECT_DIR$/prepare_data.py" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/workbench.py" afterPath="$PROJECT_DIR$/workbench.py" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/workbench.xrsl" afterPath="$PROJECT_DIR$/workbench.xrsl" />
</list>
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="TRACKING_ENABLED" value="true" />
@ -25,7 +36,7 @@
</component>
<component name="ExecutionTargetManager" SELECTED_TARGET="default_target" />
<component name="FileEditorManager">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="208">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="199">
<file leaf-file-name="test.py" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/theano_tutorial/test.py">
<provider selected="true" editor-type-id="text-editor">
@ -38,11 +49,11 @@
</provider>
</entry>
</file>
<file leaf-file-name="prepare_data.py" pinned="false" current-in-tab="false">
<file leaf-file-name="prepare_data.py" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/prepare_data.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="298">
<caret line="397" column="8" lean-forward="false" selection-start-line="397" selection-start-column="8" selection-end-line="397" selection-end-column="8" />
<state relative-caret-position="-566">
<caret line="888" column="30" lean-forward="true" selection-start-line="888" selection-start-column="30" selection-end-line="888" selection-end-column="30" />
<folding>
<element signature="e#24#63#0" expanded="true" />
<element signature="e#5979#7267#0" expanded="false" />
@ -60,6 +71,16 @@
</provider>
</entry>
</file>
<file leaf-file-name="workbench.xrsl" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/workbench.xrsl">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="216">
<caret line="12" column="14" lean-forward="false" selection-start-line="12" selection-start-column="14" selection-end-line="12" selection-end-column="14" />
<folding />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="tex_hyphenation.py" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/tex_hyphenation.py">
<provider selected="true" editor-type-id="text-editor">
@ -70,13 +91,14 @@
</provider>
</entry>
</file>
<file leaf-file-name="workbench.py" pinned="false" current-in-tab="true">
<file leaf-file-name="workbench.py" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/workbench.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="716">
<caret line="59" column="19" lean-forward="false" selection-start-line="59" selection-start-column="16" selection-end-line="59" selection-end-column="19" />
<state relative-caret-position="216">
<caret line="41" column="0" lean-forward="false" selection-start-line="41" selection-start-column="0" selection-end-line="41" selection-end-column="0" />
<folding>
<element signature="e#24#63#0" expanded="true" />
<marker date="1504771545370" expanded="true" signature="3097:3272" ph="..." />
</folding>
</state>
</provider>
@ -161,14 +183,6 @@
</component>
<component name="FindInProjectRecents">
<findStrings>
<find>_get_unresonant_silent_consonants</find>
<find>el[0]</find>
<find>max_num_vowels</find>
<find>index</find>
<find>accentuated</find>
<find>create_syll</find>
<find>shuffle_all_inputs</find>
<find>accented</find>
<find>_accented</find>
<find>size</find>
<find>decode_x</find>
@ -191,6 +205,14 @@
<find>test_and_validation_size</find>
<find>_accent_classification</find>
<find>self.y_train</find>
<find>_additional_letter_attributes</find>
<find>np.random.seed</find>
<find>round</find>
<find>is_vow</find>
<find>self._input_type == 'l'</find>
<find>print</find>
<find>np.eye</find>
<find>allow_shuffle_vector_generation</find>
</findStrings>
</component>
<component name="Git.Settings">
@ -209,9 +231,10 @@
<option value="$PROJECT_DIR$/cnn/word_accetuation/cnn_dictionary/character_based_ffnn_keras.py" />
<option value="$PROJECT_DIR$/cnn/word_accetuation/cnn_dictionary/character_based_ffnn_keras.ipynb" />
<option value="$PROJECT_DIR$/tex_hyphenation.py" />
<option value="$PROJECT_DIR$/prepare_data.py" />
<option value="$PROJECT_DIR$/notes" />
<option value="$PROJECT_DIR$/workbench.xrsl" />
<option value="$PROJECT_DIR$/workbench.py" />
<option value="$PROJECT_DIR$/prepare_data.py" />
</list>
</option>
</component>
@ -236,6 +259,7 @@
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="Scratches" />
<pane id="ProjectPane">
<subPane>
<PATH>
@ -251,7 +275,6 @@
</subPane>
</pane>
<pane id="Scope" />
<pane id="Scratches" />
</panes>
</component>
<component name="PropertiesComponent">
@ -517,7 +540,7 @@
<frame x="65" y="144" width="1855" height="1056" extended-state="6" />
<editor active="true" />
<layout>
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.16375546" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.12227074" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
<window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="true" content_ui="tabs" />
<window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="10" side_tool="false" content_ui="tabs" />
@ -550,16 +573,6 @@
<watches-manager />
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/theano_tutorial/logistic_regression.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="846">
<caret line="49" column="31" lean-forward="false" selection-start-line="49" selection-start-column="31" selection-end-line="49" selection-end-column="31" />
<folding>
<element signature="e#0#12#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/theano_tutorial/tutorial_derivates.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
@ -840,6 +853,14 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/notes">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="18">
<caret line="1" column="0" lean-forward="false" selection-start-line="1" selection-start-column="0" selection-end-line="1" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/theano_tutorial/logistic_regression.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="162">
@ -888,18 +909,29 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/notes">
<entry file="file://$PROJECT_DIR$/workbench.xrsl">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="18">
<caret line="1" column="0" lean-forward="false" selection-start-line="1" selection-start-column="0" selection-end-line="1" selection-end-column="0" />
<state relative-caret-position="216">
<caret line="12" column="14" lean-forward="false" selection-start-line="12" selection-start-column="14" selection-end-line="12" selection-end-column="14" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/workbench.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="216">
<caret line="41" column="0" lean-forward="false" selection-start-line="41" selection-start-column="0" selection-end-line="41" selection-end-column="0" />
<folding>
<element signature="e#24#63#0" expanded="true" />
<marker date="1504771545370" expanded="true" signature="3097:3272" ph="..." />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/prepare_data.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="298">
<caret line="397" column="8" lean-forward="false" selection-start-line="397" selection-start-column="8" selection-end-line="397" selection-end-column="8" />
<state relative-caret-position="-566">
<caret line="888" column="30" lean-forward="true" selection-start-line="888" selection-start-column="30" selection-end-line="888" selection-end-column="30" />
<folding>
<element signature="e#24#63#0" expanded="true" />
<element signature="e#5979#7267#0" expanded="false" />
@ -916,15 +948,5 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/workbench.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="716">
<caret line="59" column="19" lean-forward="false" selection-start-line="59" selection-start-column="16" selection-end-line="59" selection-end-column="19" />
<folding>
<element signature="e#24#63#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</component>
</project>

@ -0,0 +1,6 @@
Using Theano backend.
WARNING (theano.sandbox.cuda): The cuda backend is deprecated and will be removed in the next release (v0.10). Please switch to the gpuarray backend. You can get more information about how to switch at this URL:
https://github.com/Theano/Theano/wiki/Converting-to-the-new-gpu-back-end%28gpuarray%29
WARNING (theano.sandbox.cuda): CUDA is installed, but device gpu is not available (error: Unable to get the number of gpus available: no CUDA-capable device is detected)
WARNING (theano.tensor.blas): We did not found a dynamic library into the library_dir of the library we use for blas. If you use ATLAS, make sure to compile it with dynamics library.

@ -0,0 +1 @@
&( executable = "workbench.sh" )( stdout = "out.txt" )( stderr = "err.txt" )( cputime = "45000" )( walltime = "45000" )( memory = "12000" )( inputfiles = ( "workbench.py" ".412054917326" ) ( "prepare_data.py" ".422203461399818" ) ( "content_shuffle_vector.h5" "124536" ) ( "shuffle_vector_test.h5" "418608" ) ( "shuffle_vector_train.h5" "3443352" ) ( "shuffle_vector_validate.h5" "435920" ) ( "SlovarIJS_BESEDE_utf8.lex" "29360549" ) ( "40_epoch.h5" ".04294967295" ) ( "40_epoch_history.pkl" ".04294967295" ) ( "notes" ".791623890922" ) ( "workbench.sh" ".1433001277835" ) )( executables = "workbench.sh" )( outputfiles = ( "40_epoch.h5" "" ) ( "workbench.py" "" ) ( "workbench.sh" "" ) ( "40_epoch_history.pkl" "" ) ( "notes" "" ) ( "out.txt" "" ) ( "err.txt" "" ) ( "gmlog" "" ) )( queue = "gridlong_nsc" )( runtimeenvironment = "APPS/BASE/THEANO-GPU-0.9" )( gmlog = "gmlog" )( jobname = "accetuation_nn" )( action = "request" )( clientsoftware = "libarccompute-5.0.5" )( clientxrsl = "&( jobname = ""accetuation_nn"" )( executable = ""workbench.sh"" )( inputfiles = ( ""workbench.py"" """" ) ( ""prepare_data.py"" """" ) ( ""content_shuffle_vector.h5"" ""cnn/internal_representations/inputs/content_shuffle_vector.h5"" ) ( ""shuffle_vector_test.h5"" ""cnn/internal_representations/inputs/shuffle_vector_test.h5"" ) ( ""shuffle_vector_train.h5"" ""cnn/internal_representations/inputs/shuffle_vector_train.h5"" ) ( ""shuffle_vector_validate.h5"" ""cnn/internal_representations/inputs/shuffle_vector_validate.h5"" ) ( ""SlovarIJS_BESEDE_utf8.lex"" ""data/SlovarIJS_BESEDE_utf8.lex"" ) ( ""40_epoch.h5"" """" ) ( ""40_epoch_history.pkl"" """" ) ( ""notes"" """" ) )( outputfiles = ( ""40_epoch.h5"" """" ) ( ""workbench.py"" """" ) ( ""workbench.sh"" """" ) ( ""40_epoch_history.pkl"" """" ) ( ""notes"" """" ) )( stdout = ""out.txt"" )( stderr = ""err.txt"" )( gmlog = ""gmlog"" )( runtimeenvironment = ""APPS/BASE/THEANO-GPU-0.9"" )( gridtime = ""750"" )( memory = ""12000"" )" )( hostname = "luka-laptop" )( savestate = "yes" )

@ -0,0 +1,8 @@
runtimeenvironments=APPS/BASE/THEANO-GPU-0.9;
nodename=nsc-fp006.ijs.si
WallTime=45010s
Processors=1
UserTime=45010s
LRMSStartTime=20170906082225Z
LRMSEndTime=20170906205235Z

@ -0,0 +1,259 @@
2017-09-06T08:21:51Z Job state change UNDEFINED -> ACCEPTED Reason: (Re)Accepting new job
2017-09-06T08:21:51Z Job state change ACCEPTED -> PREPARING Reason: Starting job processing
2017-09-06T08:22:24Z Job state change PREPARING -> SUBMIT Reason: Pre-staging finished, passing job to LRMS
----- starting submit_slurm_job -----
SLURM jobname: accetuation_nn
SLURM job script built
SLURM script follows:
-------------------------------------------------------------------
#!/bin/bash -l
# SLURM batch job script built by grid-manager
#SBATCH --no-requeue
#SBATCH -e /net/hold/data1/arc/session//e1PMDmzlk9qnOeFSGmVnjcgoABFKDmABFKDmZmMKDmABFKDmnXN6sm.comment
#SBATCH -o /net/hold/data1/arc/session//e1PMDmzlk9qnOeFSGmVnjcgoABFKDmABFKDmZmMKDmABFKDmnXN6sm.comment
#SBATCH -p gridlong
#SBATCH --nice=50
#SBATCH -J 'accetuation_nn'
#SBATCH --get-user-env=10L
#SBATCH -n 1
#SBATCH --constraint=gpu --gres=gpu:1
#SBATCH -t 750:0
#SBATCH -t 750:0
#SBATCH --mem-per-cpu=12000
# run singularity image if RTE with singularity is required
if [ -z $SINGULARITY_CONTAINER ]; then
exec /bin/singularity exec -B /var/spool/slurm,/cvmfs,/net/hold/data1,/data1,/data1/slurm,/home,/usr/lib64/nvidia /net/hold/data1/singularity-images/theano-gpu-2.img $0
fi
# Overide umask of execution node (sometime values are really strange)
umask 077
# source with arguments for DASH shells
sourcewithargs() {
script=$1
shift
. $script
}
# Setting environment variables as specified by user
export 'GRID_GLOBAL_JOBID=gsiftp://nsc.ijs.si:2811/jobs/e1PMDmzlk9qnOeFSGmVnjcgoABFKDmABFKDmZmMKDmABFKDmnXN6sm'
RUNTIME_JOB_DIR=/net/hold/data1/arc/session//e1PMDmzlk9qnOeFSGmVnjcgoABFKDmABFKDmZmMKDmABFKDmnXN6sm
RUNTIME_JOB_STDIN=/dev/null
RUNTIME_JOB_STDOUT=/net/hold/data1/arc/session//e1PMDmzlk9qnOeFSGmVnjcgoABFKDmABFKDmZmMKDmABFKDmnXN6sm/out.txt
RUNTIME_JOB_STDERR=/net/hold/data1/arc/session//e1PMDmzlk9qnOeFSGmVnjcgoABFKDmABFKDmZmMKDmABFKDmnXN6sm/err.txt
RUNTIME_JOB_DIAG=/net/hold/data1/arc/session//e1PMDmzlk9qnOeFSGmVnjcgoABFKDmABFKDmZmMKDmABFKDmnXN6sm.diag
if [ ! -z "$RUNTIME_GRIDAREA_DIR" ] ; then
RUNTIME_JOB_DIR=$RUNTIME_GRIDAREA_DIR/`basename $RUNTIME_JOB_DIR`
RUNTIME_JOB_STDIN=`echo "$RUNTIME_JOB_STDIN" | sed "s#^$RUNTIME_JOB_DIR#$RUNTIME_GRIDAREA_DIR#"`
RUNTIME_JOB_STDOUT=`echo "$RUNTIME_JOB_STDOUT" | sed "s#^$RUNTIME_JOB_DIR#$RUNTIME_GRIDAREA_DIR#"`
RUNTIME_JOB_STDERR=`echo "$RUNTIME_JOB_STDERR" | sed "s#^$RUNTIME_JOB_DIR#$RUNTIME_GRIDAREA_DIR#"`
RUNTIME_JOB_DIAG=`echo "$RUNTIME_JOB_DIAG" | sed "s#^$RUNTIME_JOB_DIR#$RUNTIME_GRIDAREA_DIR#"`
RUNTIME_CONTROL_DIR=`echo "$RUNTIME_CONTROL_DIR" | sed "s#^$RUNTIME_JOB_DIR#$RUNTIME_GRIDAREA_DIR#"`
fi
RUNTIME_LOCAL_SCRATCH_DIR=${RUNTIME_LOCAL_SCRATCH_DIR:-$WORKDIR}
RUNTIME_FRONTEND_SEES_NODE=${RUNTIME_FRONTEND_SEES_NODE:-}
RUNTIME_NODE_SEES_FRONTEND=${RUNTIME_NODE_SEES_FRONTEND:-yes}
if [ ! -z "$RUNTIME_LOCAL_SCRATCH_DIR" ] && [ ! -z "$RUNTIME_NODE_SEES_FRONTEND" ]; then
RUNTIME_NODE_JOB_DIR="$RUNTIME_LOCAL_SCRATCH_DIR"/`basename "$RUNTIME_JOB_DIR"`
rm -rf "$RUNTIME_NODE_JOB_DIR"
mkdir -p "$RUNTIME_NODE_JOB_DIR"
# move directory contents
for f in "$RUNTIME_JOB_DIR"/.* "$RUNTIME_JOB_DIR"/*; do
[ "$f" = "$RUNTIME_JOB_DIR/*" ] && continue # glob failed, no files
[ "$f" = "$RUNTIME_JOB_DIR/." ] && continue
[ "$f" = "$RUNTIME_JOB_DIR/.." ] && continue
[ "$f" = "$RUNTIME_JOB_DIR/.diag" ] && continue
[ "$f" = "$RUNTIME_JOB_DIR/.comment" ] && continue
if ! mv "$f" "$RUNTIME_NODE_JOB_DIR"; then
echo "Failed to move '$f' to '$RUNTIME_NODE_JOB_DIR'" 1>&2
exit 1
fi
done
if [ ! -z "$RUNTIME_FRONTEND_SEES_NODE" ] ; then
# creating link for whole directory
ln -s "$RUNTIME_FRONTEND_SEES_NODE"/`basename "$RUNTIME_JOB_DIR"` "$RUNTIME_JOB_DIR"
else
# keep stdout, stderr and control directory on frontend
# recreate job directory
mkdir -p "$RUNTIME_JOB_DIR"
# make those files
mkdir -p `dirname "$RUNTIME_JOB_STDOUT"`
mkdir -p `dirname "$RUNTIME_JOB_STDERR"`
touch "$RUNTIME_JOB_STDOUT"
touch "$RUNTIME_JOB_STDERR"
RUNTIME_JOB_STDOUT__=`echo "$RUNTIME_JOB_STDOUT" | sed "s#^${RUNTIME_JOB_DIR}#${RUNTIME_NODE_JOB_DIR}#"`
RUNTIME_JOB_STDERR__=`echo "$RUNTIME_JOB_STDERR" | sed "s#^${RUNTIME_JOB_DIR}#${RUNTIME_NODE_JOB_DIR}#"`
rm "$RUNTIME_JOB_STDOUT__" 2>/dev/null
rm "$RUNTIME_JOB_STDERR__" 2>/dev/null
if [ ! -z "$RUNTIME_JOB_STDOUT__" ] && [ "$RUNTIME_JOB_STDOUT" != "$RUNTIME_JOB_STDOUT__" ]; then
ln -s "$RUNTIME_JOB_STDOUT" "$RUNTIME_JOB_STDOUT__"
fi
if [ "$RUNTIME_JOB_STDOUT__" != "$RUNTIME_JOB_STDERR__" ] ; then
if [ ! -z "$RUNTIME_JOB_STDERR__" ] && [ "$RUNTIME_JOB_STDERR" != "$RUNTIME_JOB_STDERR__" ]; then
ln -s "$RUNTIME_JOB_STDERR" "$RUNTIME_JOB_STDERR__"
fi
fi
if [ ! -z "$RUNTIME_CONTROL_DIR" ] ; then
# move control directory back to frontend
RUNTIME_CONTROL_DIR__=`echo "$RUNTIME_CONTROL_DIR" | sed "s#^${RUNTIME_JOB_DIR}#${RUNTIME_NODE_JOB_DIR}#"`
mv "$RUNTIME_CONTROL_DIR__" "$RUNTIME_CONTROL_DIR"
fi
fi
# adjust stdin,stdout & stderr pointers
RUNTIME_JOB_STDIN=`echo "$RUNTIME_JOB_STDIN" | sed "s#^${RUNTIME_JOB_DIR}#${RUNTIME_NODE_JOB_DIR}#"`
RUNTIME_JOB_STDOUT=`echo "$RUNTIME_JOB_STDOUT" | sed "s#^${RUNTIME_JOB_DIR}#${RUNTIME_NODE_JOB_DIR}#"`
RUNTIME_JOB_STDERR=`echo "$RUNTIME_JOB_STDERR" | sed "s#^${RUNTIME_JOB_DIR}#${RUNTIME_NODE_JOB_DIR}#"`
RUNTIME_FRONTEND_JOB_DIR="$RUNTIME_JOB_DIR"
RUNTIME_JOB_DIR="$RUNTIME_NODE_JOB_DIR"
fi
if [ -z "$RUNTIME_NODE_SEES_FRONTEND" ] ; then
mkdir -p "$RUNTIME_JOB_DIR"
fi
RESULT=0
if [ "$RESULT" = '0' ] ; then
# Running runtime scripts
export RUNTIME_CONFIG_DIR=${RUNTIME_CONFIG_DIR:-/net/hold/data1/arc/runtime/}
runtimeenvironments=
if [ ! -z "$RUNTIME_CONFIG_DIR" ] ; then
if [ -r "${RUNTIME_CONFIG_DIR}/APPS/BASE/THEANO-GPU-0.9" ] ; then
runtimeenvironments="${runtimeenvironments}APPS/BASE/THEANO-GPU-0.9;"
cmdl=${RUNTIME_CONFIG_DIR}/APPS/BASE/THEANO-GPU-0.9
sourcewithargs $cmdl 1
if [ $? -ne '0' ] ; then
echo "Runtime APPS/BASE/THEANO-GPU-0.9 script failed " 1>&2
echo "Runtime APPS/BASE/THEANO-GPU-0.9 script failed " 1>"$RUNTIME_JOB_DIAG"
exit 1
fi
fi
fi
echo "runtimeenvironments=$runtimeenvironments" >> "$RUNTIME_JOB_DIAG"
if [ ! "X$SLURM_NODEFILE" = 'X' ] ; then
if [ -r "$SLURM_NODEFILE" ] ; then
cat "$SLURM_NODEFILE" | sed 's/\(.*\)/nodename=\1/' >> "$RUNTIME_JOB_DIAG"
NODENAME_WRITTEN="1"
else
SLURM_NODEFILE=
fi
fi
if [ "$RESULT" = '0' ] ; then
# Changing to session directory
HOME=$RUNTIME_JOB_DIR
export HOME
if ! cd "$RUNTIME_JOB_DIR"; then
echo "Failed to switch to '$RUNTIME_JOB_DIR'" 1>&2
RESULT=1
fi
if [ ! -z "$RESULT" ] && [ "$RESULT" != 0 ]; then
exit $RESULT
fi
nodename=`/bin/hostname -f`
echo "nodename=$nodename" >> "$RUNTIME_JOB_DIAG"
echo "Processors=1" >> "$RUNTIME_JOB_DIAG"
executable='./workbench.sh'
# Check if executable exists
if [ ! -f "$executable" ];
then
echo "Path \"$executable\" does not seem to exist" 1>$RUNTIME_JOB_STDOUT 2>$RUNTIME_JOB_STDERR 1>&2
exit 1
fi
# See if executable is a script, and extract the name of the interpreter
line1=`dd if="$executable" count=1 2>/dev/null | head -n 1`
command=`echo $line1 | sed -n 's/^#! *//p'`
interpreter=`echo $command | awk '{print $1}'`
if [ "$interpreter" = /usr/bin/env ]; then interpreter=`echo $command | awk '{print $2}'`; fi
# If it's a script and the interpreter is not found ...
[ "x$interpreter" = x ] || type "$interpreter" > /dev/null 2>&1 || {
echo "Cannot run $executable: $interpreter: not found" 1>$RUNTIME_JOB_STDOUT 2>$RUNTIME_JOB_STDERR 1>&2
exit 1; }
GNU_TIME='/usr/bin/time'
if [ ! -z "$GNU_TIME" ] && ! "$GNU_TIME" --version >/dev/null 2>&1; then
echo "WARNING: GNU time not found at: $GNU_TIME" 2>&1;
GNU_TIME=
fi
if [ -z "$GNU_TIME" ] ; then
"./workbench.sh" <$RUNTIME_JOB_STDIN 1>$RUNTIME_JOB_STDOUT 2>$RUNTIME_JOB_STDERR
else
$GNU_TIME -o "$RUNTIME_JOB_DIAG" -a -f 'WallTime=%es\nKernelTime=%Ss\nUserTime=%Us\nCPUUsage=%P\nMaxResidentMemory=%MkB\nAverageResidentMemory=%tkB\nAverageTotalMemory=%KkB\nAverageUnsharedMemory=%DkB\nAverageUnsharedStack=%pkB\nAverageSharedMemory=%XkB\nPageSize=%ZB\nMajorPageFaults=%F\nMinorPageFaults=%R\nSwaps=%W\nForcedSwitches=%c\nWaitSwitches=%w\nInputs=%I\nOutputs=%O\nSocketReceived=%r\nSocketSent=%s\nSignals=%k\n' "./workbench.sh" <$RUNTIME_JOB_STDIN 1>$RUNTIME_JOB_STDOUT 2>$RUNTIME_JOB_STDERR
fi
RESULT=$?
fi
fi
if [ ! -z "$RUNTIME_CONFIG_DIR" ] ; then
if [ -r "${RUNTIME_CONFIG_DIR}/APPS/BASE/THEANO-GPU-0.9" ] ; then
cmdl=${RUNTIME_CONFIG_DIR}/APPS/BASE/THEANO-GPU-0.9
sourcewithargs $cmdl 2
fi
fi
if [ ! -z "$RUNTIME_LOCAL_SCRATCH_DIR" ] ; then
find ./ -type l -exec rm -f "{}" ";"
find ./ -type f -exec chmod u+w "{}" ";"
chmod -R u-w "$RUNTIME_JOB_DIR"/'40_epoch.h5' 2>/dev/null
chmod -R u-w "$RUNTIME_JOB_DIR"/'workbench.py' 2>/dev/null
chmod -R u-w "$RUNTIME_JOB_DIR"/'workbench.sh' 2>/dev/null
chmod -R u-w "$RUNTIME_JOB_DIR"/'40_epoch_history.pkl' 2>/dev/null
chmod -R u-w "$RUNTIME_JOB_DIR"/'notes' 2>/dev/null
chmod -R u-w "$RUNTIME_JOB_DIR"/'out.txt' 2>/dev/null
chmod -R u-w "$RUNTIME_JOB_DIR"/'err.txt' 2>/dev/null
chmod -R u-w "$RUNTIME_JOB_DIR"/'gmlog' 2>/dev/null
find ./ -type f -perm /200 -exec rm -f "{}" ";"
find ./ -type f -exec chmod u+w "{}" ";"
fi
if [ ! -z "$RUNTIME_LOCAL_SCRATCH_DIR" ] && [ ! -z "$RUNTIME_NODE_SEES_FRONTEND" ]; then
if [ ! -z "$RUNTIME_FRONTEND_SEES_NODE" ] ; then
# just move it
rm -rf "$RUNTIME_FRONTEND_JOB_DIR"
destdir=`dirname "$RUNTIME_FRONTEND_JOB_DIR"`
if ! mv "$RUNTIME_NODE_JOB_DIR" "$destdir"; then
echo "Failed to move '$RUNTIME_NODE_JOB_DIR' to '$destdir'" 1>&2
RESULT=1
fi
else
# remove links
rm -f "$RUNTIME_JOB_STDOUT" 2>/dev/null
rm -f "$RUNTIME_JOB_STDERR" 2>/dev/null
# move directory contents
for f in "$RUNTIME_NODE_JOB_DIR"/.* "$RUNTIME_NODE_JOB_DIR"/*; do
[ "$f" = "$RUNTIME_NODE_JOB_DIR/*" ] && continue # glob failed, no files
[ "$f" = "$RUNTIME_NODE_JOB_DIR/." ] && continue
[ "$f" = "$RUNTIME_NODE_JOB_DIR/.." ] && continue
[ "$f" = "$RUNTIME_NODE_JOB_DIR/.diag" ] && continue
[ "$f" = "$RUNTIME_NODE_JOB_DIR/.comment" ] && continue
if ! mv "$f" "$RUNTIME_FRONTEND_JOB_DIR"; then
echo "Failed to move '$f' to '$RUNTIME_FRONTEND_JOB_DIR'" 1>&2
RESULT=1
fi
done
rm -rf "$RUNTIME_NODE_JOB_DIR"
fi
fi
echo "exitcode=$RESULT" >> "$RUNTIME_JOB_DIAG"
exit $RESULT
-------------------------------------------------------------------
job submitted successfully!
local job id: 734035
----- exiting submit_slurm_job -----
2017-09-06T08:22:24Z Job state change SUBMIT -> INLRMS Reason: Job is passed to LRMS
------- Contents of output stream forwarded by the LRMS ---------
WARNING: GNU time not found at: /usr/bin/time
slurmstepd: error: *** JOB 734035 ON nsc-fp006 CANCELLED AT 2017-09-06T22:52:34 DUE TO TIME LIMIT ***
------------------------- End of output -------------------------
2017-09-06T20:55:12Z Job state change INLRMS -> FINISHING Reason: Job finished executing in LRMS
2017-09-06T20:56:12Z Job state change FINISHING -> FINISHED Reason: Stage-out finished.
2017-09-06T21:49:02Z Job state change UNDEFINED -> FINISHED Reason: (Re)Accepting new job
2017-09-06T23:49:20Z Job state change UNDEFINED -> FINISHED Reason: (Re)Accepting new job
2017-09-07T01:49:53Z Job state change UNDEFINED -> FINISHED Reason: (Re)Accepting new job
2017-09-07T03:50:36Z Job state change UNDEFINED -> FINISHED Reason: (Re)Accepting new job
2017-09-07T05:50:39Z Job state change UNDEFINED -> FINISHED Reason: (Re)Accepting new job
2017-09-07T07:50:54Z Job state change UNDEFINED -> FINISHED Reason: (Re)Accepting new job

@ -0,0 +1,39 @@
joboption_directory='/net/hold/data1/arc/session//e1PMDmzlk9qnOeFSGmVnjcgoABFKDmABFKDmZmMKDmABFKDmnXN6sm'
joboption_controldir='/var/spool/arc/jobstatus/'
joboption_arg_0='./workbench.sh'
joboption_stdin='/dev/null'
joboption_stdout='/net/hold/data1/arc/session//e1PMDmzlk9qnOeFSGmVnjcgoABFKDmABFKDmZmMKDmABFKDmnXN6sm/out.txt'
joboption_stderr='/net/hold/data1/arc/session//e1PMDmzlk9qnOeFSGmVnjcgoABFKDmABFKDmZmMKDmABFKDmnXN6sm/err.txt'
joboption_env_0=GRID_GLOBAL_JOBID='gsiftp://nsc.ijs.si:2811/jobs/e1PMDmzlk9qnOeFSGmVnjcgoABFKDmABFKDmZmMKDmABFKDmnXN6sm'
joboption_cputime=45000
joboption_walltime=45000
joboption_memory=12000
joboption_virtualmemory=
joboption_disk=
joboption_count=1
joboption_runtime_0='APPS/BASE/THEANO-GPU-0.9'
joboption_jobname='accetuation_nn'
joboption_queue='gridlong'
joboption_starttime=
joboption_gridid='e1PMDmzlk9qnOeFSGmVnjcgoABFKDmABFKDmZmMKDmABFKDmnXN6sm'
joboption_priority=50
joboption_inputfile_0='/workbench.py'
joboption_inputfile_1='/prepare_data.py'
joboption_inputfile_2='/content_shuffle_vector.h5'
joboption_inputfile_3='/shuffle_vector_test.h5'
joboption_inputfile_4='/shuffle_vector_train.h5'
joboption_inputfile_5='/shuffle_vector_validate.h5'
joboption_inputfile_6='/SlovarIJS_BESEDE_utf8.lex'
joboption_inputfile_7='/40_epoch.h5'
joboption_inputfile_8='/40_epoch_history.pkl'
joboption_inputfile_9='/notes'
joboption_inputfile_10='/workbench.sh'
joboption_outputfile_0='/40_epoch.h5'
joboption_outputfile_1='/workbench.py'
joboption_outputfile_2='/workbench.sh'
joboption_outputfile_3='/40_epoch_history.pkl'
joboption_outputfile_4='/notes'
joboption_outputfile_5='/out.txt'
joboption_outputfile_6='/err.txt'
joboption_outputfile_7='/gmlog'
joboption_jobid=734035

@ -0,0 +1,26 @@
globalid=gsiftp://nsc.ijs.si:2811/jobs/e1PMDmzlk9qnOeFSGmVnjcgoABFKDmABFKDmZmMKDmABFKDmnXN6sm
headnode=gsiftp://nsc.ijs.si:2811/jobs
interface=org.nordugrid.gridftpjob
lrms=SLURM
queue=gridlong
localid=734035
args=workbench.sh
argscode=0
subject=/C=SI/O=SiGNET/O=FRI Uni-Lj/CN=Luka Krsnik
starttime=20170906082151Z
lifetime=604800
rerun=0
downloads=0
uploads=0
jobname=accetuation_nn
gmlog=gmlog
cleanuptime=20170913205612Z
delegexpiretime=20170906200810Z
clientname=88.200.99.117:59172
delegationid=b01f4f77dbc8
sessiondir=/net/hold/data1/arc/session//e1PMDmzlk9qnOeFSGmVnjcgoABFKDmABFKDmZmMKDmABFKDmnXN6sm
diskspace=0
freestagein=no
voms=/gen.vo.sling.si
transfershare=_default
priority=50

@ -0,0 +1,8 @@
/40_epoch.h5
/workbench.py
/workbench.sh
/40_epoch_history.pkl
/notes
/out.txt
/err.txt
/gmlog

@ -0,0 +1,33 @@
<ComputingActivity xmlns="http://schemas.ogf.org/glue/2009/03/spec_2.0_r1" BaseType="Activity" CreationTime="2017-09-07T07:51:22Z" Validity="10800">
<ID>urn:caid:nsc.ijs.si:org.nordugrid.gridftpjob:e1PMDmzlk9qnOeFSGmVnjcgoABFKDmABFKDmZmMKDmABFKDmnXN6sm</ID>
<Name>accetuation_nn</Name>
<OtherInfo>SubmittedVia=org.nordugrid.gridftpjob</OtherInfo>
<Type>single</Type>
<IDFromEndpoint>urn:idfe:e1PMDmzlk9qnOeFSGmVnjcgoABFKDmABFKDmZmMKDmABFKDmnXN6sm</IDFromEndpoint>
<JobDescription>nordugrid:xrsl</JobDescription>
<State>nordugrid:FINISHED</State>
<State>bes:Finished</State>
<State>emies:terminal</State>
<State>emiesattr:client-stageout-possible</State>
<Owner>/C=SI/O=SiGNET/O=FRI Uni-Lj/CN=Luka Krsnik</Owner>
<LocalOwner>gen0006</LocalOwner>
<RequestedTotalWallTime>45000</RequestedTotalWallTime>
<RequestedTotalCPUTime>45000</RequestedTotalCPUTime>
<RequestedSlots>1</RequestedSlots>
<RequestedApplicationEnvironment>APPS/BASE/THEANO-GPU-0.9</RequestedApplicationEnvironment>
<StdIn>/dev/null</StdIn>
<StdOut>out.txt</StdOut>
<StdErr>err.txt</StdErr>
<LogDir>gmlog</LogDir>
<ExecutionNode>nsc-fp006.ijs.si</ExecutionNode>
<Queue>gridlong</Queue>
<UsedTotalWallTime>45010</UsedTotalWallTime>
<SubmissionTime>2017-09-06T08:21:51Z</SubmissionTime>
<EndTime>2017-09-07T07:50:54Z</EndTime>
<WorkingAreaEraseTime>2017-09-13T20:56:12Z</WorkingAreaEraseTime>
<ProxyExpirationTime>2017-09-06T20:08:10Z</ProxyExpirationTime>
<SubmissionHost>88.200.99.117</SubmissionHost>
<Associations>
<ComputingShareID>urn:ogf:ComputingShare:nsc.ijs.si:gridlong</ComputingShareID>
</Associations>
</ComputingActivity>

@ -801,7 +801,7 @@ class Data:
if word[i] == 1:
final_word.append(feature_dictionary[z][j][k])
i += 1
print(u''.join(final_word))
# print(u''.join(final_word))
return u''.join(final_word)
@staticmethod
@ -814,7 +814,188 @@ class Data:
i += 1
return res
def test_accuracy(self, predictions, x, x_other_features, y, dictionary, feature_dictionary, vowels, syllable_dictionary=None):
errors = []
num_of_pred = len(predictions)
num_of_correct_pred = 0
for i in range(predictions.shape[0]):
if (np.around(predictions[i]) == y[i]).all():
num_of_correct_pred += 1
else:
if self._input_type == 'l':
decoded_x = self.decode_x(x[i], dictionary)
else:
decoded_x = self.decode_syllable_x(x[i], syllable_dictionary)
errors.append([i,
decoded_x,
self.decode_x_other_features(feature_dictionary, [x_other_features[i]]),
self.assign_stress_locations(decoded_x, np.around(predictions[i]), vowels, syllables=self._input_type != 'l'),
self.assign_stress_locations(decoded_x, y[i], vowels, syllables=self._input_type != 'l')
])
return (num_of_correct_pred / float(num_of_pred)) * 100, errors
@staticmethod
def decode_syllable_x(word_encoded, syllable_dictionary):
word = []
for i in range(len(word_encoded)):
word.append(syllable_dictionary[word_encoded[i]])
return ''.join(word[::-1])
def assign_stress_locations(self, word, y, vowels, syllables=False):
if not syllables:
word_list = list(word)
else:
word_list = list(word)[::-1]
vowel_num = 0
for i in range(len(word_list)):
if self._is_vowel(word_list, i, vowels):
if word_list[i] == 'a' and y[vowel_num] == 1:
word_list[i] = 'á'
elif word_list[i] == 'e' and y[vowel_num] == 1:
word_list[i] = 'é'
elif word_list[i] == 'i' and y[vowel_num] == 1:
word_list[i] = 'í'
elif word_list[i] == 'o' and y[vowel_num] == 1:
word_list[i] = 'ó'
elif word_list[i] == 'u' and y[vowel_num] == 1:
word_list[i] = 'ú'
elif word_list[i] == 'r' and y[vowel_num] == 1:
word_list[i] = 'ŕ'
elif word_list[i] == 'A' and y[vowel_num] == 1:
word_list[i] = 'Á'
elif word_list[i] == 'E' and y[vowel_num] == 1:
word_list[i] = 'É'
elif word_list[i] == 'I' and y[vowel_num] == 1:
word_list[i] = 'Í'
elif word_list[i] == 'O' and y[vowel_num] == 1:
word_list[i] = 'Ó'
elif word_list[i] == 'U' and y[vowel_num] == 1:
word_list[i] = 'Ú'
elif word_list[i] == 'R' and y[vowel_num] == 1:
word_list[i] = 'Ŕ'
vowel_num += 1
if not syllables:
return ''.join(word_list)
else:
return ''.join(word_list[::-1])
def test_type_accuracy(self, predictions, x, x_other_features, y, dictionary, feature_dictionary, vowels, accented_vowels,
syllable_dictionary=None):
errors = []
num_of_pred = len(predictions)
num_of_correct_pred = 0
num_of_correct_pred_words = 0
accentuation_index = 0
eye = np.eye(len(accented_vowels), dtype=int)
for i in range(len(y)):
correct_prediction = True
if self._input_type == 'l':
decoded_x = self.decode_x(x[i], dictionary)
else:
decoded_x = self.decode_syllable_x(x[i], syllable_dictionary)
wrong_word = decoded_x
correct_word = decoded_x
for j in range(len(y[i])):
if y[i][j] > 0:
# ERROR AS IT IS CALCULATED
# arounded_predictions = np.around(predictions[accentuation_index]).astype(int)
# MAX ELEMENT ONLY
# arounded_predictions = np.zeros(len(predictions[accentuation_index]))
# arounded_predictions[np.argmax(predictions[accentuation_index]).astype(int)] = 1
# MAX ELEMENT AMONGT POSSIBLE ONES
# if i == 313:
# print(decoded_x)
stressed_letter = self.get_accentuated_letter(decoded_x, j, vowels, syllables=self._input_type != 'l')
possible_places = np.zeros(len(predictions[accentuation_index]))
if stressed_letter == 'r':
possible_places[0] = 1
elif stressed_letter == 'a':
possible_places[1] = 1
possible_places[2] = 1
elif stressed_letter == 'e':
possible_places[3] = 1
possible_places[4] = 1
possible_places[5] = 1
elif stressed_letter == 'i':
possible_places[6] = 1
possible_places[7] = 1
elif stressed_letter == 'o':
possible_places[8] = 1
possible_places[9] = 1
possible_places[10] = 1
elif stressed_letter == 'u':
possible_places[11] = 1
possible_places[12] = 1
possible_predictions = predictions[accentuation_index] * possible_places
arounded_predictions = np.zeros(len(predictions[accentuation_index]), dtype=int)
arounded_predictions[np.argmax(possible_predictions).astype(int)] = 1
wrong_word = self.assign_word_accentuation_type(wrong_word, j, arounded_predictions, vowels, accented_vowels,
syllables=self._input_type != 'l', debug=i == 313)
correct_word = self.assign_word_accentuation_type(correct_word, j, eye[int(y[i][j])], vowels, accented_vowels,
syllables=self._input_type != 'l', debug=i == 313)
if (eye[int(y[i][j])] == arounded_predictions).all():
num_of_correct_pred += 1
else:
correct_prediction = False
accentuation_index += 1
if correct_prediction:
num_of_correct_pred_words += 1
else:
if self._input_type == 'l':
errors.append([i,
decoded_x[::-1],
self.decode_x_other_features(feature_dictionary, [x_other_features[i]]),
wrong_word[::-1],
correct_word[::-1]
])
else:
errors.append([i,
decoded_x,
self.decode_x_other_features(feature_dictionary, [x_other_features[i]]),
wrong_word,
correct_word
])
return (num_of_correct_pred / float(num_of_pred)) * 100, (num_of_correct_pred_words / float(len(y))) * 100, errors
def get_accentuated_letter(self, word, location, vowels, syllables=False, debug=False):
# print(location)
vowel_index = 0
word_list = list(word)
if not syllables:
word_list = list(word)
else:
word_list = list(word[::-1])
for i in range(len(word_list)):
if self._is_vowel(word_list, i, vowels):
if location == vowel_index:
return word_list[i]
vowel_index += 1
def assign_word_accentuation_type(self, word, location, y, vowels, accented_vowels, syllables=False, debug=False):
vowel_index = 0
if not syllables:
word_list = list(word)
else:
word_list = list(word[::-1])
for i in range(len(word_list)):
if self._is_vowel(word_list, i, vowels):
if location == vowel_index:
if len(np.where(y == 1)[0]) == 1:
word_list[i] = accented_vowels[np.where(y == 1)[0][0]]
vowel_index += 1
if not syllables:
return ''.join(word_list)
else:
return ''.join(word_list[::-1])
# def count_vowels(content, vowels):
# num_all_vowels = 0

@ -31,30 +31,40 @@ from prepare_data import *
# data = Data('l', save_generated_data=False, number_of_syllables=True)
# syllabled letters
data = Data('l', save_generated_data=False, accent_classification=True)
data = Data('s', save_generated_data=False, accent_classification=True)
data.generate_data('letters_word_accetuation_train',
'letters_word_accetuation_test',
'letters_word_accetuation_validate', content_name='SlovarIJS_BESEDE_utf8.lex',
content_shuffle_vector='content_shuffle_vector', shuffle_vector='shuffle_vector',
inputs_location='', content_location='')
# concatenate test and train data
# data.x_train = np.concatenate((data.x_train, data.x_test), axis=0)
# data.x_other_features_train = np.concatenate((data.x_other_features_train, data.x_other_features_test), axis=0)
# data.y_train = np.concatenate((data.y_train, data.y_test), axis=0)
# concatenate all data
data.x_train = np.concatenate((data.x_train, data.x_test, data.x_validate), axis=0)
data.x_other_features_train = np.concatenate((data.x_other_features_train, data.x_other_features_test, data.x_other_features_validate), axis=0)
data.y_train = np.concatenate((data.y_train, data.y_test, data.y_validate), axis=0)
num_examples = len(data.x_train) # training set size
nn_output_dim = 13
nn_hdim = 516
batch_size = 16
# actual_epoch = 1
actual_epoch = 40
actual_epoch = 20
# num_fake_epoch = 2
num_fake_epoch = 20
# letters
conv_input_shape=(23, 36)
# conv_input_shape=(23, 36)
# syllabled letters
# conv_input_shape=(10, 5168)
# conv_input_shape=(10, 252)
# syllables
conv_input_shape=(10, 5168)
# othr_input = (140, )
@ -62,11 +72,11 @@ othr_input = (150, )
conv_input = Input(shape=conv_input_shape, name='conv_input')
# letters
x_conv = Conv1D(115, (3), padding='same', activation='relu')(conv_input)
x_conv = Conv1D(46, (3), padding='same', activation='relu')(x_conv)
# x_conv = Conv1D(115, (3), padding='same', activation='relu')(conv_input)
# x_conv = Conv1D(46, (3), padding='same', activation='relu')(x_conv)
# syllabled letters
# x_conv = Conv1D(200, (2), padding='same', activation='relu')(conv_input)
x_conv = Conv1D(200, (2), padding='same', activation='relu')(conv_input)
x_conv = MaxPooling1D(pool_size=2)(x_conv)
x_conv = Flatten()(x_conv)
@ -76,9 +86,9 @@ x = concatenate([x_conv, othr_input])
# x = Dense(1024, input_dim=(516 + 256), activation='relu')(x)
x = Dense(256, activation='relu')(x)
x = Dropout(0.3)(x)
x = Dense(512, activation='relu')(x)
x = Dense(256, activation='relu')(x)
x = Dropout(0.3)(x)
x = Dense(512, activation='relu')(x)
x = Dense(256, activation='relu')(x)
x = Dropout(0.3)(x)
x = Dense(nn_output_dim, activation='sigmoid')(x)
@ -94,8 +104,6 @@ model.compile(loss='binary_crossentropy', optimizer=opt, metrics=[actual_accurac
history = model.fit_generator(data.generator('train', batch_size, content_name='SlovarIJS_BESEDE_utf8.lex', content_location=''),
data.x_train.shape[0]/(batch_size * num_fake_epoch),
epochs=actual_epoch*num_fake_epoch,
validation_data=data.generator('test', batch_size, content_name='SlovarIJS_BESEDE_utf8.lex', content_location=''),
validation_steps=data.x_test.shape[0]/(batch_size * num_fake_epoch),
verbose=2
)

Loading…
Cancel
Save