lkrsnik/list
1
0
Fork 0
Code Issues 1 Pull Requests Releases Wiki Activity

Added some performance measures

Browse Source
This commit is contained in:
lkrsnik
2018-08-09 09:41:21 +02:00
parent 179f09c4bd
commit 9b5fa4616b
24 changed files with 734 additions and 379 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/test/java/Common.java
+38 -38
View File
@@ -19,66 +19,66 @@ public class Common {
ArrayList<String> taxonomy = new ArrayList<>();
taxonomy.add("#Ft.Z.N.N");
List<Word> words = new ArrayList<>();
words.add(new Word("ker", "ker", "Vd", taxonomy));
words.add(new Word("ima", "imeti", "Ggnste-n", taxonomy));
words.add(new Word("junak", "junak", "Somei", taxonomy));
words.add(new Word("v", "v", "Dm", taxonomy));
words.add(new Word("posesti", "posest", "Sozem", taxonomy));
words.add(new Word("nekaj", "nekaj", "Rsn", taxonomy));
words.add(new Word("o", "o", "Dm", taxonomy));
words.add(new Word("čemer", "kar", "Zz-sem", taxonomy));
words.add(new Word("se", "se", "Zp------k", taxonomy));
words.add(new Word("mu", "on", "Zotmed--k", taxonomy));
words.add(new Word("ne", "ne", "L", taxonomy));
words.add(new Word("sanja", "sanjati", "Ggnste", taxonomy));
words.add(new Word("a", "a", "Vp", taxonomy));
words.add(new Word("se", "se", "Zp------k", taxonomy));
words.add(new Word("onemu", "oni", "Zk-sed", taxonomy));
words.add(new Word("zdi", "zdeti", "Ggnste", taxonomy));
words.add(new Word("ključno", "ključen", "Ppnsei", taxonomy));
words.add(new Word("pri", "pri", "Dm", taxonomy));
words.add(new Word("operaciji", "operacija", "Sozem", taxonomy));
words.add(new Word("666", "666", "Kag", taxonomy));
words.add(new Word("ker", "ker", "Vd"));
words.add(new Word("ima", "imeti", "Ggnste-n"));
words.add(new Word("junak", "junak", "Somei"));
words.add(new Word("v", "v", "Dm"));
words.add(new Word("posesti", "posest", "Sozem"));
words.add(new Word("nekaj", "nekaj", "Rsn"));
words.add(new Word("o", "o", "Dm"));
words.add(new Word("čemer", "kar", "Zz-sem"));
words.add(new Word("se", "se", "Zp------k"));
words.add(new Word("mu", "on", "Zotmed--k"));
words.add(new Word("ne", "ne", "L"));
words.add(new Word("sanja", "sanjati", "Ggnste"));
words.add(new Word("a", "a", "Vp"));
words.add(new Word("se", "se", "Zp------k"));
words.add(new Word("onemu", "oni", "Zk-sed"));
words.add(new Word("zdi", "zdeti", "Ggnste"));
words.add(new Word("ključno", "ključen", "Ppnsei"));
words.add(new Word("pri", "pri", "Dm"));
words.add(new Word("operaciji", "operacija", "Sozem"));
words.add(new Word("666", "666", "Kag"));
testSentence = new Sentence(words, "#Ft.Z.N.N");
testSentence = new Sentence(words, taxonomy);
corpus = new ArrayList<>();
corpus.add(testSentence);
// three word sentence
testSentence = new Sentence(corpus.get(0).getSublist(0, 3), "#Ft.Z.N.N");
testSentence = new Sentence(corpus.get(0).getSublist(0, 3), taxonomy);
minCorpus = new ArrayList<>();
minCorpus.add(testSentence);
// five word sentence
words = new ArrayList<>();
words.add(new Word("ker", "ker", "Vd", taxonomy));
words.add(new Word("ima", "imeti", "Ggnste-n", taxonomy));
words.add(new Word("junak", "junak", "Somei", taxonomy));
words.add(new Word("ima", "imeti", "Ggnste-n", taxonomy));
words.add(new Word("posesti", "posest", "Sozem", taxonomy));
testSentence = new Sentence(words, "#Ft.Z.N.N");
words.add(new Word("ker", "ker", "Vd"));
words.add(new Word("ima", "imeti", "Ggnste-n"));
words.add(new Word("junak", "junak", "Somei"));
words.add(new Word("ima", "imeti", "Ggnste-n"));
words.add(new Word("posesti", "posest", "Sozem"));
testSentence = new Sentence(words, taxonomy);
midCorpus = new ArrayList<>();
midCorpus.add(testSentence);
// five word sentence - for skipgrams
words = new ArrayList<>();
words.add(new Word("ker", "ker", "Vd", taxonomy));
words.add(new Word("ima", "imeti", "Ggnste-n", taxonomy));
words.add(new Word("junak", "junak", "Somei", taxonomy));
words.add(new Word("v", "v", "Dm", taxonomy));
words.add(new Word("posesti", "posest", "Sozem", taxonomy));
testSentence = new Sentence(words, "#Ft.Z.N.N");
words.add(new Word("ker", "ker", "Vd"));
words.add(new Word("ima", "imeti", "Ggnste-n"));
words.add(new Word("junak", "junak", "Somei"));
words.add(new Word("v", "v", "Dm"));
words.add(new Word("posesti", "posest", "Sozem"));
testSentence = new Sentence(words, taxonomy);
midCorpusSkip = new ArrayList<>();
midCorpusSkip.add(testSentence);
// JOS test
words = new ArrayList<>();
words.add(new Word("junak", "junak", "Somei", taxonomy));
words.add(new Word("ima", "imeti", "Ggnste-n", taxonomy));
words.add(new Word("posesti", "posest", "Sozem", taxonomy));
testSentence = new Sentence(words, "#Ft.Z.N.N");
words.add(new Word("junak", "junak", "Somei"));
words.add(new Word("ima", "imeti", "Ggnste-n"));
words.add(new Word("posesti", "posest", "Sozem"));
testSentence = new Sentence(words, taxonomy);
josTest = new ArrayList<>();
josTest.add(testSentence);
src/test/java/NgramTests.java
+15 -15
View File
@@ -140,9 +140,9 @@ public class NgramTests {
taxonomyResult = stats.getTaxonomyResult();
assertEquals(3, taxonomyResult.get("Total").size());
assertTrue(taxonomyResult.get("Total").containsKey(new MultipleHMKeys("ker ima junak", "", "", "")));
assertTrue(taxonomyResult.get("Total").containsKey(new MultipleHMKeys("ima junak ima", "", "", "")));
assertTrue(taxonomyResult.get("Total").containsKey(new MultipleHMKeys("junak ima posesti", "", "", "")));
assertTrue(taxonomyResult.get("Total").containsKey(new MultipleHMKeys1("ker ima junak")));
assertTrue(taxonomyResult.get("Total").containsKey(new MultipleHMKeys1("ima junak ima")));
assertTrue(taxonomyResult.get("Total").containsKey(new MultipleHMKeys1("junak ima posesti")));
// tests:
// - normal ngrams - lemmas
@@ -152,9 +152,9 @@ public class NgramTests {
taxonomyResult = stats.getTaxonomyResult();
assertEquals(3, taxonomyResult.get("Total").size());
assertTrue(taxonomyResult.get("Total").containsKey(new MultipleHMKeys("ker imeti junak", "", "", "")));
assertTrue(taxonomyResult.get("Total").containsKey(new MultipleHMKeys("imeti junak imeti", "", "", "")));
assertTrue(taxonomyResult.get("Total").containsKey(new MultipleHMKeys("junak imeti posest", "", "", "")));
assertTrue(taxonomyResult.get("Total").containsKey(new MultipleHMKeys1("ker imeti junak")));
assertTrue(taxonomyResult.get("Total").containsKey(new MultipleHMKeys1("imeti junak imeti")));
assertTrue(taxonomyResult.get("Total").containsKey(new MultipleHMKeys1("junak imeti posest")));
// tests:
// - normal ngrams - msd
@@ -164,9 +164,9 @@ public class NgramTests {
taxonomyResult = stats.getTaxonomyResult();
assertEquals(3, taxonomyResult.get("Total").size());
assertTrue(taxonomyResult.get("Total").containsKey(new MultipleHMKeys("Vd Ggnste-n Somei", "", "", "")));
assertTrue(taxonomyResult.get("Total").containsKey(new MultipleHMKeys("Ggnste-n Somei Ggnste-n", "", "", "")));
assertTrue(taxonomyResult.get("Total").containsKey(new MultipleHMKeys("Somei Ggnste-n Sozem", "", "", "")));
assertTrue(taxonomyResult.get("Total").containsKey(new MultipleHMKeys1("Vd Ggnste-n Somei")));
assertTrue(taxonomyResult.get("Total").containsKey(new MultipleHMKeys1("Ggnste-n Somei Ggnste-n")));
assertTrue(taxonomyResult.get("Total").containsKey(new MultipleHMKeys1("Somei Ggnste-n Sozem")));
// tests:
// - ngrams - word - regex filter
@@ -182,7 +182,7 @@ public class NgramTests {
taxonomyResult = stats.getTaxonomyResult();
assertEquals(1, taxonomyResult.get("Total").size());
assertTrue(taxonomyResult.get("Total").containsKey(new MultipleHMKeys("junak ima posesti", "", "", "")));
assertTrue(taxonomyResult.get("Total").containsKey(new MultipleHMKeys1("junak ima posesti")));
// tests:
// - ngrams - word - regex filter
@@ -198,7 +198,7 @@ public class NgramTests {
taxonomyResult = stats.getTaxonomyResult();
assertEquals(1, taxonomyResult.get("Total").size());
assertTrue(taxonomyResult.get("Total").containsKey(new MultipleHMKeys("ima junak", "", "", "")));
assertTrue(taxonomyResult.get("Total").containsKey(new MultipleHMKeys1("ima junak")));
}
@@ -316,7 +316,7 @@ public class NgramTests {
Set<String> bigrams = new HashSet<>(Arrays.asList("ker ima", "ima junak", "junak v", "v posesti"));
Set<MultipleHMKeys> bigramsMultipleHMKeys = taxonomyResult.get("Total").keySet();
Set<String> bigramsActual = new HashSet<>(bigramsMultipleHMKeys.stream().map(MultipleHMKeys::getKey).collect(Collectors.toList()));
Set<String> bigramsActual = new HashSet<>(bigramsMultipleHMKeys.stream().map(MultipleHMKeys::getK1).collect(Collectors.toList()));
assertEquals(bigrams, bigramsActual);
// test:
@@ -329,7 +329,7 @@ public class NgramTests {
Set<String> twoSkipBigrams = new HashSet<>(Arrays.asList("ker ima", "ker junak", "ker v", "ima junak", "ima v", "ima posesti", "junak v", "junak posesti", "v posesti"));
Set<MultipleHMKeys> twoSkipBigramsMultipleHMKeys = taxonomyResult.get("Total").keySet();
Set<String> twoSkipBigramsActual = new HashSet<>(twoSkipBigramsMultipleHMKeys.stream().map(MultipleHMKeys::getKey).collect(Collectors.toList()));
Set<String> twoSkipBigramsActual = new HashSet<>(twoSkipBigramsMultipleHMKeys.stream().map(MultipleHMKeys::getK1).collect(Collectors.toList()));
assertEquals(twoSkipBigrams, twoSkipBigramsActual);
@@ -342,7 +342,7 @@ public class NgramTests {
taxonomyResult = stats.getTaxonomyResult();
Set<String> trigrams = new HashSet<>(Arrays.asList("ker ima junak", "ima junak v", "junak v posesti"));
Set<MultipleHMKeys> trigramsMultipleHMKeys = taxonomyResult.get("Total").keySet();
Set<String> trigramsActual = new HashSet<>(trigramsMultipleHMKeys.stream().map(MultipleHMKeys::getKey).collect(Collectors.toList()));
Set<String> trigramsActual = new HashSet<>(trigramsMultipleHMKeys.stream().map(MultipleHMKeys::getK1).collect(Collectors.toList()));
assertEquals(trigrams, trigramsActual);
@@ -355,7 +355,7 @@ public class NgramTests {
taxonomyResult = stats.getTaxonomyResult();
HashSet<String> twoSkipTrigrams = new HashSet<>(Arrays.asList("ker ima junak", "ker ima v", "ker ima posesti", "ker junak v", "ker junak posesti", "ker v posesti", "ima junak v", "ima junak posesti", "ima v posesti", "junak v posesti"));
Set<MultipleHMKeys> twoSkipTrigramsMultipleHMKeys = taxonomyResult.get("Total").keySet();
Set<String> twoSkipTrigramsActual = new HashSet<>(twoSkipTrigramsMultipleHMKeys.stream().map(MultipleHMKeys::getKey).collect(Collectors.toList()));
Set<String> twoSkipTrigramsActual = new HashSet<>(twoSkipTrigramsMultipleHMKeys.stream().map(MultipleHMKeys::getK1).collect(Collectors.toList()));
assertEquals(twoSkipTrigrams, twoSkipTrigramsActual);
}