Project copied
This commit is contained in:
@@ -0,0 +1,85 @@
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import data.Sentence;
|
||||
import data.Word;
|
||||
|
||||
public class Common {
|
||||
|
||||
public static List<Sentence> corpus;
|
||||
public static List<Sentence> minCorpus;
|
||||
public static List<Sentence> midCorpus;
|
||||
public static List<Sentence> midCorpusSkip;
|
||||
public static List<Sentence> josTest;
|
||||
|
||||
static {
|
||||
Sentence testSentence;
|
||||
|
||||
// full sentence
|
||||
List<Word> words = new ArrayList<>();
|
||||
words.add(new Word("ker", "ker", "Vd"));
|
||||
words.add(new Word("ima", "imeti", "Ggnste-n"));
|
||||
words.add(new Word("junak", "junak", "Somei"));
|
||||
words.add(new Word("v", "v", "Dm"));
|
||||
words.add(new Word("posesti", "posest", "Sozem"));
|
||||
words.add(new Word("nekaj", "nekaj", "Rsn"));
|
||||
words.add(new Word("o", "o", "Dm"));
|
||||
words.add(new Word("čemer", "kar", "Zz-sem"));
|
||||
words.add(new Word("se", "se", "Zp------k"));
|
||||
words.add(new Word("mu", "on", "Zotmed--k"));
|
||||
words.add(new Word("ne", "ne", "L"));
|
||||
words.add(new Word("sanja", "sanjati", "Ggnste"));
|
||||
words.add(new Word("a", "a", "Vp"));
|
||||
words.add(new Word("se", "se", "Zp------k"));
|
||||
words.add(new Word("onemu", "oni", "Zk-sed"));
|
||||
words.add(new Word("zdi", "zdeti", "Ggnste"));
|
||||
words.add(new Word("ključno", "ključen", "Ppnsei"));
|
||||
words.add(new Word("pri", "pri", "Dm"));
|
||||
words.add(new Word("operaciji", "operacija", "Sozem"));
|
||||
words.add(new Word("666", "666", "Kag"));
|
||||
|
||||
testSentence = new Sentence(words, "#Ft.Z.N.N");
|
||||
corpus = new ArrayList<>();
|
||||
corpus.add(testSentence);
|
||||
|
||||
// three word sentence
|
||||
testSentence = new Sentence(corpus.get(0).getSublist(0, 3), "#Ft.Z.N.N");
|
||||
minCorpus = new ArrayList<>();
|
||||
minCorpus.add(testSentence);
|
||||
|
||||
// five word sentence
|
||||
words = new ArrayList<>();
|
||||
words.add(new Word("ker", "ker", "Vd"));
|
||||
words.add(new Word("ima", "imeti", "Ggnste-n"));
|
||||
words.add(new Word("junak", "junak", "Somei"));
|
||||
words.add(new Word("ima", "imeti", "Ggnste-n"));
|
||||
words.add(new Word("posesti", "posest", "Sozem"));
|
||||
testSentence = new Sentence(words, "#Ft.Z.N.N");
|
||||
|
||||
midCorpus = new ArrayList<>();
|
||||
midCorpus.add(testSentence);
|
||||
|
||||
// five word sentence - for skipgrams
|
||||
words = new ArrayList<>();
|
||||
words.add(new Word("ker", "ker", "Vd"));
|
||||
words.add(new Word("ima", "imeti", "Ggnste-n"));
|
||||
words.add(new Word("junak", "junak", "Somei"));
|
||||
words.add(new Word("v", "v", "Dm"));
|
||||
words.add(new Word("posesti", "posest", "Sozem"));
|
||||
testSentence = new Sentence(words, "#Ft.Z.N.N");
|
||||
|
||||
midCorpusSkip = new ArrayList<>();
|
||||
midCorpusSkip.add(testSentence);
|
||||
|
||||
// JOS test
|
||||
words = new ArrayList<>();
|
||||
words.add(new Word("junak", "junak", "Somei"));
|
||||
words.add(new Word("ima", "imeti", "Ggnste-n"));
|
||||
words.add(new Word("posesti", "posest", "Sozem"));
|
||||
testSentence = new Sentence(words, "#Ft.Z.N.N");
|
||||
|
||||
josTest = new ArrayList<>();
|
||||
josTest.add(testSentence);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
import java.io.File;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.io.IOCase;
|
||||
import org.apache.commons.io.filefilter.FileFilterUtils;
|
||||
import org.apache.commons.io.filefilter.TrueFileFilter;
|
||||
import org.junit.Test;
|
||||
|
||||
import data.*;
|
||||
import javafx.collections.ObservableList;
|
||||
|
||||
public class CorpusTests {
|
||||
|
||||
@Test
|
||||
public void solarTest() {
|
||||
//File selectedDirectory = new File("/home/andrej/Desktop/corpus-analyzer/src/main/resources/Solar");
|
||||
// File selectedDirectory = new File("/home/andrej/Desktop/corpus-analyzer/src/main/resources/GOS");
|
||||
File selectedDirectory = new File("/home/andrej/Desktop/corpus-analyzer/src/main/resources/Gigafida_subset");
|
||||
|
||||
Settings.resultsFilePath = new File(selectedDirectory.getAbsolutePath().concat(File.separator));
|
||||
|
||||
Settings.corpus = FileUtils.listFiles(selectedDirectory, FileFilterUtils.suffixFileFilter("xml", IOCase.INSENSITIVE), TrueFileFilter.INSTANCE);
|
||||
|
||||
File f = Settings.corpus.iterator().next();
|
||||
|
||||
Statistics stats = new Statistics(AnalysisLevel.STRING_LEVEL, 2, 0, CalculateFor.WORD);
|
||||
// stats.setCorpusType(CorpusType.GOS);
|
||||
stats.setCorpusType(CorpusType.SOLAR);
|
||||
|
||||
// XML_processing.readXMLGos(f.toString(), stats);
|
||||
// XML_processing.readXML(f.toString(), stats);
|
||||
// XML_processing.readXMLHeaderTag(f.toString(), "stats");
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test() {
|
||||
ObservableList<String> var = GosTaxonomy.getForComboBox();
|
||||
String debug = "";
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
import static junit.framework.Assert.*;
|
||||
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import org.rocksdb.RocksDB;
|
||||
|
||||
import util.db.RDB;
|
||||
|
||||
public class DBTest {
|
||||
|
||||
static {
|
||||
RocksDB.loadLibrary();
|
||||
}
|
||||
|
||||
// @Test
|
||||
public void dbConnectorTest() throws UnsupportedEncodingException {
|
||||
String key1 = "alfa";
|
||||
AtomicLong value1 = new AtomicLong(10);
|
||||
String key2 = "beta";
|
||||
AtomicLong value2 = new AtomicLong(20);
|
||||
String key3 = "alfa";
|
||||
AtomicLong value3 = new AtomicLong(50);
|
||||
String key4 = "theta";
|
||||
AtomicLong value4 = new AtomicLong(40);
|
||||
|
||||
HashMap<String, AtomicLong> results = new HashMap<>();
|
||||
results.put(key1, value1);
|
||||
results.put(key2, value2);
|
||||
|
||||
RDB db = new RDB();
|
||||
db.writeBatch(results);
|
||||
|
||||
// let's check how that fared out
|
||||
Map<String, AtomicLong> dumpedResults = db.getDump();
|
||||
// should have 2 items
|
||||
assertEquals(2, dumpedResults.size());
|
||||
|
||||
// entry comparison
|
||||
assertTrue(dumpedResults.containsKey(key1));
|
||||
assertTrue(value1.longValue() == dumpedResults.get(key1).longValue());
|
||||
assertTrue(dumpedResults.containsKey(key2));
|
||||
assertTrue(value2.longValue() == dumpedResults.get(key2).longValue());
|
||||
|
||||
results = new HashMap<>();
|
||||
results.put(key3, value3);
|
||||
results.put(key4, value4);
|
||||
db.writeBatch(results);
|
||||
dumpedResults = db.getDump();
|
||||
|
||||
// should have 3 items with alfa's value reflecting summation
|
||||
assertEquals(3, dumpedResults.size());
|
||||
|
||||
// entry comparison
|
||||
assertTrue(dumpedResults.containsKey(key1));
|
||||
assertTrue(value1.longValue() + value3.longValue() == dumpedResults.get(key1).longValue());
|
||||
assertTrue(dumpedResults.containsKey(key2));
|
||||
assertTrue(value2.longValue() == dumpedResults.get(key2).longValue());
|
||||
assertTrue(dumpedResults.containsKey(key4));
|
||||
assertTrue(value4.longValue() == dumpedResults.get(key4).longValue());
|
||||
|
||||
db.delete();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,334 @@
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
import alg.ngram.Ngrams;
|
||||
import data.*;
|
||||
|
||||
@SuppressWarnings({"Duplicates", "unused"})
|
||||
public class NgramTests {
|
||||
|
||||
@Test
|
||||
public void letterNgramsTest() {
|
||||
Map<String, AtomicLong> result = null;
|
||||
|
||||
Filter filter = new Filter();
|
||||
filter.setAl(AnalysisLevel.STRING_LEVEL);
|
||||
filter.setStringLength(4);
|
||||
filter.setNgramValue(0); // letters
|
||||
filter.setCalculateFor(CalculateFor.WORD);
|
||||
|
||||
Corpus testCorpus = new Corpus();
|
||||
testCorpus.setCorpusType(CorpusType.GIGAFIDA);
|
||||
testCorpus.setDetectedCorpusFiles(new ArrayList<>());
|
||||
|
||||
// tests:
|
||||
// - no regex
|
||||
StatisticsNew stats = new StatisticsNew(testCorpus, filter, false);
|
||||
Ngrams.calculateForAll(Common.minCorpus, stats);
|
||||
result = stats.getResult();
|
||||
|
||||
// tests:
|
||||
// - algorithm skips words that are shorter than set length value
|
||||
assertEquals(2, result.size());
|
||||
assertTrue(result.containsKey("juna"));
|
||||
assertEquals(1, result.get("juna").longValue());
|
||||
assertTrue(result.containsKey("unak"));
|
||||
assertEquals(1, result.get("unak").longValue());
|
||||
|
||||
// tests:
|
||||
// - map update (count) works ok
|
||||
filter.setStringLength(3);
|
||||
stats = new StatisticsNew(testCorpus, filter, false);
|
||||
Ngrams.calculateForAll(Common.midCorpus, stats);
|
||||
result = stats.getResult();
|
||||
|
||||
assertEquals(2, result.get("ima").longValue());
|
||||
|
||||
// tests:
|
||||
// - pre-check for the following regex test - this one should include word "ima", next one shouldn't
|
||||
filter.setStringLength(3);
|
||||
|
||||
stats = new StatisticsNew(testCorpus, filter, false);
|
||||
Ngrams.calculateForAll(Common.midCorpus, stats);
|
||||
result = stats.getResult();
|
||||
|
||||
assertTrue(result.containsKey("ima"));
|
||||
|
||||
// tests:
|
||||
// - regex: S.* // vsi samostalniki
|
||||
ArrayList<Pattern> msdRegex = new ArrayList<>();
|
||||
msdRegex.add(Pattern.compile("S.*"));
|
||||
filter.setMsd(msdRegex);
|
||||
|
||||
stats = new StatisticsNew(testCorpus, filter, false);
|
||||
Ngrams.calculateForAll(Common.midCorpus, stats);
|
||||
result = stats.getResult();
|
||||
|
||||
assertFalse(result.containsKey("ima"));
|
||||
|
||||
// tests:
|
||||
// - more precise regex
|
||||
msdRegex = new ArrayList<>();
|
||||
msdRegex.add(Pattern.compile("S.z.*")); // should include "posesti", but not "junak"
|
||||
filter.setMsd(msdRegex);
|
||||
filter.setStringLength(5);
|
||||
|
||||
stats = new StatisticsNew(testCorpus, filter, false);
|
||||
Ngrams.calculateForAll(Common.midCorpus, stats);
|
||||
result = stats.getResult();
|
||||
|
||||
assertFalse(result.containsKey("junak"));
|
||||
assertEquals(3, result.size());
|
||||
|
||||
// tests:
|
||||
// - trickier regex
|
||||
msdRegex = new ArrayList<>();
|
||||
msdRegex.add(Pattern.compile(".{2}")); // should count only for msd="Vd" - "ker"
|
||||
filter.setMsd(msdRegex);
|
||||
filter.setStringLength(3);
|
||||
|
||||
stats = new StatisticsNew(testCorpus, filter, false);
|
||||
Ngrams.calculateForAll(Common.midCorpus, stats);
|
||||
result = stats.getResult();
|
||||
|
||||
assertEquals(1, result.size());
|
||||
assertTrue(result.containsKey("ker"));
|
||||
assertEquals(1, result.get("ker").longValue());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void wordsNgramsTest() {
|
||||
Map<String, AtomicLong> result = null;
|
||||
|
||||
Filter filter = new Filter();
|
||||
filter.setAl(AnalysisLevel.STRING_LEVEL);
|
||||
filter.setNgramValue(3);
|
||||
|
||||
Corpus testCorpus = new Corpus();
|
||||
testCorpus.setCorpusType(CorpusType.GIGAFIDA);
|
||||
testCorpus.setDetectedCorpusFiles(new ArrayList<>());
|
||||
|
||||
// tests:
|
||||
// - normal ngrams - word
|
||||
// midCorpus contains 5 words which should make for 3 3-grams
|
||||
filter.setCalculateFor(CalculateFor.WORD);
|
||||
StatisticsNew stats = new StatisticsNew(testCorpus, filter, false);
|
||||
Ngrams.calculateForAll(Common.midCorpus, stats);
|
||||
result = stats.getResult();
|
||||
|
||||
assertEquals(3, result.size());
|
||||
assertTrue(result.containsKey("ker ima junak"));
|
||||
assertTrue(result.containsKey("ima junak ima"));
|
||||
assertTrue(result.containsKey("junak ima posesti"));
|
||||
|
||||
// tests:
|
||||
// - normal ngrams - lemmas
|
||||
filter.setCalculateFor(CalculateFor.LEMMA);
|
||||
stats = new StatisticsNew(testCorpus, filter, false);
|
||||
Ngrams.calculateForAll(Common.midCorpus, stats);
|
||||
result = stats.getResult();
|
||||
|
||||
assertEquals(3, result.size());
|
||||
assertTrue(result.containsKey("ker imeti junak"));
|
||||
assertTrue(result.containsKey("imeti junak imeti"));
|
||||
assertTrue(result.containsKey("junak imeti posest"));
|
||||
|
||||
// tests:
|
||||
// - normal ngrams - msd
|
||||
filter.setCalculateFor(CalculateFor.MORPHOSYNTACTIC_PROPERTY);
|
||||
stats = new StatisticsNew(testCorpus, filter, false);
|
||||
Ngrams.calculateForAll(Common.midCorpus, stats);
|
||||
result = stats.getResult();
|
||||
|
||||
assertEquals(3, result.size());
|
||||
assertTrue(result.containsKey("Vd Ggnste-n Somei"));
|
||||
assertTrue(result.containsKey("Ggnste-n Somei Ggnste-n"));
|
||||
assertTrue(result.containsKey("Somei Ggnste-n Sozem"));
|
||||
|
||||
// tests:
|
||||
// - ngrams - word - regex filter
|
||||
filter.setCalculateFor(CalculateFor.WORD);
|
||||
ArrayList<Pattern> msdRegex = new ArrayList<>();
|
||||
msdRegex.add(Pattern.compile("S.*"));
|
||||
msdRegex.add(Pattern.compile("G.*"));
|
||||
msdRegex.add(Pattern.compile(".*"));
|
||||
filter.setMsd(msdRegex);
|
||||
|
||||
stats = new StatisticsNew(testCorpus, filter, false);
|
||||
Ngrams.calculateForAll(Common.midCorpus, stats);
|
||||
result = stats.getResult();
|
||||
|
||||
assertEquals(1, result.size());
|
||||
assertTrue(result.containsKey("junak ima posesti"));
|
||||
|
||||
// tests:
|
||||
// - ngrams - word - regex filter
|
||||
filter.setCalculateFor(CalculateFor.WORD);
|
||||
filter.setNgramValue(2);
|
||||
msdRegex = new ArrayList<>();
|
||||
msdRegex.add(Pattern.compile("G.*"));
|
||||
msdRegex.add(Pattern.compile("Some.*"));
|
||||
filter.setMsd(msdRegex);
|
||||
|
||||
stats = new StatisticsNew(testCorpus, filter, false);
|
||||
Ngrams.calculateForAll(Common.midCorpus, stats);
|
||||
result = stats.getResult();
|
||||
|
||||
assertEquals(1, result.size());
|
||||
assertTrue(result.containsKey("ima junak"));
|
||||
}
|
||||
|
||||
|
||||
// @Test
|
||||
// public void ngramsTest() {
|
||||
// // minimal compliance test
|
||||
// Statistics stats = new Statistics(AnalysisLevel.STRING_LEVEL, 1, null, CalculateFor.MORPHOSYNTACTIC_SPECS);
|
||||
//
|
||||
// Map<String, AtomicLong> results = recalculate(minCorpus, stats);
|
||||
//
|
||||
// // 1-gram minCorpusa should equal minCorpus' size
|
||||
// assertEquals(minCorpus.get(0).getWords().size(), results.size());
|
||||
//
|
||||
// // each resulting word should have a frequency of 1
|
||||
// List<Word> words = minCorpus.get(0).getWords();
|
||||
// for (int i = 0; i < results.size(); i++) {
|
||||
// Word w = words.get(i);
|
||||
// AtomicLong frequency = results.get(w.getMsd());
|
||||
// assertEquals(1, frequency.intValue());
|
||||
// }
|
||||
//
|
||||
// // repeat for 2grams
|
||||
// stats = new Statistics(AnalysisLevel.STRING_LEVEL, 2, null, CalculateFor.MORPHOSYNTACTIC_SPECS);
|
||||
// results = recalculate(minCorpus, stats);
|
||||
//
|
||||
// // 2-gram of a 3 item corpus should equal 2 (first two words and second two words)
|
||||
// assertEquals(2, results.size());
|
||||
//
|
||||
// // add a filter
|
||||
// stats = new Statistics(AnalysisLevel.STRING_LEVEL, 1, null, CalculateFor.MORPHOSYNTACTIC_PROPERTY);
|
||||
//
|
||||
// List<String> morphosyntacticFilter = new ArrayList<>();
|
||||
// morphosyntacticFilter.add("Sozem");
|
||||
// stats.setMorphosyntacticFilter(morphosyntacticFilter);
|
||||
//
|
||||
// results = recalculate(minCorpus, stats);
|
||||
//
|
||||
// // since min corpus doesn't contain Sozem, results should be empty
|
||||
// assertEquals(0, results.size());
|
||||
//
|
||||
// stats = new Statistics(AnalysisLevel.STRING_LEVEL, 1, null, CalculateFor.MORPHOSYNTACTIC_PROPERTY);
|
||||
// morphosyntacticFilter = new ArrayList<>();
|
||||
// morphosyntacticFilter.add("Somei");
|
||||
// stats.setMorphosyntacticFilter(morphosyntacticFilter);
|
||||
// results = recalculate(minCorpus, stats);
|
||||
//
|
||||
// // since we have 1 Somei, 1 result
|
||||
// assertEquals(1, results.size());
|
||||
// assertEquals(1, results.get("Somei").intValue());
|
||||
//
|
||||
// // actual filter with wildcards
|
||||
// // 1gram
|
||||
// stats = new Statistics(AnalysisLevel.STRING_LEVEL, 1, null, CalculateFor.MORPHOSYNTACTIC_PROPERTY);
|
||||
// morphosyntacticFilter = new ArrayList<>();
|
||||
// morphosyntacticFilter.add("So***");
|
||||
// stats.setMorphosyntacticFilter(morphosyntacticFilter);
|
||||
// results = recalculate(minCorpus, stats);
|
||||
//
|
||||
// assertEquals(1, results.size());
|
||||
// assertEquals(1, results.get("Somei").intValue());
|
||||
//
|
||||
// // 2gram
|
||||
// stats = new Statistics(AnalysisLevel.STRING_LEVEL, 2, null, CalculateFor.MORPHOSYNTACTIC_PROPERTY);
|
||||
// morphosyntacticFilter = new ArrayList<>();
|
||||
// morphosyntacticFilter.add("Ggns*e-n");
|
||||
// morphosyntacticFilter.add("So***");
|
||||
// stats.setMorphosyntacticFilter(morphosyntacticFilter);
|
||||
// results = recalculate(minCorpus, stats);
|
||||
//
|
||||
// assertEquals(1, results.size());
|
||||
// assertEquals(1, results.get("Ggnste-n Somei").intValue());
|
||||
//
|
||||
// // 2gram midCorpus
|
||||
// stats = new Statistics(AnalysisLevel.STRING_LEVEL, 2, null, CalculateFor.MORPHOSYNTACTIC_PROPERTY);
|
||||
// morphosyntacticFilter = new ArrayList<>();
|
||||
// morphosyntacticFilter.add("Ggns*e-n");
|
||||
// morphosyntacticFilter.add("So***");
|
||||
// stats.setMorphosyntacticFilter(morphosyntacticFilter);
|
||||
// results = recalculate(midCorpus, stats);
|
||||
//
|
||||
// assertEquals(2, results.size());
|
||||
// assertEquals(1, results.get("Ggnste-n Somei").intValue());
|
||||
// assertEquals(1, results.get("Ggnste-n Sozem").intValue());
|
||||
// }
|
||||
|
||||
private Map<String, AtomicLong> recalculate(List<Sentence> corpus, Statistics stats) {
|
||||
// calculateForAll(corpus, stats);
|
||||
return stats.getResult();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void skipgramsTest() {
|
||||
Map<String, AtomicLong> result = null;
|
||||
|
||||
Filter filter = new Filter();
|
||||
filter.setAl(AnalysisLevel.STRING_LEVEL);
|
||||
filter.setCalculateFor(CalculateFor.WORD);
|
||||
|
||||
Corpus testCorpus = new Corpus();
|
||||
testCorpus.setCorpusType(CorpusType.GIGAFIDA);
|
||||
testCorpus.setDetectedCorpusFiles(new ArrayList<>());
|
||||
|
||||
// tests:
|
||||
// - bigrams
|
||||
filter.setNgramValue(2);
|
||||
StatisticsNew stats = new StatisticsNew(testCorpus, filter, false);
|
||||
Ngrams.calculateForAll(Common.midCorpusSkip, stats);
|
||||
result = stats.getResult();
|
||||
|
||||
Set<String> bigrams = new HashSet<>(Arrays.asList("ker ima", "ima junak", "junak v", "v posesti"));
|
||||
Set<String> bigramsActual = result.keySet();
|
||||
assertEquals(bigrams, bigramsActual);
|
||||
|
||||
// test:
|
||||
// - two skip bigrams
|
||||
filter.setNgramValue(2);
|
||||
filter.setSkipValue(2);
|
||||
stats = new StatisticsNew(testCorpus, filter, false);
|
||||
Ngrams.calculateForAll(Common.midCorpusSkip, stats);
|
||||
result = stats.getResult();
|
||||
|
||||
Set<String> twoSkipBigrams = new HashSet<>(Arrays.asList("ker ima", "ker junak", "ker v", "ima junak", "ima v", "ima posesti", "junak v", "junak posesti", "v posesti"));
|
||||
Set<String> twoSkipBigramsActual = result.keySet();
|
||||
|
||||
assertEquals(twoSkipBigrams, twoSkipBigramsActual);
|
||||
|
||||
// tests:
|
||||
// - trigrams
|
||||
filter.setNgramValue(3);
|
||||
filter.setSkipValue(null);
|
||||
stats = new StatisticsNew(testCorpus, filter, false);
|
||||
Ngrams.calculateForAll(Common.midCorpusSkip, stats);
|
||||
result = stats.getResult();
|
||||
Set<String> trigrams = new HashSet<>(Arrays.asList("ker ima junak", "ima junak v", "junak v posesti"));
|
||||
Set<String> trigramsActual = result.keySet();
|
||||
|
||||
assertEquals(trigrams, trigramsActual);
|
||||
|
||||
// tests:
|
||||
// - two skip trigrams
|
||||
filter.setNgramValue(3);
|
||||
filter.setSkipValue(2);
|
||||
stats = new StatisticsNew(testCorpus, filter, false);
|
||||
Ngrams.calculateForAll(Common.midCorpusSkip, stats);
|
||||
result = stats.getResult();
|
||||
HashSet<String> twoSkipTrigrams = new HashSet<>(Arrays.asList("ker ima junak", "ker ima v", "ker ima posesti", "ker junak v", "ker junak posesti", "ker v posesti", "ima junak v", "ima junak posesti", "ima v posesti", "junak v posesti"));
|
||||
Set<String> twoSkipTrigramsActual = result.keySet();
|
||||
|
||||
assertEquals(twoSkipTrigrams, twoSkipTrigramsActual);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
import alg.inflectedJOS.WordFormation;
|
||||
import alg.ngram.Ngrams;
|
||||
import data.*;
|
||||
|
||||
public class WordFormationTest {
|
||||
|
||||
@Test
|
||||
public void calculationTest() throws UnsupportedEncodingException {
|
||||
Map<String, AtomicLong> result = null;
|
||||
|
||||
Filter filter = new Filter();
|
||||
filter.setAl(AnalysisLevel.STRING_LEVEL);
|
||||
filter.setNgramValue(1);
|
||||
|
||||
Corpus testCorpus = new Corpus();
|
||||
testCorpus.setCorpusType(CorpusType.GIGAFIDA);
|
||||
testCorpus.setDetectedCorpusFiles(new ArrayList<>());
|
||||
|
||||
// tests:
|
||||
// - normal ngrams - word
|
||||
// midCorpus contains 5 words which should make for 3 3-grams
|
||||
filter.setCalculateFor(CalculateFor.MORPHOSYNTACTIC_PROPERTY);
|
||||
StatisticsNew stats = new StatisticsNew(testCorpus, filter, false);
|
||||
Ngrams.calculateForAll(Common.josTest, stats);
|
||||
result = stats.getResult();
|
||||
WordFormation.calculateStatistics(stats);
|
||||
Object[][] resultArr = stats.getResultCustom();
|
||||
String debug = "";
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAnything() {
|
||||
String a = "Somei";
|
||||
String b = "SomeiD";
|
||||
|
||||
String c = a.substring(0, 5);
|
||||
String d = b.substring(0, 5);
|
||||
|
||||
String debug = "";
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import java.util.HashSet;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
import data.Enums.WordLevelDefaultValues;
|
||||
|
||||
public class WordLevelTest {
|
||||
|
||||
@Test
|
||||
public void testResourceFiles() {
|
||||
HashSet<String> suffixes = WordLevelDefaultValues.getSuffixes();
|
||||
String debug = "";
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
import data.Word;
|
||||
|
||||
public class WordTest {
|
||||
@Test
|
||||
public void paddingTest() {
|
||||
Word w1 = new Word("w1", "l1", "Somei");
|
||||
Word w2 = new Word("w2", "l2", "Sometd");
|
||||
|
||||
// w1's msd should get padded
|
||||
String msd1 = w1.getMsd();
|
||||
String msd2 = w2.getMsd();
|
||||
assertEquals(msd1.length(), msd2.length());
|
||||
assertEquals(Word.PAD_CHARACTER, msd1.charAt(msd1.length() - 1));
|
||||
|
||||
w1 = new Word("w1", "l1", "Gp-g");
|
||||
w2 = new Word("w2", "l2", "Gp-g---d");
|
||||
|
||||
// w1's msd should get padded
|
||||
msd1 = w1.getMsd();
|
||||
msd2 = w2.getMsd();
|
||||
assertEquals(msd1.length(), msd2.length());
|
||||
assertEquals(Word.PAD_CHARACTER, msd1.charAt(msd1.length() - 1));
|
||||
assertEquals(Word.PAD_CHARACTER, msd2.charAt(2));
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void cvvTest() {
|
||||
String siAlphabet = "abcčdefghijklmnoprsštuvzž";
|
||||
String siAlphabetCvv = "VCCCCVCCCVCCCCCVCCCCCVCCC";
|
||||
|
||||
Word w1 = new Word(siAlphabet, "l1", null);
|
||||
assertEquals(siAlphabetCvv, w1.getCVVWord());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user