import static org.junit.Assert.* ;
import java.util.* ;
import java.util.concurrent.atomic.AtomicLong ;
import java.util.regex.Pattern ;
import java.util.stream.Collectors ;
import javafx.collections.FXCollections ;
import org.junit.Test ;
import alg.ngram.Ngrams ;
import data.* ;
@SuppressWarnings ( { "Duplicates" , "unused" } )
public class NgramTests {
@Test
public void letterNgramsTest ( ) {
Map < String , AtomicLong > result = null ;
Filter filter = new Filter ( ) ;
filter . setAl ( AnalysisLevel . STRING_LEVEL ) ;
filter . setStringLength ( 4 ) ;
filter . setNgramValue ( 0 ) ; // letters
filter . setCalculateFor ( CalculateFor . WORD ) ;
ArrayList < String > tax = new ArrayList < > ( ) ;
tax . add ( "SSJ.T.P.C" ) ;
filter . setTaxonomy ( tax ) ;
Corpus testCorpus = new Corpus ( ) ;
testCorpus . setCorpusType ( CorpusType . GIGAFIDA ) ;
testCorpus . setDetectedCorpusFiles ( new ArrayList < > ( ) ) ;
ArrayList < String > taxForCombo = new ArrayList < > ( ) ;
taxForCombo . add ( "SSJ.T.P.C" ) ;
testCorpus . setTaxonomy ( FXCollections . observableArrayList ( taxForCombo ) ) ;
// tests:
// - no regex
StatisticsNew stats = new StatisticsNew ( testCorpus , filter , false ) ;
Ngrams . calculateForAll ( Common . minCorpus , stats ) ;
result = stats . getResult ( ) ;
// tests:
// - algorithm skips words that are shorter than set length value
assertEquals ( 2 , result . size ( ) ) ;
assertTrue ( result . containsKey ( "juna" ) ) ;
assertEquals ( 1 , result . get ( "juna" ) . longValue ( ) ) ;
assertTrue ( result . containsKey ( "unak" ) ) ;
assertEquals ( 1 , result . get ( "unak" ) . longValue ( ) ) ;
// tests:
// - map update (count) works ok
filter . setStringLength ( 3 ) ;
stats = new StatisticsNew ( testCorpus , filter , false ) ;
Ngrams . calculateForAll ( Common . midCorpus , stats ) ;
result = stats . getResult ( ) ;
assertEquals ( 2 , result . get ( "ima" ) . longValue ( ) ) ;
// tests:
// - pre-check for the following regex test - this one should include word "ima", next one shouldn't
filter . setStringLength ( 3 ) ;
stats = new StatisticsNew ( testCorpus , filter , false ) ;
Ngrams . calculateForAll ( Common . midCorpus , stats ) ;
result = stats . getResult ( ) ;
assertTrue ( result . containsKey ( "ima" ) ) ;
// tests:
// - regex: S.* // vsi samostalniki
ArrayList < Pattern > msdRegex = new ArrayList < > ( ) ;
msdRegex . add ( Pattern . compile ( "S.*" ) ) ;
filter . setMsd ( msdRegex ) ;
stats = new StatisticsNew ( testCorpus , filter , false ) ;
Ngrams . calculateForAll ( Common . midCorpus , stats ) ;
result = stats . getResult ( ) ;
assertFalse ( result . containsKey ( "ima" ) ) ;
// tests:
// - more precise regex
msdRegex = new ArrayList < > ( ) ;
msdRegex . add ( Pattern . compile ( "S.z.*" ) ) ; // should include "posesti", but not "junak"
filter . setMsd ( msdRegex ) ;
filter . setStringLength ( 5 ) ;
stats = new StatisticsNew ( testCorpus , filter , false ) ;
Ngrams . calculateForAll ( Common . midCorpus , stats ) ;
result = stats . getResult ( ) ;
assertFalse ( result . containsKey ( "junak" ) ) ;
assertEquals ( 3 , result . size ( ) ) ;
// tests:
// - trickier regex
msdRegex = new ArrayList < > ( ) ;
msdRegex . add ( Pattern . compile ( ".{2}" ) ) ; // should count only for msd="Vd" - "ker"
filter . setMsd ( msdRegex ) ;
filter . setStringLength ( 3 ) ;
stats = new StatisticsNew ( testCorpus , filter , false ) ;
Ngrams . calculateForAll ( Common . midCorpus , stats ) ;
result = stats . getResult ( ) ;
assertEquals ( 1 , result . size ( ) ) ;
assertTrue ( result . containsKey ( "ker" ) ) ;
assertEquals ( 1 , result . get ( "ker" ) . longValue ( ) ) ;
}
@Test
public void wordsNgramsTest ( ) {
Map < String , Map < MultipleHMKeys , AtomicLong > > taxonomyResult ;
Filter filter = new Filter ( ) ;
filter . setAl ( AnalysisLevel . STRING_LEVEL ) ;
filter . setNgramValue ( 3 ) ;
ArrayList < String > tax = new ArrayList < > ( ) ;
tax . add ( "SSJ.T.P.C" ) ;
filter . setTaxonomy ( tax ) ;
ArrayList < String > mKeys = new ArrayList < > ( ) ;
//mKeys.add("lema");
filter . setMultipleKeys ( mKeys ) ;
Corpus testCorpus = new Corpus ( ) ;
testCorpus . setCorpusType ( CorpusType . GIGAFIDA ) ;
testCorpus . setDetectedCorpusFiles ( new ArrayList < > ( ) ) ;
ArrayList < String > taxForCombo = new ArrayList < > ( ) ;
taxForCombo . add ( "SSJ.T.P.C" ) ;
testCorpus . setTaxonomy ( FXCollections . observableArrayList ( taxForCombo ) ) ;
// tests:
// - normal ngrams - word
// midCorpus contains 5 words which should make for 3 3-grams
filter . setCalculateFor ( CalculateFor . WORD ) ;
StatisticsNew stats = new StatisticsNew ( testCorpus , filter , false ) ;
Ngrams . calculateForAll ( Common . midCorpus , stats ) ;
taxonomyResult = stats . getTaxonomyResult ( ) ;
assertEquals ( 3 , taxonomyResult . get ( "Total" ) . size ( ) ) ;
assertTrue ( taxonomyResult . get ( "Total" ) . containsKey ( new MultipleHMKeys ( "ker ima junak" , "" , "" , "" ) ) ) ;
assertTrue ( taxonomyResult . get ( "Total" ) . containsKey ( new MultipleHMKeys ( "ima junak ima" , "" , "" , "" ) ) ) ;
assertTrue ( taxonomyResult . get ( "Total" ) . containsKey ( new MultipleHMKeys ( "junak ima posesti" , "" , "" , "" ) ) ) ;
// tests:
// - normal ngrams - lemmas
filter . setCalculateFor ( CalculateFor . LEMMA ) ;
stats = new StatisticsNew ( testCorpus , filter , false ) ;
Ngrams . calculateForAll ( Common . midCorpus , stats ) ;
taxonomyResult = stats . getTaxonomyResult ( ) ;
assertEquals ( 3 , taxonomyResult . get ( "Total" ) . size ( ) ) ;
assertTrue ( taxonomyResult . get ( "Total" ) . containsKey ( new MultipleHMKeys ( "ker imeti junak" , "" , "" , "" ) ) ) ;
assertTrue ( taxonomyResult . get ( "Total" ) . containsKey ( new MultipleHMKeys ( "imeti junak imeti" , "" , "" , "" ) ) ) ;
assertTrue ( taxonomyResult . get ( "Total" ) . containsKey ( new MultipleHMKeys ( "junak imeti posest" , "" , "" , "" ) ) ) ;
// tests:
// - normal ngrams - msd
filter . setCalculateFor ( CalculateFor . MORPHOSYNTACTIC_PROPERTY ) ;
stats = new StatisticsNew ( testCorpus , filter , false ) ;
Ngrams . calculateForAll ( Common . midCorpus , stats ) ;
taxonomyResult = stats . getTaxonomyResult ( ) ;
assertEquals ( 3 , taxonomyResult . get ( "Total" ) . size ( ) ) ;
assertTrue ( taxonomyResult . get ( "Total" ) . containsKey ( new MultipleHMKeys ( "Vd Ggnste-n Somei" , "" , "" , "" ) ) ) ;
assertTrue ( taxonomyResult . get ( "Total" ) . containsKey ( new MultipleHMKeys ( "Ggnste-n Somei Ggnste-n" , "" , "" , "" ) ) ) ;
assertTrue ( taxonomyResult . get ( "Total" ) . containsKey ( new MultipleHMKeys ( "Somei Ggnste-n Sozem" , "" , "" , "" ) ) ) ;
// tests:
// - ngrams - word - regex filter
filter . setCalculateFor ( CalculateFor . WORD ) ;
ArrayList < Pattern > msdRegex = new ArrayList < > ( ) ;
msdRegex . add ( Pattern . compile ( "S.*" ) ) ;
msdRegex . add ( Pattern . compile ( "G.*" ) ) ;
msdRegex . add ( Pattern . compile ( ".*" ) ) ;
filter . setMsd ( msdRegex ) ;
stats = new StatisticsNew ( testCorpus , filter , false ) ;
Ngrams . calculateForAll ( Common . midCorpus , stats ) ;
taxonomyResult = stats . getTaxonomyResult ( ) ;
assertEquals ( 1 , taxonomyResult . get ( "Total" ) . size ( ) ) ;
assertTrue ( taxonomyResult . get ( "Total" ) . containsKey ( new MultipleHMKeys ( "junak ima posesti" , "" , "" , "" ) ) ) ;
// tests:
// - ngrams - word - regex filter
filter . setCalculateFor ( CalculateFor . WORD ) ;
filter . setNgramValue ( 2 ) ;
msdRegex = new ArrayList < > ( ) ;
msdRegex . add ( Pattern . compile ( "G.*" ) ) ;
msdRegex . add ( Pattern . compile ( "Some.*" ) ) ;
filter . setMsd ( msdRegex ) ;
stats = new StatisticsNew ( testCorpus , filter , false ) ;
Ngrams . calculateForAll ( Common . midCorpus , stats ) ;
taxonomyResult = stats . getTaxonomyResult ( ) ;
assertEquals ( 1 , taxonomyResult . get ( "Total" ) . size ( ) ) ;
assertTrue ( taxonomyResult . get ( "Total" ) . containsKey ( new MultipleHMKeys ( "ima junak" , "" , "" , "" ) ) ) ;
}
// @Test
// public void ngramsTest() {
// // minimal compliance test
// Statistics stats = new Statistics(AnalysisLevel.STRING_LEVEL, 1, null, CalculateFor.MORPHOSYNTACTIC_SPECS);
//
// Map<String, AtomicLong> results = recalculate(minCorpus, stats);
//
// // 1-gram minCorpusa should equal minCorpus' size
// assertEquals(minCorpus.get(0).getWords().size(), results.size());
//
// // each resulting word should have a frequency of 1
// List<Word> words = minCorpus.get(0).getWords();
// for (int i = 0; i < results.size(); i++) {
// Word w = words.get(i);
// AtomicLong frequency = results.get(w.getMsd());
// assertEquals(1, frequency.intValue());
// }
//
// // repeat for 2grams
// stats = new Statistics(AnalysisLevel.STRING_LEVEL, 2, null, CalculateFor.MORPHOSYNTACTIC_SPECS);
// results = recalculate(minCorpus, stats);
//
// // 2-gram of a 3 item corpus should equal 2 (first two words and second two words)
// assertEquals(2, results.size());
//
// // add a filter
// stats = new Statistics(AnalysisLevel.STRING_LEVEL, 1, null, CalculateFor.MORPHOSYNTACTIC_PROPERTY);
//
// List<String> morphosyntacticFilter = new ArrayList<>();
// morphosyntacticFilter.add("Sozem");
// stats.setMorphosyntacticFilter(morphosyntacticFilter);
//
// results = recalculate(minCorpus, stats);
//
// // since min corpus doesn't contain Sozem, results should be empty
// assertEquals(0, results.size());
//
// stats = new Statistics(AnalysisLevel.STRING_LEVEL, 1, null, CalculateFor.MORPHOSYNTACTIC_PROPERTY);
// morphosyntacticFilter = new ArrayList<>();
// morphosyntacticFilter.add("Somei");
// stats.setMorphosyntacticFilter(morphosyntacticFilter);
// results = recalculate(minCorpus, stats);
//
// // since we have 1 Somei, 1 result
// assertEquals(1, results.size());
// assertEquals(1, results.get("Somei").intValue());
//
// // actual filter with wildcards
// // 1gram
// stats = new Statistics(AnalysisLevel.STRING_LEVEL, 1, null, CalculateFor.MORPHOSYNTACTIC_PROPERTY);
// morphosyntacticFilter = new ArrayList<>();
// morphosyntacticFilter.add("So***");
// stats.setMorphosyntacticFilter(morphosyntacticFilter);
// results = recalculate(minCorpus, stats);
//
// assertEquals(1, results.size());
// assertEquals(1, results.get("Somei").intValue());
//
// // 2gram
// stats = new Statistics(AnalysisLevel.STRING_LEVEL, 2, null, CalculateFor.MORPHOSYNTACTIC_PROPERTY);
// morphosyntacticFilter = new ArrayList<>();
// morphosyntacticFilter.add("Ggns*e-n");
// morphosyntacticFilter.add("So***");
// stats.setMorphosyntacticFilter(morphosyntacticFilter);
// results = recalculate(minCorpus, stats);
//
// assertEquals(1, results.size());
// assertEquals(1, results.get("Ggnste-n Somei").intValue());
//
// // 2gram midCorpus
// stats = new Statistics(AnalysisLevel.STRING_LEVEL, 2, null, CalculateFor.MORPHOSYNTACTIC_PROPERTY);
// morphosyntacticFilter = new ArrayList<>();
// morphosyntacticFilter.add("Ggns*e-n");
// morphosyntacticFilter.add("So***");
// stats.setMorphosyntacticFilter(morphosyntacticFilter);
// results = recalculate(midCorpus, stats);
//
// assertEquals(2, results.size());
// assertEquals(1, results.get("Ggnste-n Somei").intValue());
// assertEquals(1, results.get("Ggnste-n Sozem").intValue());
// }
private Map < String , AtomicLong > recalculate ( List < Sentence > corpus , Statistics stats ) {
// calculateForAll(corpus, stats);
return stats . getResult ( ) ;
}
@Test
public void skipgramsTest ( ) {
Map < String , Map < MultipleHMKeys , AtomicLong > > taxonomyResult ;
Filter filter = new Filter ( ) ;
filter . setAl ( AnalysisLevel . STRING_LEVEL ) ;
filter . setCalculateFor ( CalculateFor . WORD ) ;
ArrayList < String > tax = new ArrayList < > ( ) ;
tax . add ( "SSJ.T.P.C" ) ;
filter . setTaxonomy ( tax ) ;
Corpus testCorpus = new Corpus ( ) ;
testCorpus . setCorpusType ( CorpusType . GIGAFIDA ) ;
testCorpus . setDetectedCorpusFiles ( new ArrayList < > ( ) ) ;
ArrayList < String > taxForCombo = new ArrayList < > ( ) ;
taxForCombo . add ( "tisk-periodično-časopis" ) ;
testCorpus . setTaxonomy ( FXCollections . observableArrayList ( taxForCombo ) ) ;
// tests:
// - bigrams
filter . setNgramValue ( 2 ) ;
StatisticsNew stats = new StatisticsNew ( testCorpus , filter , false ) ;
Ngrams . calculateForAll ( Common . midCorpusSkip , stats ) ;
taxonomyResult = stats . getTaxonomyResult ( ) ;
Set < String > bigrams = new HashSet < > ( Arrays . asList ( "ker ima" , "ima junak" , "junak v" , "v posesti" ) ) ;
Set < MultipleHMKeys > bigramsMultipleHMKeys = taxonomyResult . get ( "Total" ) . keySet ( ) ;
Set < String > bigramsActual = new HashSet < > ( bigramsMultipleHMKeys . stream ( ) . map ( MultipleHMKeys : : getKey ) . collect ( Collectors . toList ( ) ) ) ;
assertEquals ( bigrams , bigramsActual ) ;
// test:
// - two skip bigrams
filter . setNgramValue ( 2 ) ;
filter . setSkipValue ( 2 ) ;
stats = new StatisticsNew ( testCorpus , filter , false ) ;
Ngrams . calculateForAll ( Common . midCorpusSkip , stats ) ;
taxonomyResult = stats . getTaxonomyResult ( ) ;
Set < String > twoSkipBigrams = new HashSet < > ( Arrays . asList ( "ker ima" , "ker junak" , "ker v" , "ima junak" , "ima v" , "ima posesti" , "junak v" , "junak posesti" , "v posesti" ) ) ;
Set < MultipleHMKeys > twoSkipBigramsMultipleHMKeys = taxonomyResult . get ( "Total" ) . keySet ( ) ;
Set < String > twoSkipBigramsActual = new HashSet < > ( twoSkipBigramsMultipleHMKeys . stream ( ) . map ( MultipleHMKeys : : getKey ) . collect ( Collectors . toList ( ) ) ) ;
assertEquals ( twoSkipBigrams , twoSkipBigramsActual ) ;
// tests:
// - trigrams
filter . setNgramValue ( 3 ) ;
filter . setSkipValue ( null ) ;
stats = new StatisticsNew ( testCorpus , filter , false ) ;
Ngrams . calculateForAll ( Common . midCorpusSkip , stats ) ;
taxonomyResult = stats . getTaxonomyResult ( ) ;
Set < String > trigrams = new HashSet < > ( Arrays . asList ( "ker ima junak" , "ima junak v" , "junak v posesti" ) ) ;
Set < MultipleHMKeys > trigramsMultipleHMKeys = taxonomyResult . get ( "Total" ) . keySet ( ) ;
Set < String > trigramsActual = new HashSet < > ( trigramsMultipleHMKeys . stream ( ) . map ( MultipleHMKeys : : getKey ) . collect ( Collectors . toList ( ) ) ) ;
assertEquals ( trigrams , trigramsActual ) ;
// tests:
// - two skip trigrams
filter . setNgramValue ( 3 ) ;
filter . setSkipValue ( 2 ) ;
stats = new StatisticsNew ( testCorpus , filter , false ) ;
Ngrams . calculateForAll ( Common . midCorpusSkip , stats ) ;
taxonomyResult = stats . getTaxonomyResult ( ) ;
HashSet < String > twoSkipTrigrams = new HashSet < > ( Arrays . asList ( "ker ima junak" , "ker ima v" , "ker ima posesti" , "ker junak v" , "ker junak posesti" , "ker v posesti" , "ima junak v" , "ima junak posesti" , "ima v posesti" , "junak v posesti" ) ) ;
Set < MultipleHMKeys > twoSkipTrigramsMultipleHMKeys = taxonomyResult . get ( "Total" ) . keySet ( ) ;
Set < String > twoSkipTrigramsActual = new HashSet < > ( twoSkipTrigramsMultipleHMKeys . stream ( ) . map ( MultipleHMKeys : : getKey ) . collect ( Collectors . toList ( ) ) ) ;
assertEquals ( twoSkipTrigrams , twoSkipTrigramsActual ) ;
}
}