13 files changed, 384 insertions, 0 deletions
diff --git a/morfologik-speller/src/test/java/morfologik/speller/HMatrixTest.java b/morfologik-speller/src/test/java/morfologik/speller/HMatrixTest.java
new file mode 100644
index 0000000..38aa76d
--- /dev/null
+++ b/morfologik-speller/src/test/java/morfologik/speller/HMatrixTest.java
@@ -0,0 +1,21 @@
+package morfologik.speller;
+
+import static org.junit.Assert.*;
+
+import morfologik.speller.HMatrix;
+
+import org.junit.Test;
+
+public class HMatrixTest {
+    
+    private static final int MAX_WORD_LENGTH = 120;
+
+    @Test
+    public void stressTestInit() {
+        for (int i = 0; i < 10; i++) { // test if we don't get beyond array limits etc.
+            HMatrix H = new HMatrix(i, MAX_WORD_LENGTH);
+            assertEquals(0, H.get(1, 1));
+        }
+    }
+
+}
diff --git a/morfologik-speller/src/test/java/morfologik/speller/SpellerTest.java b/morfologik-speller/src/test/java/morfologik/speller/SpellerTest.java
new file mode 100644
index 0000000..48ed2c1
--- /dev/null
+++ b/morfologik-speller/src/test/java/morfologik/speller/SpellerTest.java
@@ -0,0 +1,272 @@
+package morfologik.speller;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.List;
+
+import morfologik.stemming.Dictionary;
+
+import org.fest.assertions.api.Assertions;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class SpellerTest {
+  private static Dictionary dictionary;
+
+  @BeforeClass
+  public static void setup() throws Exception {
+    final URL url = SpellerTest.class.getResource("slownik.dict");
+    dictionary = Dictionary.read(url);
+  }
+
+  /*
+    @Test
+    public void testAbka() throws Exception {
+        final Speller spell = new Speller(dictionary, 2);
+        System.out.println("Replacements:");
+        for (String s : spell.findReplacements("abka")) {
+            System.out.println(s);
+        }
+    }
+   */
+
+  @Test
+  public void testRunonWords() throws IOException {
+    final Speller spell = new Speller(dictionary);
+    Assertions.assertThat(spell.replaceRunOnWords("abaka")).isEmpty();
+    Assertions.assertThat(spell.replaceRunOnWords("abakaabace")).contains("abaka abace");
+
+    // Test on an morphological dictionary - should work as well
+    final URL url1 = getClass().getResource("test-infix.dict");
+    final Speller spell1 = new Speller(Dictionary.read(url1));
+    assertTrue(spell1.replaceRunOnWords("Rzekunia").isEmpty());
+    assertTrue(spell1.replaceRunOnWords("RzekuniaRzeczypospolitej").contains("Rzekunia Rzeczypospolitej"));
+    assertTrue(spell1.replaceRunOnWords("RzekuniaRze").isEmpty()); //Rze is not found but is a prefix
+  }
+
+  @Test
+  public void testIsInDictionary() throws IOException {
+    // Test on an morphological dictionary, including separators
+    final URL url1 = getClass().getResource("test-infix.dict");
+    final Speller spell1 = new Speller(Dictionary.read(url1));
+    assertTrue(spell1.isInDictionary("Rzekunia"));
+    assertTrue(!spell1.isInDictionary("Rzekunia+"));
+    assertTrue(!spell1.isInDictionary("Rzekunia+aaa"));
+    // test UTF-8 dictionary
+    final URL url = getClass().getResource("test-utf-spell.dict");
+    final Speller spell = new Speller(Dictionary.read(url));
+    assertTrue(spell.isInDictionary("jaźń"));
+    assertTrue(spell.isInDictionary("zażółć"));
+    assertTrue(spell.isInDictionary("żółwiową"));
+    assertTrue(spell.isInDictionary("ćwikła"));
+    assertTrue(spell.isInDictionary("Żebrowski"));
+    assertTrue(spell.isInDictionary("Święto"));
+    assertTrue(spell.isInDictionary("Świerczewski"));
+    assertTrue(spell.isInDictionary("abc"));
+  }
+
+  @Test
+  public void testFindReplacements() throws IOException {
+    final Speller spell = new Speller(dictionary, 1);
+    assertTrue(spell.findReplacements("abka").contains("abak"));
+    //check if we get only dictionary words...
+    List<String> reps = spell.findReplacements("bak");
+    for (final String word: reps) {
+      assertTrue(spell.isInDictionary(word));
+    }
+    assertTrue(spell.findReplacements("abka~~").isEmpty()); // 2 characters more -> edit distance too large
+    assertTrue(!spell.findReplacements("Rezkunia").contains("Rzekunia"));
+
+    final URL url1 = getClass().getResource("test-infix.dict");
+    final Speller spell1 = new Speller(Dictionary.read(url1));
+    assertTrue(spell1.findReplacements("Rezkunia").contains("Rzekunia"));
+    //diacritics
+    assertTrue(spell1.findReplacements("Rzękunia").contains("Rzekunia"));
+    //we should get no candidates for correct words
+    assertTrue(spell1.isInDictionary("Rzekunia"));
+    assertTrue(spell1.findReplacements("Rzekunia").isEmpty());
+    //and no for things that are too different from the dictionary
+    assertTrue(spell1.findReplacements("Strefakibica").isEmpty());
+    //nothing for nothing
+    assertTrue(spell1.findReplacements("").isEmpty());
+    //nothing for weird characters
+    assertTrue(spell1.findReplacements("\u0000").isEmpty());
+    //nothing for other characters
+    assertTrue(spell1.findReplacements("«…»").isEmpty());
+    //nothing for separator
+    assertTrue(spell1.findReplacements("+").isEmpty());
+
+  }
+
+  @Test
+  public void testFrequencyNonUTFDictionary() throws IOException {
+    final URL url1 = getClass().getResource("test_freq_iso.dict");
+    final Speller spell = new Speller(Dictionary.read(url1));
+    assertTrue(spell.isInDictionary("a"));
+    assertTrue(!spell.isInDictionary("aõh")); //non-encodable in UTF-8
+  }
+
+  @Test
+  public void testFindReplacementsInUTF() throws IOException {
+    final URL url = getClass().getResource("test-utf-spell.dict");
+    final Speller spell = new Speller(Dictionary.read(url));
+    assertTrue(spell.findReplacements("gęslą").contains("gęślą"));
+    assertTrue(spell.findReplacements("ćwikla").contains("ćwikła"));
+    assertTrue(spell.findReplacements("Swierczewski").contains("Świerczewski"));
+    assertTrue(spell.findReplacements("zółwiową").contains("żółwiową"));
+    assertTrue(spell.findReplacements("Żebrowsk").contains("Żebrowski"));
+    assertTrue(spell.findReplacements("święto").contains("Święto"));
+    //note: no diacritics here, but we still get matches!
+    assertTrue(spell.findReplacements("gesla").contains("gęślą"));
+    assertTrue(spell.findReplacements("swieto").contains("Święto"));
+    assertTrue(spell.findReplacements("zolwiowa").contains("żółwiową"));
+    //using equivalent characters 'x' = 'ź'
+    assertTrue(spell.findReplacements("jexn").contains("jaźń"));
+    // 'u' = 'ó', so the edit distance is still small...
+    assertTrue(spell.findReplacements("zażulv").contains("zażółć"));
+    // 'rz' = 'ż', so the edit distance is still small, but with string replacements...
+    assertTrue(spell.findReplacements("zarzulv").contains("zażółć"));
+    assertTrue(spell.findReplacements("Rzebrowski").contains("Żebrowski"));
+    assertTrue(spell.findReplacements("rzółw").contains("żółw"));
+    assertTrue(spell.findReplacements("Świento").contains("Święto"));
+    // avoid mixed-case words as suggestions when using replacements ('rz' = 'ż')
+    assertTrue(spell.findReplacements("zArzółć").get(0).equals("zażółć"));
+  }
+
+  @Test
+  public void testFindReplacementsUsingFrequency() throws IOException {
+    final URL url = getClass().getResource("dict-with-freq.dict");
+    final Speller spell = new Speller(Dictionary.read(url));
+
+    //check if we get only dictionary words...
+    List<String> reps = spell.findReplacements("jist");
+    for (final String word: reps) {
+      assertTrue(spell.isInDictionary(word));
+    }
+    // get replacements ordered by frequency
+    assertTrue(reps.get(0).equals("just"));
+    assertTrue(reps.get(1).equals("list"));
+    assertTrue(reps.get(2).equals("fist"));
+    assertTrue(reps.get(3).equals("mist"));
+    assertTrue(reps.get(4).equals("jest"));
+    assertTrue(reps.get(5).equals("dist"));
+    assertTrue(reps.get(6).equals("gist"));
+  }
+
+  @Test
+  public void testIsMisspelled() throws IOException {
+    final URL url = getClass().getResource("test-utf-spell.dict");
+    final Speller spell = new Speller(Dictionary.read(url));
+    assertTrue(!spell.isMisspelled("Paragraf22"));  //ignorujemy liczby
+    assertTrue(!spell.isMisspelled("!"));  //ignorujemy znaki przestankowe
+    assertTrue(spell.isMisspelled("dziekie"));  //test, czy znajdujemy błąd
+    assertTrue(!spell.isMisspelled("SłowozGarbem"));  //ignorujemy słowa w stylu wielbłąda
+    assertTrue(!spell.isMisspelled("Ćwikła"));  //i małe litery
+    assertTrue(!spell.isMisspelled("TOJESTTEST"));  //i wielkie litery
+    final Speller oldStyleSpell = new Speller(dictionary, 1);
+    assertTrue(oldStyleSpell.isMisspelled("Paragraf22"));  // nie ignorujemy liczby
+    assertTrue(oldStyleSpell.isMisspelled("!"));  //nie ignorujemy znaków przestankowych
+    // assertTrue(oldStyleSpell.isMisspelled("SłowozGarbem"));  //ignorujemy słowa w stylu wielbłąda
+    assertTrue(oldStyleSpell.isMisspelled("Abaka"));  //i małe litery
+    final URL url1 = getClass().getResource("test-infix.dict");
+    final Speller spell1 = new Speller(Dictionary.read(url1));
+    assertTrue(!spell1.isMisspelled("Rzekunia"));
+    assertTrue(spell1.isAllUppercase("RZEKUNIA"));
+    assertTrue(spell1.isMisspelled("RZEKUNIAA")); // finds a typo here
+    assertTrue(!spell1.isMisspelled("RZEKUNIA")); // but not here
+  }
+
+  @Test
+  public void testCamelCase() {
+    final Speller spell = new Speller(dictionary, 1);
+    assertTrue(spell.isCamelCase("CamelCase"));
+    assertTrue(!spell.isCamelCase("Camel"));
+    assertTrue(!spell.isCamelCase("CAMEL"));
+    assertTrue(!spell.isCamelCase("camel"));
+    assertTrue(!spell.isCamelCase("cAmel"));
+    assertTrue(!spell.isCamelCase("CAmel"));
+    assertTrue(!spell.isCamelCase(""));
+    assertTrue(!spell.isCamelCase(null));
+  }
+  
+  @Test
+  public void testCapitalizedWord() {
+    final Speller spell = new Speller(dictionary, 1);
+    assertTrue(spell.isNotCapitalizedWord("CamelCase"));
+    assertTrue(!spell.isNotCapitalizedWord("Camel"));
+    assertTrue(spell.isNotCapitalizedWord("CAMEL"));
+    assertTrue(spell.isNotCapitalizedWord("camel"));
+    assertTrue(spell.isNotCapitalizedWord("cAmel"));
+    assertTrue(spell.isNotCapitalizedWord("CAmel"));
+    assertTrue(spell.isNotCapitalizedWord(""));
+  }
+
+  @Test
+  public void testGetAllReplacements() throws IOException {
+    final URL url = getClass().getResource("test-utf-spell.dict");
+    final Speller spell = new Speller(Dictionary.read(url));
+    assertTrue(spell.isMisspelled("rzarzerzarzu"));
+    assertEquals("[rzarzerzarzu]",
+        Arrays.toString(spell.getAllReplacements("rzarzerzarzu", 0, 0).toArray()));
+  }
+
+  @Test
+  public void testEditDistanceCalculation() throws IOException {
+    final Speller spell = new Speller(dictionary, 5);
+    //test examples from Oflazer's paper
+    assertTrue(getEditDistance(spell, "recoginze", "recognize") == 1);
+    assertTrue(getEditDistance(spell, "sailn", "failing") == 3);
+    assertTrue(getEditDistance(spell, "abc", "abcd") == 1);
+    assertTrue(getEditDistance(spell, "abc", "abcde") == 2);
+    //test words from fsa_spell output
+    assertTrue(getEditDistance(spell, "abka", "abaka") == 1);
+    assertTrue(getEditDistance(spell, "abka", "abakan") == 2);
+    assertTrue(getEditDistance(spell, "abka", "abaką") == 2);
+    assertTrue(getEditDistance(spell, "abka", "abaki") == 2);
+  }
+
+  @Test
+  public void testCutOffEditDistance() throws IOException {
+    final Speller spell2 = new Speller(dictionary, 2); //note: threshold = 2
+    //test cut edit distance - reprter / repo from Oflazer
+    assertTrue(getCutOffDistance(spell2, "repo", "reprter") == 1);
+    assertTrue(getCutOffDistance(spell2, "reporter", "reporter") == 0);
+  }
+
+  private int getCutOffDistance(final Speller spell, final String word, final String candidate) {
+    // assuming there is no pair-replacement 
+    spell.setWordAndCandidate(word, candidate);
+    final int [] ced = new int[spell.getCandLen() - spell.getWordLen()];
+    for (int i = 0; i < spell.getCandLen() - spell.getWordLen(); i++) {
+      
+      ced[i] = spell.cuted(spell.getWordLen() + i, spell.getWordLen() + i, spell.getWordLen() + i);
+    }
+    Arrays.sort(ced);
+    //and the min value...
+    if (ced.length > 0) {
+      return ced[0];
+    }
+    return 0;
+  }
+
+  private int getEditDistance(final Speller spell, final String word, final String candidate) {
+    // assuming there is no pair-replacement
+    spell.setWordAndCandidate(word, candidate);
+    final int maxDistance = spell.getEffectiveED();
+    final int candidateLen = spell.getCandLen();
+    final int wordLen = spell.getWordLen();
+    int ed = 0;
+    for (int i = 0; i < candidateLen; i++) {
+      if (spell.cuted(i, i, i) <= maxDistance) {
+        if (Math.abs(wordLen - 1 - i) <= maxDistance) {
+          ed = spell.ed(wordLen - 1, i, wordLen - 1, i);
+        }
+      }
+    }
+    return ed;
+  }
+}
+\ No newline at end of file
diff --git a/morfologik-speller/src/test/resources/morfologik/speller/dict-with-freq.dict b/morfologik-speller/src/test/resources/morfologik/speller/dict-with-freq.dict
new file mode 100644
index 0000000..609a267
--- /dev/null
+++ b/morfologik-speller/src/test/resources/morfologik/speller/dict-with-freq.dict
diff --git a/morfologik-speller/src/test/resources/morfologik/speller/dict-with-freq.info b/morfologik-speller/src/test/resources/morfologik/speller/dict-with-freq.info
new file mode 100644
index 0000000..1203602
--- /dev/null
+++ b/morfologik-speller/src/test/resources/morfologik/speller/dict-with-freq.info
@@ -0,0 +1,15 @@
+#
+# Dictionary properties.
+#
+
+fsa.dict.separator=+
+fsa.dict.encoding=iso-8859-2
+
+fsa.dict.uses-prefixes=false
+fsa.dict.uses-infixes=false
+
+fsa.dict.frequency-included=true
+
+fsa.dict.speller.locale=en_US
+fsa.dict.speller.ignore-diacritics=true
+#fsa.dict.speller.replacement-pairs=ninties 1990s, teached taught, rised rose, a ei, ei a, a ey, ey a, ai ie, ie ai, are air, are ear, are eir, air are, air ere, ere air, ere ear, ere eir, ear are, ear air, ear ere, eir are, eir ere, ch te, te ch, ch ti, ti ch, ch tu, tu ch, ch s, s ch, ch k, k ch, f ph, ph f, gh f, f gh, i igh, igh i, i uy, uy i, i ee, ee i, j di, di j, j gg, gg j, j ge, ge j, s ti, ti s, s ci, ci s, k cc, cc k, k qu, qu k, kw qu, o eau, eau o, o ew, ew o, oo ew, ew oo, ew ui, ui ew, oo ui, ui oo, ew u, u ew, oo u, u oo, u oe, oe u, u ieu, ieu u, ue ew, ew ue, uff ough, oo ieu, ieu oo, ier ear, ear ier, ear air, air ear, w qu, qu w, z ss, ss z, shun tion, shun sion, shun cion
+\ No newline at end of file
diff --git a/morfologik-speller/src/test/resources/morfologik/speller/dict-with-freq.txt b/morfologik-speller/src/test/resources/morfologik/speller/dict-with-freq.txt
new file mode 100644
index 0000000..a12876c
--- /dev/null
+++ b/morfologik-speller/src/test/resources/morfologik/speller/dict-with-freq.txt
@@ -0,0 +1,21 @@
+ageist+C
+deist+G
+didst+A
+digest+J
+direst+E
+dist+G
+divest+I
+fist+J
+gist+G
+grist+I
+heist+I
+hist+A
+jest+H
+jilt+D
+joist+F
+just+P
+licit+F
+list+O
+mist+J
+weest+A
+wist+C
diff --git a/morfologik-speller/src/test/resources/morfologik/speller/slownik.dict b/morfologik-speller/src/test/resources/morfologik/speller/slownik.dict
new file mode 100644
index 0000000..b650702
--- /dev/null
+++ b/morfologik-speller/src/test/resources/morfologik/speller/slownik.dict
diff --git a/morfologik-speller/src/test/resources/morfologik/speller/slownik.info b/morfologik-speller/src/test/resources/morfologik/speller/slownik.info
new file mode 100644
index 0000000..25aef99
--- /dev/null
+++ b/morfologik-speller/src/test/resources/morfologik/speller/slownik.info
@@ -0,0 +1,14 @@
+#
+# Dictionary properties.
+#
+
+fsa.dict.separator=+
+fsa.dict.encoding=Cp1250
+
+fsa.dict.uses-prefixes=false
+fsa.dict.uses-infixes=false
+
+fsa.dict.speller.ignore-diacritics=false
+fsa.dict.speller.ignore-numbers=false
+fsa.dict.speller.convert-case=false
+fsa.dict.speller.ignore-punctuation=false
+\ No newline at end of file
diff --git a/morfologik-speller/src/test/resources/morfologik/speller/test-infix.dict b/morfologik-speller/src/test/resources/morfologik/speller/test-infix.dict
new file mode 100644
index 0000000..cc91f70
--- /dev/null
+++ b/morfologik-speller/src/test/resources/morfologik/speller/test-infix.dict
diff --git a/morfologik-speller/src/test/resources/morfologik/speller/test-infix.info b/morfologik-speller/src/test/resources/morfologik/speller/test-infix.info
new file mode 100644
index 0000000..9ba1066
--- /dev/null
+++ b/morfologik-speller/src/test/resources/morfologik/speller/test-infix.info
@@ -0,0 +1,10 @@
+#
+# Dictionary properties.
+#
+
+fsa.dict.separator=+
+fsa.dict.encoding=iso-8859-2
+
+fsa.dict.uses-prefixes=true
+fsa.dict.uses-infixes=true
+fsa.dict.speller.ignore-all-uppercase=false
+\ No newline at end of file
diff --git a/morfologik-speller/src/test/resources/morfologik/speller/test-utf-spell.dict b/morfologik-speller/src/test/resources/morfologik/speller/test-utf-spell.dict
new file mode 100644
index 0000000..63ff635
--- /dev/null
+++ b/morfologik-speller/src/test/resources/morfologik/speller/test-utf-spell.dict
diff --git a/morfologik-speller/src/test/resources/morfologik/speller/test-utf-spell.info b/morfologik-speller/src/test/resources/morfologik/speller/test-utf-spell.info
new file mode 100644
index 0000000..b13d12f
--- /dev/null
+++ b/morfologik-speller/src/test/resources/morfologik/speller/test-utf-spell.info
@@ -0,0 +1,15 @@
+#
+# Dictionary properties.
+# UTF-8 encoding or native2ascii has to be used for non-ASCII data.
+#
+
+fsa.dict.separator=+
+fsa.dict.encoding=utf-8
+
+fsa.dict.uses-prefixes=false
+fsa.dict.uses-infixes=false
+
+fsa.dict.speller.locale=pl_PL
+fsa.dict.speller.ignore-diacritics=true
+fsa.dict.speller.equivalent-chars=x ź, l ł, u ó, ó u
+fsa.dict.speller.replacement-pairs=rz ż, ż rz, ch h, h ch, ę en, en ę
diff --git a/morfologik-speller/src/test/resources/morfologik/speller/test_freq_iso.dict b/morfologik-speller/src/test/resources/morfologik/speller/test_freq_iso.dict
new file mode 100644
index 0000000..69c5a99
--- /dev/null
+++ b/morfologik-speller/src/test/resources/morfologik/speller/test_freq_iso.dict
diff --git a/morfologik-speller/src/test/resources/morfologik/speller/test_freq_iso.info b/morfologik-speller/src/test/resources/morfologik/speller/test_freq_iso.info
new file mode 100644
index 0000000..353deac
--- /dev/null
+++ b/morfologik-speller/src/test/resources/morfologik/speller/test_freq_iso.info
@@ -0,0 +1,16 @@
+#
+# Dictionary properties.
+#
+
+fsa.dict.separator=+
+fsa.dict.encoding=iso-8859-2
+
+fsa.dict.uses-prefixes=false
+fsa.dict.uses-infixes=false
+
+fsa.dict.frequency-included=true
+
+fsa.dict.speller.locale=pl_PL
+fsa.dict.speller.ignore-diacritics=true
+fsa.dict.speller.equivalent-chars=x ź, l ł, u ó, ó u
+fsa.dict.speller.replacement-pairs=ź zi, ł eu, ć ci, ć dż, ć dź, ć dz, c dz, ch h, ci ć, cz czy, dź ć, dź dzi, dż ć, dz ć, dzi dź, edzil ędził, ę em, ę en, ei eja, eja ei, em ę, en ę, eu ł, h ch, he chę, śi ś, ii ija, ija ii, iosc ość, ise się, loz łos, ni ń, ńi ń, ń ni, ą oł, oł ą, oi oja, oja oi, ą om, om ą, ą on, on ą, ru kró, ż rz, rz ż, rz sz, scia ścią, ś si, si ś, sić ść, s sną, sz ż, sz rz, tro rot, u y, wu wy, yi yja, yja yi, zal rzał, zekac rzekać, zi ź, zl azł, z żn, z rz, chłopcowi chłopcu, bratowi bratu, aleji alei, lubieć lubić, nei nie, źmie zmie, piatek piątek, pokuj pokój, poszłem poszedłem, prosze proszę, rząda żąda, sa są, sei się, standart standard, trzcionk czcionk, szłem szedłem, pry przy
+\ No newline at end of file