1 files changed, 30 insertions, 0 deletions
diff --git a/spellchecker/src/com/intellij/spellchecker/tokenizer/Tokenizer.java b/spellchecker/src/com/intellij/spellchecker/tokenizer/Tokenizer.java
new file mode 100644
index 00000000..0c55bf16
--- /dev/null
+++ b/spellchecker/src/com/intellij/spellchecker/tokenizer/Tokenizer.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2000-2009 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.spellchecker.tokenizer;
+
+import com.intellij.openapi.util.TextRange;
+import com.intellij.psi.PsiElement;
+import org.jetbrains.annotations.NotNull;
+
+public abstract class Tokenizer<T extends PsiElement> {
+
+  public abstract void tokenize(@NotNull T element, TokenConsumer consumer);
+
+  @NotNull
+  public TextRange getHighlightingRange(PsiElement element, int offset, TextRange textRange) {
+    return TextRange.from(offset + textRange.getStartOffset(), textRange.getLength());
+  }
+}