public class WhitespaceWordTokenizer extends AbstractWordTokenizer implements WordTokenizer
abbreviations, aposTokens, apostropheCanBeQuote, coalesceAsterisks, coalesceHyphens, contractions, contractionsURL, hyphensMatcher, hyphensPattern, logger, preTokenizer
Constructor and Description |
---|
WhitespaceWordTokenizer()
Create a word tokenizer than breaks on whitespace only.
|
Modifier and Type | Method and Description |
---|---|
java.util.List<java.lang.String> |
extractWords(java.lang.String text)
Break text into word tokens.
|
addWordToSentence, findWordOffsets, getLogger, getPreTokenizer, isClosingQuote, isLetterOrSingleQuote, isMultipleHyphens, isSingleOpeningQuote, loadContractions, preprocessToken, setAbbreviations, setAposTokens, setLogger, setPreTokenizer, splitToken
close
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
addWordToSentence, close, findWordOffsets, getPreTokenizer, preprocessToken, setAbbreviations, setAposTokens, setPreTokenizer
close
public WhitespaceWordTokenizer()
public java.util.List<java.lang.String> extractWords(java.lang.String text)
extractWords
in interface WordTokenizer
extractWords
in class AbstractWordTokenizer
text
- Text to break into word tokens.