### Question:
Tokenizer { public static List<String> tokenize(String inputString) { Iterator<Token> tokenIterator = new WordTokenizer().iterateTokens(inputString); return CollectionHelper.newArrayList(CollectionHelper.convert(tokenIterator, Token.VALUE_CONVERTER)); } private  Tokenizer(); static List<String> tokenize(String inputString); static Set<String> calculateCharNGrams(String string, int n); static Set<String> calculateAllCharEdgeNGrams(String string, int n1, int n2); static Set<String> calculateAllCharEdgeNGrams(String string, int n1, int n2, boolean mustHitLeftEdge); static Set<String> calculateCharEdgeNGrams(String string, int n, boolean mustHitLeftEdge); static Set<String> calculateWordNGrams(String string, int n); static List<String> calculateWordNGramsAsList(String string, int n); static Set<String> calculateAllCharNGrams(String string, int n1, int n2); static Set<String> calculateAllWordNGrams(String string, int n1, int n2); static Set<List<String>> computeSplits(String string, int n1, int n2, int maxSplits); static List<String> computeStartingWordNGrams(String string, int n1, int n2); static String getSentence(String string, int position); static List<String> getSentences(String inputText, boolean onlyRealSentences); static List<String> getSentences(String inputText, boolean onlyRealSentences, Language language); static List<String> getSentences(String inputText); static List<String> getSentences(String inputText, Language language); static String getPhraseFromBeginningOfSentence(String inputString); static String getPhraseToEndOfSentence(String string); static void main(String[] args); static final String TOKEN_SPLIT_REGEX; static final String SENTENCE_SPLIT_REGEX_EN; static final String SENTENCE_SPLIT_REGEX_DE; }### Answer:
@Test public void testTokenize() { List<String> tokens = Tokenizer.tokenize("That poster costs $22.40. twenty-one."); assertEquals(7, tokens.size()); tokens = Tokenizer.tokenize("Mr. <MUSICIAN>John Hiatt</MUSICIAN> is awesome."); assertEquals(8, tokens.size()); tokens = Tokenizer.tokenize("Mr. '<MUSICIAN>John Hiatt</MUSICIAN>' is awesome."); assertEquals(10, tokens.size()); tokens = Tokenizer.tokenize("Mr. ^<MUSICIAN>John Hiatt</MUSICIAN>) is awesome!!!"); assertEquals(10, tokens.size()); tokens = Tokenizer.tokenize("asp.net is very web 2.0. isn't it? web2.0, .net"); assertEquals(14, tokens.size()); tokens = Tokenizer.tokenize("40,000 residents"); assertEquals(2, tokens.size()); tokens = Tokenizer .tokenize("The United States of America are often called the USA, the U.S.A., or simply the U.S. The U.N. has its headquarter in N.Y.C. on the east coast."); assertEquals(30, tokens.size()); }