### Question:
TwokenizeTokenizer implements TextTokenizer { @Override public Iterator<Token> iterateTokens(final String text) { final Iterator<String> tokens = Twokenize.tokenizeRawTweetText(text).iterator(); return new AbstractIterator<Token>() { int endPosition = 0; @Override protected Token getNext() throws Finished { if (tokens.hasNext()) { String token = tokens.next(); int startPosition = text.indexOf(token, endPosition); if (startPosition == -1) { startPosition = endPosition + 1; } endPosition = startPosition + token.length(); return new ImmutableToken(startPosition, token); } throw FINISHED; } }; }  @Override Iterator<Token> iterateTokens(final String text);  }### Answer:
@Test public void testTwokenizeTokenizer() { Iterator<Token> spans = tokenizer.iterateTokens(TWEET); int i = 0; while (spans.hasNext()) { Token span = spans.next(); assertEquals(TOKENS[i++], span.getValue()); } }
@Test public void testTwokenizeProblem() { tokenizer.iterateTokens(TWEET2); }