### Question:
CompletionTokenizer { public List<Token> tokenize(String input) { return tokenizeSubstring(input, 0, input.length()); } CompletionTokenizer(TokenCompletion spec); List<Token> tokenize(String input); List<Token> tokenizeSubstring(String input, int beginIndex, int rightBound);  }### Answer:
@Test public void oneToken() { List<Token> tokens = tokenizer.tokenize("!"); assertTokensEqual(of(FACTORIAL), tokens); }
@Test public void emptyInput() { List<Token> tokens = tokenizer.tokenize(""); assertTrue(tokens.isEmpty()); }
@Test public void blankInput() { List<Token> tokens = tokenizer.tokenize(" \t "); assertTrue(tokens.isEmpty()); }
@Test public void simpleExpression() { List<Token> tokens = tokenizer.tokenize("1+2"); assertTokensEqual(of(integer(1), PLUS, integer(2)), tokens); }
@Test public void simpleExpressionWithWhitespaces() { List<Token> tokens = tokenizer.tokenize(" 1\t * 2"); assertTokensEqual(of(integer(1), MUL, integer(2)), tokens); }
@Test public void incrementAndPlus() { List<Token> tokens = tokenizer.tokenize("+++"); assertTokensEqual(of(INCREMENT, PLUS), tokens); }
@Test public void valueTokens() { List<Token> tokens = tokenizer.tokenize("value * aaaa + posValue"); assertTokensEqual(of(value(), MUL, complex(), PLUS, pos()), tokens); }
@Test public void emptyStrings() { List<Token> tokens = tokenizer.tokenize("\"\"\"\" '' ''"); assertTokensEqual(of(doubleQtd(""), doubleQtd(""), singleQtd(""), singleQtd("")), tokens); }
@Test public void stringLiterals() { final String strBody = "These items must not be tokenized: 10 + 17! * id.value"; List<Token> tokens = tokenizer.tokenize("id + \"" + strBody + "\"'" + strBody + "'"); assertTokensEqual(of(ID, PLUS, doubleQtd(strBody), singleQtd(strBody)), tokens); }
@Test public void nestedQuotesOfDifferentKind() { List<Token> tokens = tokenizer.tokenize("\"'\"'\"'"); assertTokensEqual(of(doubleQtd("'"), singleQtd("\"")), tokens); }
@Test public void tripleQuotedString() { List<Token> tokens = tokenizer.tokenize("'''x'''"); assertTokensEqual(of(tripleQtd("x")), tokens); }
@Test public void longCorrectExpression() { List<Token> tokens = tokenizer.tokenize("\t( 10) * 5! + id.value ++ + '\"text 1\"' + \"text 2\""); assertTokensEqual(of(LP, integer(10), RP, MUL, integer(5), FACTORIAL, PLUS, ID, DOT, value(), INCREMENT, PLUS, singleQtd("\"text 1\""), PLUS, doubleQtd("text 2")), tokens); }
@Test public void oneComment() { List<Token> tokens = tokenizer.tokenize("#comment"); assertTokensEqual(of(comment("comment")), tokens); }