### Question:
Tokenizer { public static List<Token> tokenize(final String str) { return new Tokenizer(str).tokenize(); } private  Tokenizer(final String str); static List<Token> tokenize(final String str);  }### Answer:
@Test public void testSelectByVariable() { final List<Token> tokens = Tokenizer.tokenize("SELECT w FROM Widget w WHERE id = :id"); assertNotNull(tokens); assertEquals(9, tokens.size()); final List<Token> expected = Arrays.asList( new Token(TokenType.KEYWORD_SELECT, "SELECT", 1, 1), new Token(TokenType.SYMBOL, "w", 1, 8), new Token(TokenType.KEYWORD_FROM, "FROM", 1, 10), new Token(TokenType.SYMBOL, "Widget", 1, 15), new Token(TokenType.SYMBOL, "w", 1, 22), new Token(TokenType.KEYWORD_WHERE, "WHERE", 1, 24), new Token(TokenType.SYMBOL, "id", 1, 30), new Token(TokenType.EQUALS, "=", 1, 33), new Token(TokenType.NAMED_PARAMETER, "id", 1, 35)); assertEquals(expected, tokens); }
@Test public void testSelectByNumber() { final List<Token> tokens = Tokenizer.tokenize("SELECT w FROM Widget w WHERE id = 123"); assertNotNull(tokens); assertEquals(9, tokens.size()); final List<Token> expected = Arrays.asList( new Token(TokenType.KEYWORD_SELECT, "SELECT", 1, 1), new Token(TokenType.SYMBOL, "w", 1, 8), new Token(TokenType.KEYWORD_FROM, "FROM", 1, 10), new Token(TokenType.SYMBOL, "Widget", 1, 15), new Token(TokenType.SYMBOL, "w", 1, 22), new Token(TokenType.KEYWORD_WHERE, "WHERE", 1, 24), new Token(TokenType.SYMBOL, "id", 1, 30), new Token(TokenType.EQUALS, "=", 1, 33), new Token(TokenType.NUMBER, "123", 1, 35)); assertEquals(expected, tokens); }
@Test public void testSelectByString() { final List<Token> tokens = Tokenizer.tokenize("SELECT w FROM Widget w WHERE id = 'xyz'"); assertNotNull(tokens); assertEquals(9, tokens.size()); final List<Token> expected = Arrays.asList( new Token(TokenType.KEYWORD_SELECT, "SELECT", 1, 1), new Token(TokenType.SYMBOL, "w", 1, 8), new Token(TokenType.KEYWORD_FROM, "FROM", 1, 10), new Token(TokenType.SYMBOL, "Widget", 1, 15), new Token(TokenType.SYMBOL, "w", 1, 22), new Token(TokenType.KEYWORD_WHERE, "WHERE", 1, 24), new Token(TokenType.SYMBOL, "id", 1, 30), new Token(TokenType.EQUALS, "=", 1, 33), new Token(TokenType.STRING, "xyz", 1, 35)); assertEquals(expected, tokens); }