### Question:
SqlTokenizer { public String next() { if (head >= chars.length) return null; doubleQuote = false; singleQuote = false; backQuote = false; escape = false; boolean done = false; int parens = 0; for (; head < chars.length && !done; head++) { char c = chars[head]; switch (c) { case '\\': token.append(c); escape = !escape; continue; case '(': if (!escaped() && !quoted()) { if (parens == 0 && token.length() > 0) { head--; done = true; break; } parens += 1; } token.append(c); continue; case ')': if (!escaped() && !quoted()) { parens -= 1; if (parens == 0) { token.append(c); done = true; break; } } token.append(c); continue; case '\"': if (!(escape || singleQuote || backQuote || parens > 0)) { if (!doubleQuote && token.length() > 0) { head--; done = true; break; } doubleQuote = !doubleQuote; if (!doubleQuote) { token.append(c); done = true; break; } } token.append(c); continue; case '\'': if (!(escape || doubleQuote || backQuote || parens > 0)) { if (!singleQuote && token.length() > 0) { head--; done = true; break; } singleQuote = !singleQuote; if (!singleQuote) { token.append(c); done = true; break; } } token.append(c); continue; case '`': if (!(escape || doubleQuote || singleQuote || parens > 0)) { if (!backQuote && token.length() > 0) { head--; done = true; break; } backQuote = !backQuote; if (!backQuote) { token.append(c); done = true; break; } } token.append(c); continue; default: escape = false; if (quoted() || parens > 0) { token.append(c); continue; } if (token.length() > 0) { char previousC = token.charAt(token.length() - 1); if (!isAlphaNum(previousC)) { head--; done = true; break; } else if (isAlphaNum(previousC) && !isAlphaNum(c)) { head--; done = true; break; } } token.append(c); } } String str = token.toString(); token = new StringBuilder(); return str; } SqlTokenizer(String chars); String nextClause(); String next();  }### Answer:
@Test public void testTokens() throws Exception { String[][] tests = new String[][]{ { "SELECT * FROM (SELECT *, 'Threatened' as 'Victim' FROM Entry WHERE threatened = true UNION SELECT *, 'Tortured' as 'Victim' FROM Entry WHERE tortured = true UNION SELECT *, 'Taken Captive' as 'Victim' FROM Entry WHERE captive = true )as t", "SELECT| |*| |FROM| |(SELECT *, 'Threatened' as 'Victim' FROM Entry WHERE threatened = true UNION SELECT *, 'Tortured' as 'Victim' FROM Entry WHERE tortured = true UNION SELECT *, 'Taken Captive' as 'Victim' FROM Entry WHERE captive = true )|as| |t" } }; for (String[] test : tests) { String input = test[0]; List<String> expected = Arrays.asList(test[1].split("\\|")); String next; List<String> actual = new ArrayList<>(); SqlTokenizer tok = new SqlTokenizer(input); while ((next = tok.next()) != null) { actual.add(next); } if (!Utils.testCompare(expected.toString(), actual.toString())) fail(); } }