### Question:
Token { public static List<Token> tokenize(final String input) { List<Token> result = new ArrayList<Token>(); if (input == null) { return result; } final int WHITESPACE = 0; final int WORD = 1; final int STRINGDQ = 2; final int STRINGSQ = 3; final int COMMENT = 4; final int ESCAPE = 5; int state = WHITESPACE; int lastState = state; char ch; int tokenIndex = -1; StringBuilder token = new StringBuilder(""); for (int i = 0; i < input.length(); i++) { ch = input.charAt(i); switch (state) { case WHITESPACE: if (Character.isWhitespace(ch)) { } else if (Character.isLetterOrDigit(ch) || ch == '_') { state = WORD; tokenIndex = i; token.append(ch); } else if (ch == '"') { state = STRINGDQ; tokenIndex = i; } else if (ch == '\'') { state = STRINGSQ; tokenIndex = i; } else if (ch == '#') { state = COMMENT; } else if (ch == '\\' && i < input.length()-1) { token.append(input.charAt(i+1)); tokenIndex = i; i++; } else { state = WORD; tokenIndex = i; token.append(ch); } break; case WORD: if (Character.isWhitespace(ch)) { result.add(new Token(tokenIndex, token.toString())); token.setLength(0); state = WHITESPACE; } else if (Character.isLetterOrDigit(ch) || ch == '_') { token.append(ch); } else if (ch == '"') { state = STRINGDQ; } else if (ch == '\'') { state = STRINGSQ; } else if (ch == '#') { result.add(new Token(tokenIndex, token.toString())); token.setLength(0); state = COMMENT; } else if (ch == '\\') { lastState = state; state = ESCAPE; } else { token.append(ch); } break; case STRINGDQ: if (ch == '"') { state = WORD; } else if (ch == '\\') { lastState = state; state = ESCAPE; } else { token.append(ch); } break; case STRINGSQ: if (ch == '\'') { state = WORD; } else if (ch == '\\') { lastState = state; state = ESCAPE; } else { token.append(ch); } break; case COMMENT: break; case ESCAPE: token.append(ch); state = lastState; break; default: assert false : "Unknown state in Shell.tokenize() state machine"; break; } } if (state == WORD || state == STRINGDQ || state == STRINGSQ || (state == WHITESPACE && !token.toString().trim().isEmpty())) { result.add(new Token(tokenIndex, token.toString())); } return result; } Token(int index, String string); final int getIndex(); final String getString(); @Override String toString(); @Override boolean equals(Object obj); @Override int hashCode(); static List<Token> tokenize(final String input); static boolean isCustomizedParamOrder(List<Token> tokens); static String escapeString(String input);  }### Answer:
@Test public void testTokenize() { String[] cases = { "", "aSingleToken", "a b c", "an's g'ri # quotation test", "Shell instance = new Shell(new ShellTest(), System.out", "dir \"E:\\ASG\\!dynamic\\projects\" \t-l 3492.9 ", "a b c ''", " \\\" ", "--b bValue --a aValue", "command1 param1 && command2 param2", "command1 param1 ; command2 param2" }; int[] sizes = { 0, 1, 3, 1, 7, 4, 4, 1, 4, 5, 5}; for (int i = 0; i < cases.length; i++) { List<Token> result = Token.tokenize(cases[i]); assertEquals(sizes[i], result.size()); } }