### Question:
WhiteSpaceTokenizer implements RedPenTokenizer { @Override public List<TokenElement> tokenize(String content) { List<TokenElement> tokens = new ArrayList<>(); String surface = ""; int offset = 0; List<String> tags = new ArrayList<>(); for (int i = 0, l = content.length(); i < l; i++) { char ch = content.charAt(i); if ((DELIMITERS.indexOf(ch) != -1) || isWrappedDelimiters(content, i)) { if (isSuitableToken(surface)) { tokens.add(new TokenElement(surface, tags, offset)); } if (!isWhitespace(ch) && ch != '\u00A0') { tokens.add(new TokenElement(String.valueOf(ch), tags, i)); } surface = ""; offset = -1; } else { if (offset < 0) { offset = i; } surface += ch; } } if (isSuitableToken(surface)) { tokens.add(new TokenElement(surface, tags, offset)); } return tokens; } WhiteSpaceTokenizer(); @Override List<TokenElement> tokenize(String content);  }### Answer:
@Test void testTokenize() { RedPenTokenizer tokenizer = new WhiteSpaceTokenizer(); List<TokenElement> results = tokenizer.tokenize("this is a\u00A0pen"); assertEquals(4, results.size()); assertEquals("this", results.get(0).getSurface()); assertEquals(0, results.get(0).getTags().size()); assertEquals("is", results.get(1).getSurface()); assertEquals(0, results.get(1).getTags().size()); assertEquals("a", results.get(2).getSurface()); assertEquals(0, results.get(2).getTags().size()); assertEquals("pen", results.get(3).getSurface()); assertEquals(0, results.get(3).getTags().size()); }
@Test void testTokenizeSentenceWithNoSpaceBracket() { RedPenTokenizer tokenizer = new WhiteSpaceTokenizer(); List<TokenElement> results = tokenizer.tokenize("distributed(cluster) systems are good"); assertEquals(7, results.size()); assertEquals("distributed", results.get(0).getSurface()); assertEquals(0, results.get(0).getTags().size()); assertEquals("cluster", results.get(2).getSurface()); assertEquals(0, results.get(1).getTags().size()); assertEquals("systems", results.get(4).getSurface()); assertEquals(0, results.get(2).getTags().size()); assertEquals("are", results.get(5).getSurface()); assertEquals(0, results.get(3).getTags().size()); assertEquals("good", results.get(6).getSurface()); assertEquals(0, results.get(4).getTags().size()); }
@Test void testTokenizeSentenceEndsWithPeriod() { RedPenTokenizer tokenizer = new WhiteSpaceTokenizer(); List<TokenElement> results = tokenizer.tokenize("I am an engineer."); assertEquals(5, results.size()); assertEquals("I", results.get(0).getSurface()); assertEquals("am", results.get(1).getSurface()); assertEquals("an", results.get(2).getSurface()); assertEquals("engineer", results.get(3).getSurface()); assertEquals(".", results.get(4).getSurface()); }
@Test void testTokenizeSentenceWithContraction() { RedPenTokenizer tokenizer = new WhiteSpaceTokenizer(); List<TokenElement> results = tokenizer.tokenize("I'm an engineer"); assertEquals(3, results.size()); assertEquals("I'm", results.get(0).getSurface()); assertEquals("an", results.get(1).getSurface()); assertEquals("engineer", results.get(2).getSurface()); }