### Question:
TagTokenizer implements Tokenizer { @Override @NonNull public List<Range> findTokenRanges(CharSequence charSequence, int start, int end) { ArrayList<Range>result = new ArrayList<>(); if (start == end) { return result; } int tokenStart = Integer.MAX_VALUE; for (int cursor = start; cursor < end; ++cursor) { char character = charSequence.charAt(cursor); if (isTokenTerminator(character)) { if (cursor - 1 > tokenStart) { result.add(new Range(tokenStart, cursor)); } tokenStart = Integer.MAX_VALUE; } if (tagPrefixes.contains(character)) { tokenStart = cursor; } } if (end > tokenStart) { result.add(new Range(tokenStart, end)); } return result; } TagTokenizer();  TagTokenizer(List<Character> tagPrefixes); @SuppressWarnings({"WeakerAccess", "unchecked"})  TagTokenizer(Parcel in); @Override boolean containsTokenTerminator(CharSequence charSequence); @Override @NonNull List<Range> findTokenRanges(CharSequence charSequence, int start, int end); @Override @NonNull CharSequence wrapTokenValue(CharSequence text); @Override int describeContents(); @Override void writeToParcel(Parcel parcel, int i); static final Parcelable.Creator<TagTokenizer> CREATOR; }### Answer:
@Test public void testBasicTagDetection() { String test = "@bears #tokens"; List<Range> ranges = tokenizer.findTokenRanges(test, 0, test.length()); assertEquals(Arrays.asList(new Range(0, 6), new Range(7, 14)), ranges); assertEquals("@bears", test.substring(ranges.get(0).start, ranges.get(0).end)); assertEquals("#tokens", test.substring(ranges.get(1).start, ranges.get(1).end)); }
@Test public void testSequentialTagDetection() { String test = "@@bears#tokens#"; assertEquals(Arrays.asList(new Range(1,7), new Range(7,14), new Range(14, 15)), tokenizer.findTokenRanges(test, 0, test.length())); }
@Test public void testNonTokenInput() { String test = "This is some input with @names and #hash #tags inside"; assertEquals(Arrays.asList(new Range(24,30), new Range(35,40), new Range(41,46)), tokenizer.findTokenRanges(test, 0, test.length())); }
@Test public void testMissingTokenContentInput() { String test = "@token @ asdm @asjdfhajks sdfasdf"; assertEquals(Arrays.asList(new Range(0,6), new Range(25, 36)), tokenizer.findTokenRanges(test, 0, test.length())); }
@Test public void allowsOneCharacterCandidateRangeMatches() { String text = "#"; List<Range> ranges = tokenizer.findTokenRanges(text, 0, text.length()); assertEquals(Collections.singletonList(new Range(0,1)), ranges); }
@Test public void allowsOneCharacterCandidateRangeMatchesWithWhitespace() { String text = " #"; List<Range> ranges = tokenizer.findTokenRanges(text, 0, text.length()); assertEquals(Collections.singletonList(new Range(1,2)), ranges); }
@Test public void matchesSingleLetterTokens() { String text = "#t#r #a##b"; List<Range> ranges = tokenizer.findTokenRanges(text, 0, text.length()); assertEquals(Arrays.asList(new Range(0, 2), new Range(2,4), new Range(5,7), new Range(8,10)), ranges); }