### Question:
KoreanCharacterTokenizer extends Tokenizer { @Override public final boolean incrementToken() throws IOException { clearAttributes(); logger.debug("incrementToken"); int length = 0; int start = -1; int end = -1; char[] buffer = termAtt.buffer(); while (true) { if (bufferIndex >= dataLen) { offset += dataLen; if(!charUtils.fill(ioBuffer, input)) { dataLen = 0; if (length > 0) { break; } else { finalOffset = correctOffset(offset); return false; } } dataLen = ioBuffer.getLength(); bufferIndex = 0; } final int c = charUtils.codePointAt(ioBuffer.getBuffer(), bufferIndex); final int charCount = Character.charCount(c); bufferIndex += charCount; if (isTokenChar(c)) { if (length > 0) { if(Character.isDigit(preChar)) preCharType = this.DIGIT; else if(preChar < 127) preCharType = this.ALPHA; else preCharType = this.KOREAN; if(Character.isDigit(c)) nowCharType = this.DIGIT; else if(c < 127) nowCharType = this.ALPHA; else nowCharType = this.KOREAN; if(preCharType != nowCharType) { bufferIndex--; termAtt.setLength(length); offsetAtt.setOffset(correctOffset(start), correctOffset(start+length)); typeAtt.setType("word"); positionAtt.setPositionIncrement(1); return true; } } preChar = c; if (length == 0) { assert start == -1; start = offset + bufferIndex - charCount; end = start; } else if (length >= buffer.length-1) { buffer = termAtt.resizeBuffer(2+length); } end += charCount; length += Character.toChars(normalize(c), buffer, length); if (length >= MAX_WORD_LEN) break; } else if (length > 0) { break; } } termAtt.setLength(length); assert start != -1; offsetAtt.setOffset(correctOffset(start), finalOffset = correctOffset(end)); typeAtt.setType("word"); positionAtt.setPositionIncrement(1); return true; } KoreanCharacterTokenizer(Reader input);  KoreanCharacterTokenizer(Version matchVersion, Reader input);  KoreanCharacterTokenizer(Version matchVersion, AttributeSource source,
			Reader input);  KoreanCharacterTokenizer(Version matchVersion, AttributeFactory factory,
			Reader input); @Override final boolean incrementToken(); @Override final void end(); @Override void reset();  }### Answer:
@Test public void testIncrementToken() throws IOException { CharTermAttribute charTermAtt = tokenizer.getAttribute(CharTermAttribute.class); OffsetAttribute offSetAtt = tokenizer.getAttribute(OffsetAttribute.class); while(tokenizer.incrementToken()) { TestToken t = getToken(charTermAtt.toString(), offSetAtt.startOffset(), offSetAtt.endOffset()); System.out.println("termAtt.term() : " + charTermAtt.toString()); System.out.println("offSetAtt : " + offSetAtt.startOffset()); System.out.println("offSetAtt : " + offSetAtt.endOffset()); Assert.assertTrue(tokenizedToken.contains(t)); } }