### Question:
TurkishTokenizer { public List<Token> tokenize(File file) throws IOException { return getAllTokens(lexerInstance(CharStreams.fromPath(file.toPath()))); } private  TurkishTokenizer(long acceptedTypeBits); static Builder builder(); boolean isTypeAccepted(Token.Type i); boolean isTypeIgnored(Token.Type i); List<Token> tokenize(File file); List<Token> tokenize(String input); List<Token> tokenize(Reader reader); List<String> tokenizeToStrings(String input); Iterator<Token> getTokenIterator(String input); Iterator<Token> getTokenIterator(File file); Iterator<Token> getTokenIterator(Reader reader); static Token convert(org.antlr.v4.runtime.Token token); static Token convert(org.antlr.v4.runtime.Token token, Token.Type type); static Token.Type convertType(org.antlr.v4.runtime.Token token); static final TurkishTokenizer ALL; static final TurkishTokenizer DEFAULT; }### Answer:
@Test public void testTokenBoundaries() { TurkishTokenizer t = TurkishTokenizer.ALL; List<Token> tokens = t.tokenize("bir av. geldi."); Token t0 = tokens.get(0); Assert.assertEquals("bir", t0.getText()); Assert.assertEquals(0, t0.getStart()); Assert.assertEquals(2, t0.getEnd()); Token t1 = tokens.get(1); Assert.assertEquals(" ", t1.getText()); Assert.assertEquals(3, t1.getStart()); Assert.assertEquals(3, t1.getEnd()); Token t2 = tokens.get(2); Assert.assertEquals("av.", t2.getText()); Assert.assertEquals(4, t2.getStart()); Assert.assertEquals(6, t2.getEnd()); Token t3 = tokens.get(3); Assert.assertEquals(" ", t3.getText()); Assert.assertEquals(7, t3.getStart()); Assert.assertEquals(7, t3.getEnd()); Token t4 = tokens.get(4); Assert.assertEquals("geldi", t4.getText()); Assert.assertEquals(8, t4.getStart()); Assert.assertEquals(12, t4.getEnd()); Token t5 = tokens.get(5); Assert.assertEquals(".", t5.getText()); Assert.assertEquals(13, t5.getStart()); Assert.assertEquals(13, t5.getEnd()); }
@Test @Ignore("Not an actual test. Requires external data.") public void performance() throws IOException { TurkishTokenizer tokenizer = TurkishTokenizer.DEFAULT; for (int it = 0; it < 5; it++) { List<String> lines = Files.readAllLines( Paths.get("/media/aaa/Data/aaa/corpora/dunya.100k")); Stopwatch clock = Stopwatch.createStarted(); long tokenCount = 0; for (String line : lines) { List<Token> tokens = tokenizer.tokenize(line); tokenCount += tokens.size(); } long elapsed = clock.elapsed(TimeUnit.MILLISECONDS); Log.info("Token count = %d ", tokenCount); Log.info("Speed (tps) = %.1f", tokenCount * 1000d / elapsed); } }