### Question:
PageAnalyzer { public static HtmlAnalysisResult analyze(Map<String, String> config, String url) { try { String userAgent = config.getOrDefault(CONFIG_USER_AGENT, DEFAULT_USER_AGENT); HttpResponse<String> response = Unirest.get(url) .header("User-Agent", userAgent) .asString(); return analyze(config, url, response.getBody(), response.getStatus(), response.getHeaders()); } catch (UnirestException e) { throw new RuntimeException(e); } }  static HtmlAnalysisResult analyze(Map<String, String> config, String url); static HtmlAnalysisResult analyze(Map<String, String> config, String url, String html); static HtmlAnalysisResult analyze(Map<String, String> config, String url, String html, Integer status, Map<String, List<String>> headers); static void parseRobotsTxt(String userAgent, String robotsUrl, String robotsTxt, HtmlAnalysisResult result); static final String CONFIG_USER_AGENT; static final String CONFIG_ANALYZE_ROBOTS_TXT; }### Answer:
@Test public void headersAndStatus() { Map<String, List<String>> headers = Maps.newHashMap(); headers.put("Etag", Lists.newArrayList("c1dc8d7be85325149", "ed5fc4d62b84752")); headers.put("Date", Lists.newArrayList("Wed, 11 Jan 2017 13:00:18 GMT")); HashMap<String, String> config = Maps.newHashMap(); HtmlAnalysisResult result = PageAnalyzer.analyze(config, "http: assertEquals(new Integer(200), result.getHttpStatus()); assertEquals(2, result.getHeaders().size()); assertEquals("c1dc8d7be85325149\ned5fc4d62b84752", result.getHeaders().get("Etag")); assertEquals("Wed, 11 Jan 2017 13:00:18 GMT", result.getHeaders().get("Date")); }
@Test public void htmlParsing() { HashMap<String, String> config = Maps.newHashMap(); HtmlAnalysisResult result = PageAnalyzer.analyze(config, "https: assertEquals("Bloomberg.com", result.getTitle()); assertEquals(33, result.getMetaValues().size()); assertTrue(result.getMetaValues().contains("<meta property=\"og:url\" content=\"https: assertEquals(361, result.getLinks().size()); assertTrue(result.getLinks().contains("https: }
@Test @Ignore public void fetchAndParse() { HashMap<String, String> config = Maps.newHashMap(); config.put(PageAnalyzer.CONFIG_ANALYZE_ROBOTS_TXT, "true"); HtmlAnalysisResult result = PageAnalyzer.analyze(config, "http: assertEquals("TokenMill - Natural Language Processing", result.getTitle()); assertEquals(10, result.getMetaValues().size()); assertEquals(42, result.getLinks().size()); assertTrue(result.getLinks().contains("http: assertTrue(result.getRobotsAllowedAll()); assertFalse(result.getRobotsAllowedNone()); assertTrue(result.getRobotsAllowedHome()); assertEquals(Lists.newArrayList(), result.getRobotsSitemaps()); assertEquals(Long.MIN_VALUE, (long) result.getRobotsCrawlDelay()); }
@Test @Ignore public void fetchAndParseRobotsTxt() { HashMap<String, String> config = Maps.newHashMap(); config.put(PageAnalyzer.CONFIG_ANALYZE_ROBOTS_TXT, "true"); HtmlAnalysisResult result = PageAnalyzer.analyze(config, "https: assertFalse(result.getRobotsAllowedAll()); assertFalse(result.getRobotsAllowedNone()); assertTrue(result.getRobotsAllowedHome()); assertTrue(result.getRobotsSitemaps().contains("http: assertEquals(Long.MIN_VALUE, (long) result.getRobotsCrawlDelay()); }