Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.nutch.fetcher; import java.io.IOException; import java.net.MalformedURLException; import java.text.SimpleDateFormat; import java.util.Map.Entry; // Slf4j Logging imports import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.io.*; import org.apache.hadoop.fs.*; import org.apache.hadoop.conf.*; import org.apache.hadoop.mapred.*; import org.apache.hadoop.util.*; import org.apache.nutch.crawl.CrawlDatum; import org.apache.nutch.crawl.NutchWritable; import org.apache.nutch.crawl.SignatureFactory; import org.apache.nutch.metadata.Metadata; import org.apache.nutch.metadata.Nutch; import org.apache.nutch.net.*; import org.apache.nutch.protocol.*; import org.apache.nutch.parse.*; import org.apache.nutch.scoring.ScoringFilters; import org.apache.nutch.util.*; /** The fetcher. Most of the work is done by plugins. */ public class OldFetcher extends Configured implements Tool, MapRunnable<WritableComparable, Writable, Text, NutchWritable> { public static final Logger LOG = LoggerFactory.getLogger(OldFetcher.class); public static final int PERM_REFRESH_TIME = 5; public static final String CONTENT_REDIR = "content"; public static final String PROTOCOL_REDIR = "protocol"; public static class InputFormat extends SequenceFileInputFormat<WritableComparable, Writable> { /** Don't split inputs, to keep things polite. */ public InputSplit[] getSplits(JobConf job, int nSplits) throws IOException { FileStatus[] files = listStatus(job); FileSystem fs = FileSystem.get(job); InputSplit[] splits = new InputSplit[files.length]; for (int i = 0; i < files.length; i++) { FileStatus cur = files[i]; splits[i] = new FileSplit(cur.getPath(), 0, cur.getLen(), (String[]) null); } return splits; } } private RecordReader<WritableComparable, Writable> input; private OutputCollector<Text, NutchWritable> output; private Reporter reporter; private String segmentName; private int activeThreads; private int maxRedirect; private long start = System.currentTimeMillis(); // start time of fetcher run private long lastRequestStart = start; private long bytes; // total bytes fetched private int pages; // total pages fetched private int errors; // total pages errored private boolean storingContent; private boolean parsing; private class FetcherThread extends Thread { private Configuration conf; private URLFilters urlFilters; private ScoringFilters scfilters; private ParseUtil parseUtil; private URLNormalizers normalizers; private ProtocolFactory protocolFactory; private boolean redirecting; private int redirectCount; private String reprUrl; public FetcherThread(Configuration conf) { this.setDaemon(true); // don't hang JVM on exit this.setName("FetcherThread"); // use an informative name this.conf = conf; this.urlFilters = new URLFilters(conf); this.scfilters = new ScoringFilters(conf); this.parseUtil = new ParseUtil(conf); this.protocolFactory = new ProtocolFactory(conf); this.normalizers = new URLNormalizers(conf, URLNormalizers.SCOPE_FETCHER); } public void run() { synchronized (OldFetcher.this) { activeThreads++; } // count threads try { Text key = new Text(); CrawlDatum datum = new CrawlDatum(); while (true) { // TODO : NUTCH-258 ... // If something bad happened, then exit // if (conf.getBoolean("fetcher.exit", false)) { // break; // ] try { // get next entry from input if (!input.next(key, datum)) { break; // at eof, exit } } catch (IOException e) { if (LOG.isErrorEnabled()) { LOG.error("fetcher caught:" + e.toString()); } break; } synchronized (OldFetcher.this) { lastRequestStart = System.currentTimeMillis(); } // url may be changed through redirects. Text url = new Text(key); Text reprUrlWritable = (Text) datum.getMetaData().get(Nutch.WRITABLE_REPR_URL_KEY); if (reprUrlWritable == null) { reprUrl = key.toString(); } else { reprUrl = reprUrlWritable.toString(); } try { if (LOG.isInfoEnabled()) { LOG.info("fetching " + url); } // fetch the page redirectCount = 0; do { if (LOG.isDebugEnabled()) { LOG.debug("redirectCount=" + redirectCount); } redirecting = false; Protocol protocol = this.protocolFactory.getProtocol(url.toString()); ProtocolOutput output = protocol.getProtocolOutput(url, datum); ProtocolStatus status = output.getStatus(); Content content = output.getContent(); ParseStatus pstatus = null; String urlString = url.toString(); if (reprUrl != null && !reprUrl.equals(urlString)) { datum.getMetaData().put(Nutch.WRITABLE_REPR_URL_KEY, new Text(reprUrl)); } switch (status.getCode()) { case ProtocolStatus.SUCCESS: // got a page pstatus = output(url, datum, content, status, CrawlDatum.STATUS_FETCH_SUCCESS); updateStatus(content.getContent().length); if (pstatus != null && pstatus.isSuccess() && pstatus.getMinorCode() == ParseStatus.SUCCESS_REDIRECT) { String newUrl = pstatus.getMessage(); int refreshTime = Integer.valueOf(pstatus.getArgs()[1]); url = handleRedirect(url, datum, urlString, newUrl, refreshTime < PERM_REFRESH_TIME, CONTENT_REDIR); } break; case ProtocolStatus.MOVED: // redirect case ProtocolStatus.TEMP_MOVED: int code; boolean temp; if (status.getCode() == ProtocolStatus.MOVED) { code = CrawlDatum.STATUS_FETCH_REDIR_PERM; temp = false; } else { code = CrawlDatum.STATUS_FETCH_REDIR_TEMP; temp = true; } output(url, datum, content, status, code); String newUrl = status.getMessage(); url = handleRedirect(url, datum, urlString, newUrl, temp, PROTOCOL_REDIR); break; // failures - increase the retry counter case ProtocolStatus.EXCEPTION: logError(url, status.getMessage()); /* FALLTHROUGH */ case ProtocolStatus.RETRY: // retry case ProtocolStatus.WOULDBLOCK: case ProtocolStatus.BLOCKED: output(url, datum, null, status, CrawlDatum.STATUS_FETCH_RETRY); break; // permanent failures case ProtocolStatus.GONE: // gone case ProtocolStatus.NOTFOUND: case ProtocolStatus.ACCESS_DENIED: case ProtocolStatus.ROBOTS_DENIED: output(url, datum, null, status, CrawlDatum.STATUS_FETCH_GONE); break; case ProtocolStatus.NOTMODIFIED: output(url, datum, null, status, CrawlDatum.STATUS_FETCH_NOTMODIFIED); break; default: if (LOG.isWarnEnabled()) { LOG.warn("Unknown ProtocolStatus: " + status.getCode()); } output(url, datum, null, status, CrawlDatum.STATUS_FETCH_GONE); } if (redirecting && redirectCount >= maxRedirect) { if (LOG.isInfoEnabled()) { LOG.info(" - redirect count exceeded " + url); } output(url, datum, null, status, CrawlDatum.STATUS_FETCH_GONE); } } while (redirecting && (redirectCount < maxRedirect)); } catch (Throwable t) { // unexpected exception logError(url, t.toString()); output(url, datum, null, null, CrawlDatum.STATUS_FETCH_RETRY); } } } catch (Throwable e) { if (LOG.isErrorEnabled()) { LOG.error("fetcher caught:" + e.toString()); } } finally { synchronized (OldFetcher.this) { activeThreads--; } // count threads } } private Text handleRedirect(Text url, CrawlDatum datum, String urlString, String newUrl, boolean temp, String redirType) throws MalformedURLException, URLFilterException { newUrl = normalizers.normalize(newUrl, URLNormalizers.SCOPE_FETCHER); newUrl = urlFilters.filter(newUrl); if (newUrl != null && !newUrl.equals(urlString)) { reprUrl = URLUtil.chooseRepr(reprUrl, newUrl, temp); url = new Text(newUrl); if (maxRedirect > 0) { redirecting = true; redirectCount++; if (LOG.isDebugEnabled()) { LOG.debug(" - " + redirType + " redirect to " + url + " (fetching now)"); } return url; } else { CrawlDatum newDatum = new CrawlDatum(); if (reprUrl != null) { newDatum.getMetaData().put(Nutch.WRITABLE_REPR_URL_KEY, new Text(reprUrl)); } output(url, newDatum, null, null, CrawlDatum.STATUS_LINKED); if (LOG.isDebugEnabled()) { LOG.debug(" - " + redirType + " redirect to " + url + " (fetching later)"); } return null; } } else { if (LOG.isDebugEnabled()) { LOG.debug(" - " + redirType + " redirect skipped: " + (newUrl != null ? "to same url" : "filtered")); } return null; } } private void logError(Text url, String message) { if (LOG.isInfoEnabled()) { LOG.info("fetch of " + url + " failed with: " + message); } synchronized (OldFetcher.this) { // record failure errors++; } } private ParseStatus output(Text key, CrawlDatum datum, Content content, ProtocolStatus pstatus, int status) { datum.setStatus(status); datum.setFetchTime(System.currentTimeMillis()); if (pstatus != null) datum.getMetaData().put(Nutch.WRITABLE_PROTO_STATUS_KEY, pstatus); ParseResult parseResult = null; if (content != null) { Metadata metadata = content.getMetadata(); // add segment to metadata metadata.set(Nutch.SEGMENT_NAME_KEY, segmentName); // add score to content metadata so that ParseSegment can pick it up. try { scfilters.passScoreBeforeParsing(key, datum, content); } catch (Exception e) { if (LOG.isWarnEnabled()) { LOG.warn("Couldn't pass score, url " + key + " (" + e + ")"); } } /* Note: Fetcher will only follow meta-redirects coming from the * original URL. */ if (parsing && status == CrawlDatum.STATUS_FETCH_SUCCESS) { try { parseResult = this.parseUtil.parse(content); } catch (Exception e) { LOG.warn("Error parsing: " + key + ": " + StringUtils.stringifyException(e)); } if (parseResult == null) { byte[] signature = SignatureFactory.getSignature(getConf()).calculate(content, new ParseStatus().getEmptyParse(conf)); datum.setSignature(signature); } } /* Store status code in content So we can read this value during * parsing (as a separate job) and decide to parse or not. */ content.getMetadata().add(Nutch.FETCH_STATUS_KEY, Integer.toString(status)); } try { output.collect(key, new NutchWritable(datum)); if (content != null && storingContent) output.collect(key, new NutchWritable(content)); if (parseResult != null) { for (Entry<Text, Parse> entry : parseResult) { Text url = entry.getKey(); Parse parse = entry.getValue(); ParseStatus parseStatus = parse.getData().getStatus(); if (!parseStatus.isSuccess()) { LOG.warn("Error parsing: " + key + ": " + parseStatus); parse = parseStatus.getEmptyParse(getConf()); } // Calculate page signature. For non-parsing fetchers this will // be done in ParseSegment byte[] signature = SignatureFactory.getSignature(getConf()).calculate(content, parse); // Ensure segment name and score are in parseData metadata parse.getData().getContentMeta().set(Nutch.SEGMENT_NAME_KEY, segmentName); parse.getData().getContentMeta().set(Nutch.SIGNATURE_KEY, StringUtil.toHexString(signature)); // Pass fetch time to content meta parse.getData().getContentMeta().set(Nutch.FETCH_TIME_KEY, Long.toString(datum.getFetchTime())); if (url.equals(key)) datum.setSignature(signature); try { scfilters.passScoreAfterParsing(url, content, parse); } catch (Exception e) { if (LOG.isWarnEnabled()) { LOG.warn("Couldn't pass score, url " + key + " (" + e + ")"); } } output.collect(url, new NutchWritable(new ParseImpl(new ParseText(parse.getText()), parse.getData(), parse.isCanonical()))); } } } catch (IOException e) { if (LOG.isErrorEnabled()) { LOG.error("fetcher caught:" + e.toString()); } } // return parse status if it exits if (parseResult != null && !parseResult.isEmpty()) { Parse p = parseResult.get(content.getUrl()); if (p != null) { return p.getData().getStatus(); } } return null; } } private synchronized void updateStatus(int bytesInPage) throws IOException { pages++; bytes += bytesInPage; } private void reportStatus() throws IOException { String status; synchronized (this) { long elapsed = (System.currentTimeMillis() - start) / 1000; status = pages + " pages, " + errors + " errors, " + Math.round(((float) pages * 10) / elapsed) / 10.0 + " pages/s, " + Math.round(((((float) bytes) * 8) / 1024) / elapsed) + " kb/s, "; } reporter.setStatus(status); } public OldFetcher() { } public OldFetcher(Configuration conf) { setConf(conf); } public void configure(JobConf job) { setConf(job); this.segmentName = job.get(Nutch.SEGMENT_NAME_KEY); this.storingContent = isStoringContent(job); this.parsing = isParsing(job); // if (job.getBoolean("fetcher.verbose", false)) { // LOG.setLevel(Level.FINE); // } } public void close() { } public static boolean isParsing(Configuration conf) { return conf.getBoolean("fetcher.parse", true); } public static boolean isStoringContent(Configuration conf) { return conf.getBoolean("fetcher.store.content", true); } public void run(RecordReader<WritableComparable, Writable> input, OutputCollector<Text, NutchWritable> output, Reporter reporter) throws IOException { this.input = input; this.output = output; this.reporter = reporter; this.maxRedirect = getConf().getInt("http.redirect.max", 3); int threadCount = getConf().getInt("fetcher.threads.fetch", 10); if (LOG.isInfoEnabled()) { LOG.info("OldFetcher: threads: " + threadCount); } for (int i = 0; i < threadCount; i++) { // spawn threads new FetcherThread(getConf()).start(); } // select a timeout that avoids a task timeout long timeout = getConf().getInt("mapred.task.timeout", 10 * 60 * 1000) / 2; do { // wait for threads to exit try { Thread.sleep(1000); } catch (InterruptedException e) { } reportStatus(); // some requests seem to hang, despite all intentions synchronized (this) { if ((System.currentTimeMillis() - lastRequestStart) > timeout) { if (LOG.isWarnEnabled()) { LOG.warn("Aborting with " + activeThreads + " hung threads."); } return; } } } while (activeThreads > 0); } public void fetch(Path segment, int threads) throws IOException { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); long start = System.currentTimeMillis(); if (LOG.isInfoEnabled()) { LOG.info("OldFetcher: starting at " + sdf.format(start)); LOG.info("OldFetcher: segment: " + segment); } JobConf job = new NutchJob(getConf()); job.setJobName("fetch " + segment); job.setInt("fetcher.threads.fetch", threads); job.set(Nutch.SEGMENT_NAME_KEY, segment.getName()); // for politeness, don't permit parallel execution of a single task job.setSpeculativeExecution(false); FileInputFormat.addInputPath(job, new Path(segment, CrawlDatum.GENERATE_DIR_NAME)); job.setInputFormat(InputFormat.class); job.setMapRunnerClass(OldFetcher.class); FileOutputFormat.setOutputPath(job, segment); job.setOutputFormat(FetcherOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(NutchWritable.class); JobClient.runJob(job); long end = System.currentTimeMillis(); LOG.info("OldFetcher: finished at " + sdf.format(end) + ", elapsed: " + TimingUtil.elapsedTime(start, end)); } /** Run the fetcher. */ public static void main(String[] args) throws Exception { int res = ToolRunner.run(NutchConfiguration.create(), new OldFetcher(), args); System.exit(res); } public int run(String[] args) throws Exception { String usage = "Usage: OldFetcher <segment> [-threads n] [-noParsing]"; if (args.length < 1) { System.err.println(usage); return -1; } Path segment = new Path(args[0]); int threads = getConf().getInt("fetcher.threads.fetch", 10); boolean parsing = true; for (int i = 1; i < args.length; i++) { // parse command line if (args[i].equals("-threads")) { // found -threads option threads = Integer.parseInt(args[++i]); } else if (args[i].equals("-noParsing")) parsing = false; } getConf().setInt("fetcher.threads.fetch", threads); if (!parsing) { getConf().setBoolean("fetcher.parse", parsing); } try { fetch(segment, threads); // run the Fetcher return 0; } catch (Exception e) { LOG.error("OldFetcher: " + StringUtils.stringifyException(e)); return -1; } } }