Improved Documentation of Hadoop/MapReduce WebFinder Example

Author: Thomas Weise

hadoop/README.md

@@ -235,8 +235,8 @@ In order to run Hadoop in a pseudo-distributed fashion, we need to enable passwo
 <li>In the terminal, execute <code>ssh localhost</code> to test if you can open a <a href="https://en.wikipedia.org/wiki/Secure&#95;Shell">secure shell</a> connection to your current, local computer <a href="http://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/SingleCluster.html#Setup&#95;passphraseless&#95;ssh">without needing a password</a>.
 </li>
 <li>It may say something like:
-<pre>ssh: connect to host localhost port 22: Connection refused</pre>.
-If it does say this, then do
+<pre>ssh: connect to host localhost port 22: Connection refused</pre>
+If it does say this (i.e., you did not install the pre-requisites&hellip;), then do
 <pre>sudo apt-get install ssh</pre>
 and it may say something like
 <pre>

hadoop/webFinder/src/main/java/webFinder/WebFinderDriver.java

@@ -14,6 +14,10 @@
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 
+/**
+ * The driver of the web finder sets up the distributed computation by
+ * defining what the mapper and reducer classes, amongst other things.
+ */
 public class WebFinderDriver extends Configured implements Tool {
 
  public static void main(final String[] args) throws Exception {
@@ -27,6 +31,7 @@ public static void main(final String[] args) throws Exception {
  }
  }
 
+ /** Setting up the computation. */
  @Override
  public int run(final String[] args) throws Exception {
  final Configuration conf;

hadoop/webFinder/src/main/java/webFinder/WebFinderMapper.java

@@ -13,12 +13,22 @@
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.log4j.Logger;
 
-public class WebFinderMapper
- extends Mapper<LongWritable, Text, Text, Text> {
+/**
+ * This is the Mapper component of the Web Finder example. Its input are
+ * text lines, where each line stands for a website URL. It finds all
+ * resources that are loaded by a given website URL and emits tuples of
+ * kind {@code <resource URL, website URL>}.
+ */
+public class WebFinderMapper extends
+ Mapper<LongWritable, Text, Text, Text> {
 
  /** the logger we use */
  private static Logger LOGGER = Logger.getLogger(WebFinderMapper.class);
 
+ /**
+ * Map tuples of type {@code <line number, website url text>} to tuples
+ * of kind {@code <resource url text, website url text>}.
+ */
  @Override
  protected void map(final LongWritable offset, final Text line,
  final Context context) throws IOException, InterruptedException {
@@ -171,8 +181,8 @@ private static final void __load(final int remainingDepth,
  error.addSuppressed(error2);
  error.addSuppressed(error3);
  if (WebFinderMapper.LOGGER != null) {
- WebFinderMapper.LOGGER
- .warn("Error while trying to build URL with string '"
+ WebFinderMapper.LOGGER.warn(
+ "Error while trying to build URL with string '"
  + test + "' under load URL '"
  + loadUrl.toString() + "' for base URL '"
  + baseUrl.toString() + "'.", error2);
@@ -215,8 +225,8 @@ private static final void __load(final int remainingDepth,
  }
  } catch (final Throwable error) {
  if (WebFinderMapper.LOGGER != null) {
- WebFinderMapper.LOGGER.warn(
- "Error while trying to load URL '" + loadUrl + "'.", error);
+ WebFinderMapper.LOGGER.warn("Error while trying to load URL '"
+ + loadUrl + "'.", error);
  }
  }
 
@@ -314,7 +324,7 @@ public static final void main(final String[] args) throws Throwable {
 
  /** the link descriptions */
  static final __LinkDesc[] DESCS = { //
-  new __LinkDesc(false, "<link rel=\"stylesheet\"", "href="), //
+ new __LinkDesc(false, "<link rel=\"stylesheet\"", "href="), //
  new __LinkDesc(false, "<link rel='stylesheet'", "href="), //
  new __LinkDesc(false, "<img", "src="), //
  new __LinkDesc(false, "<script", "src="), //

hadoop/webFinder/src/main/java/webFinder/WebFinderReducer.java

@@ -11,9 +11,23 @@
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Reducer;
 
-public class WebFinderReducer
- extends Reducer<Text, Text, Text, List<Text>> {
+/**
+ * This is the reducer component of the web finder example. For each key (
+ * {@code resource URL}) of the tuples produced by the mapper, it receives
+ * the list of all values ({@code website URLs}). If such a list contains
+ * more than one unique element, this means that the resource is shared by
+ * multiple websites. This reducer emits tuples of the form
+ * {@code <resource URL, list of website urls>}.
+ */
+public class WebFinderReducer extends
+ Reducer<Text, Text, Text, List<Text>> {
 
+ /**
+ * The actual reduction step: From the tuples of form
+ * {@code <resource URL, iterable of referencing website URLs>}, select
+ * all resources referenced by more than one unique website. For these,
+ * output tuples of the form {@code <resource URL, list of website URLs>}.
+ */
  @Override
  protected void reduce(final Text key, final Iterable<Text> values,
  final Context context) throws IOException, InterruptedException {