remove infrequent words

Author: lukaselmer

remove_infrequent_words10/log.log

@@ -3,4 +3,4 @@
 MIN_WORD_COUNT=500
 OUT_DIR=`pwd`
 
-MIN_WORD_COUNT="${MIN_WORD_COUNT}" OUTPUT="hdfs://dco-node121.dco.ethz.ch:54310/cw-combined-pruned-${MIN_WORD_COUNT}" /root/spark/bin/spark-submit --num-executors 20 --class RemoveInfrequentWordsApp ${OUT_DIR}/run.jar > ${OUT_DIR}/log.log 2>&1
+MIN_WORD_COUNT="${MIN_WORD_COUNT}" OUTPUT="hdfs://dco-node121.dco.ethz.ch:54310/cw-combined-pruned-${MIN_WORD_COUNT}" /root/spark/bin/spark-submit --total-executor-cores 20 --class RemoveInfrequentWordsApp ${OUT_DIR}/run.jar > ${OUT_DIR}/log.log 2>&1

remove_infrequent_words10/run.sh

@@ -0,0 +1,3 @@
+repo
+run.jar
+

remove_infrequent_words11/.gitignore

@@ -0,0 +1,13 @@
+#! /bin/bash
+
+rm -f run.jar
+
+cd repo
+git pull
+
+cd src/remove_infrequent_words
+sbt assembly
+
+cd ../../..
+cp repo/src/remove_infrequent_words/target/scala-2.10/RemoveInfrequentWordsApp-assembly-1.0.jar run.jar
+

remove_infrequent_words11/build.sh

@@ -0,0 +1,17 @@
+# Simple word count on the cleaned dataset
+
+## Version
+
+https://github.com/lukaselmer/ethz-web-scale-data-mining-project/tree/8dbe8eb1f4e885cc212e0161492a6a5a7a696d05/src/remove_infrequent_words
+
+## Usage
+
+<param1>=<value1> <param2>=<value2> ... mahout-submit ...
+Options
+MIN_WORD_COUNT: number, how many times a word has to occur at least to be kept
+[optional] MAX_WORD_COUNT: default: Int.MaxInt
+[optional] OUTPUT: default: hdfs://dco-node121.dco.ethz.ch:54310/cw-combined-pruned
+[optional] INPUT_COMBINED: default: hdfs://dco-node121.dco.ethz.ch:54310/cw-combined
+[optional] INPUT_WORDCOUNT: default: hdfs://dco-node121.dco.ethz.ch:54310/cw-wordcount/wordcounts.txt
+
+

remove_infrequent_words11/log.log

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+MIN_WORD_COUNT=1000
+OUT_DIR=`pwd`
+
+MIN_WORD_COUNT="${MIN_WORD_COUNT}" OUTPUT="hdfs://dco-node121.dco.ethz.ch:54310/cw-combined-pruned-${MIN_WORD_COUNT}" /root/spark/bin/spark-submit --total-executor-cores 20 --class RemoveInfrequentWordsApp ${OUT_DIR}/run.jar > ${OUT_DIR}/log.log 2>&1

remove_infrequent_words11/readme.md

@@ -0,0 +1,3 @@
+repo
+run.jar
+

remove_infrequent_words11/run.sh

@@ -0,0 +1,13 @@
+#! /bin/bash
+
+rm -f run.jar
+
+cd repo
+git pull
+
+cd src/remove_infrequent_words
+sbt assembly
+
+cd ../../..
+cp repo/src/remove_infrequent_words/target/scala-2.10/RemoveInfrequentWordsApp-assembly-1.0.jar run.jar
+