download_data

#!/bin/bash
set -x

cd /usr/local/hadoop/
bin/hdfs namenode -format

sbin/start-dfs.sh

bin/hdfs dfs -mkdir /user
bin/hdfs dfs -mkdir /user/hduser

bin/hdfs dfs -put /home/domi/Desktop/wikiscraps/en/* wc_input
bin/hdfs dfs -cat wc_input/*


hadoop jar share/hadoop/tools/lib/hadoop-streaming-2.7.2.jar -input wc_input -output wc_output -mapper /home/domi/PycharmProjects/wiki_scraper/wc_mapper.py -reducer /home/domi/PycharmProjects/wiki_scraper/wc_reducer.py

bin/hdfs dfs -cat wc_output/* > /home/hduser/wc_output.txt

bin/hdfs dfs -cat wc_output/*

head -200 /home/hduser/wc_output.txt > /home/hduser/en_200_most_frequent.txt

sbin/stop-dfs.sh