API tools faq
paste
desdemona

run_map_reduce_all_languages.sh

desdemona
May 30th, 2016
503
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Bash 4.50 KB | None | 0 0
raw download clone embed print report
  1. #!/bin/bash
  2.  
  3. set -x
  4.  
  5. #en
  6. cd /usr/local/hadoop/
  7. bin/hdfs namenode -format
  8. sbin/start-dfs.sh
  9. bin/hdfs dfs -mkdir /user
  10. bin/hdfs dfs -mkdir /user/domi
  11. bin/hdfs dfs -put /home/domi/Desktop/wikiscraps/en wc_input
  12. bin/hadoop jar share/hadoop/tools/lib/hadoop-streaming-2.7.2.jar -input wc_input -output wc_output -mapper /home/domi/PycharmProjects/HadoopWordCount/wc/wc_mapper.py -reducer /home/domi/PycharmProjects/HadoopWordCount/wc/wc_reducer.py
  13.  
  14. bin/hdfs dfs -cat wc_output/* > /home/domi/wc_output.txt
  15. bin/hdfs dfs -cat wc_output/*
  16. head -200 /home/domi/wc_output.txt > /home/domi/PycharmProjects/HadoopWordCount/most_frequent/en_200_most_frequent.txt
  17. sbin/stop-dfs.sh
  18. for x in `cd /etc/init.d ; ls hadoop*` ; do sudo service $x stop ; done
  19. rm -rf /tmp/hadoop-domi/dfs/*
  20.  
  21. #pl
  22. cd /usr/local/hadoop/
  23. bin/hdfs namenode -format
  24. sbin/start-dfs.sh
  25. bin/hdfs dfs -mkdir /user
  26. bin/hdfs dfs -mkdir /user/domi
  27. bin/hdfs dfs -put /home/domi/Desktop/wikiscraps/pl wc_input
  28. bin/hadoop jar share/hadoop/tools/lib/hadoop-streaming-2.7.2.jar -input wc_input -output wc_output -mapper /home/domi/PycharmProjects/HadoopWordCount/wc/wc_mapper.py -reducer /home/domi/PycharmProjects/HadoopWordCount/wc/wc_reducer.py
  29.  
  30. bin/hdfs dfs -cat wc_output/* > /home/domi/wc_output.txt
  31. bin/hdfs dfs -cat wc_output/*
  32. head -200 /home/domi/wc_output.txt > /home/domi/PycharmProjects/HadoopWordCount/most_frequent/pl_200_most_frequent.txt
  33. sbin/stop-dfs.sh
  34. for x in `cd /etc/init.d ; ls hadoop*` ; do sudo service $x stop ; done
  35. rm -rf /tmp/hadoop-domi/dfs/*
  36.  
  37. #hu
  38. cd /usr/local/hadoop/
  39. bin/hdfs namenode -format
  40. sbin/start-dfs.sh
  41. bin/hdfs dfs -mkdir /user
  42. bin/hdfs dfs -mkdir /user/domi
  43. bin/hdfs dfs -put /home/domi/Desktop/wikiscraps/hu wc_input
  44. bin/hadoop jar share/hadoop/tools/lib/hadoop-streaming-2.7.2.jar -input wc_input -output wc_output -mapper /home/domi/PycharmProjects/HadoopWordCount/wc/wc_mapper.py -reducer /home/domi/PycharmProjects/HadoopWordCount/wc/wc_reducer.py
  45.  
  46. bin/hdfs dfs -cat wc_output/* > /home/domi/wc_output.txt
  47. bin/hdfs dfs -cat wc_output/*
  48. head -200 /home/domi/wc_output.txt > /home/domi/PycharmProjects/HadoopWordCount/most_frequent/hu_200_most_frequent.txt
  49. sbin/stop-dfs.sh
  50. for x in `cd /etc/init.d ; ls hadoop*` ; do sudo service $x stop ; done
  51. rm -rf /tmp/hadoop-domi/dfs/*
  52.  
  53. #de
  54. cd /usr/local/hadoop/
  55. bin/hdfs namenode -format
  56. sbin/start-dfs.sh
  57. bin/hdfs dfs -mkdir /user
  58. bin/hdfs dfs -mkdir /user/domi
  59. bin/hdfs dfs -put /home/domi/Desktop/wikiscraps/de wc_input
  60. bin/hadoop jar share/hadoop/tools/lib/hadoop-streaming-2.7.2.jar -input wc_input -output wc_output -mapper /home/domi/PycharmProjects/HadoopWordCount/wc/wc_mapper.py -reducer /home/domi/PycharmProjects/HadoopWordCount/wc/wc_reducer.py
  61.  
  62. bin/hdfs dfs -cat wc_output/* > /home/domi/wc_output.txt
  63. bin/hdfs dfs -cat wc_output/*
  64. head -200 /home/domi/wc_output.txt > /home/domi/PycharmProjects/HadoopWordCount/most_frequent/de_200_most_frequent.txt
  65. sbin/stop-dfs.sh
  66. for x in `cd /etc/init.d ; ls hadoop*` ; do sudo service $x stop ; done
  67. rm -rf /tmp/hadoop-domi/dfs/*
  68.  
  69. #sv
  70. cd /usr/local/hadoop/
  71. bin/hdfs namenode -format
  72. sbin/start-dfs.sh
  73. bin/hdfs dfs -mkdir /user
  74. bin/hdfs dfs -mkdir /user/domi
  75. bin/hdfs dfs -put /home/domi/Desktop/wikiscraps/sv wc_input
  76. bin/hadoop jar share/hadoop/tools/lib/hadoop-streaming-2.7.2.jar -input wc_input -output wc_output -mapper /home/domi/PycharmProjects/HadoopWordCount/wc/wc_mapper.py -reducer /home/domi/PycharmProjects/HadoopWordCount/wc/wc_reducer.py
  77.  
  78. bin/hdfs dfs -cat wc_output/* > /home/domi/wc_output.txt
  79. bin/hdfs dfs -cat wc_output/*
  80. head -200 /home/domi/wc_output.txt > /home/domi/PycharmProjects/HadoopWordCount/most_frequent/sv_200_most_frequent.txt
  81. sbin/stop-dfs.sh
  82. for x in `cd /etc/init.d ; ls hadoop*` ; do sudo service $x stop ; done
  83. rm -rf /tmp/hadoop-domi/dfs/*
  84.  
  85. #nl
  86. cd /usr/local/hadoop/
  87. bin/hdfs namenode -format
  88. sbin/start-dfs.sh
  89. bin/hdfs dfs -mkdir /user
  90. bin/hdfs dfs -mkdir /user/domi
  91. bin/hdfs dfs -put /home/domi/Desktop/wikiscraps/nl wc_input
  92. bin/hadoop jar share/hadoop/tools/lib/hadoop-streaming-2.7.2.jar -input wc_input -output wc_output -mapper /home/domi/PycharmProjects/HadoopWordCount/wc/wc_mapper.py -reducer /home/domi/PycharmProjects/HadoopWordCount/wc/wc_reducer.py
  93.  
  94. bin/hdfs dfs -cat wc_output/* > /home/domi/wc_output.txt
  95. bin/hdfs dfs -cat wc_output/*
  96. head -200 /home/domi/wc_output.txt > /home/domi/PycharmProjects/HadoopWordCount/most_frequent/nl_200_most_frequent.txt
  97. sbin/stop-dfs.sh
  98. for x in `cd /etc/init.d ; ls hadoop*` ; do sudo service $x stop ; done
  99. rm -rf /tmp/hadoop-domi/dfs/*
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!