Advertisement
huutho_96

Script

Nov 6th, 2018
302
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Bash 1.41 KB | None | 0 0
  1. # Install java
  2. sudo apt-get update
  3. sudo apt-get install software-properties-common -y
  4. sudo add-apt-repository ppa:webupd8team/java
  5. sudo apt-get update
  6.  
  7. sudo echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections
  8.  
  9. sudo apt-get install oracle-java8-installer -y
  10. sudo apt-get install oracle-java8-set-default -y
  11.  
  12. # Set JAVA_HOME and JRE_HOME
  13. echo "export JAVA_HOME=/usr/lib/jvm/java-8-oracle" >> ~/.bashrc
  14. echo "export JRE_HOME=/usr/lib/jvm/java-8-oracle/jre" >> ~/.bashrc
  15.  
  16. # install anaconda with python 3.6.5
  17. sudo cd /tmp
  18. sudo apt-get install curl -y
  19. sudo curl -O https://repo.anaconda.com/archive/Anaconda3-5.2.0-Linux-x86_64.sh
  20. sudo bash Anaconda3-5.2.0-Linux-x86_64.sh -b
  21.  
  22. # Install pyspark
  23. echo "export PATH=/root/anaconda3/bin:$PATH" >> ~/.bashrc
  24. sudo /root/anaconda3/bin/conda install -c conda-forge pyspark
  25.  
  26.  
  27. sudo apt-get install openssh-server openssh-client -y
  28. sudo /etc/init.d/ssh restart
  29.  
  30.  
  31. cd /root && wget https://archive.apache.org/dist/spark/spark-2.3.1/spark-2.3.1-bin-hadoop2.7.tgz && tar xzf spark-2.3.1-bin-hadoop2.7.tgz
  32.  
  33. echo 'export PYSPARK_DRIVER_PYTHON=ipython3' >> ~/.bashrc
  34. echo 'export PYSPARK_DRIVER_PYTHON_OPTS="notebook"' >> ~/.bashrc
  35. echo 'export PATH="/home/louis/anaconda3/bin:$PATH"' >> ~/.bashrc
  36. echo 'export SPARK_HOME="/root/spark-2.3.1-bin-hadoop2.7/"' >> ~/.bashrc
  37. echo 'export PATH=$PATH:$SPARK_HOME/bin' >> ~/.bashrc
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement