Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- # Install java
- sudo apt-get update
- sudo apt-get install software-properties-common -y
- sudo add-apt-repository ppa:webupd8team/java
- sudo apt-get update
- sudo echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections
- sudo apt-get install oracle-java8-installer -y
- sudo apt-get install oracle-java8-set-default -y
- # Set JAVA_HOME and JRE_HOME
- echo "export JAVA_HOME=/usr/lib/jvm/java-8-oracle" >> ~/.bashrc
- echo "export JRE_HOME=/usr/lib/jvm/java-8-oracle/jre" >> ~/.bashrc
- # install anaconda with python 3.6.5
- sudo cd /tmp
- sudo apt-get install curl -y
- sudo curl -O https://repo.anaconda.com/archive/Anaconda3-5.2.0-Linux-x86_64.sh
- sudo bash Anaconda3-5.2.0-Linux-x86_64.sh -b
- # Install pyspark
- echo "export PATH=/root/anaconda3/bin:$PATH" >> ~/.bashrc
- sudo /root/anaconda3/bin/conda install -c conda-forge pyspark
- sudo apt-get install openssh-server openssh-client -y
- sudo /etc/init.d/ssh restart
- cd /root && wget https://archive.apache.org/dist/spark/spark-2.3.1/spark-2.3.1-bin-hadoop2.7.tgz && tar xzf spark-2.3.1-bin-hadoop2.7.tgz
- echo 'export PYSPARK_DRIVER_PYTHON=ipython3' >> ~/.bashrc
- echo 'export PYSPARK_DRIVER_PYTHON_OPTS="notebook"' >> ~/.bashrc
- echo 'export PATH="/home/louis/anaconda3/bin:$PATH"' >> ~/.bashrc
- echo 'export SPARK_HOME="/root/spark-2.3.1-bin-hadoop2.7/"' >> ~/.bashrc
- echo 'export PATH=$PATH:$SPARK_HOME/bin' >> ~/.bashrc
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement