Spark集群安装和使用_export spark_library_path

$ yum list |grep spark
spark-core.noarch                 1.2.0+cdh5.3.0+364-1.cdh5.3.0.p0.36.el6 @cdh
spark-history-server.noarch       1.2.0+cdh5.3.0+364-1.cdh5.3.0.p0.36.el6 @cdh
spark-master.noarch               1.2.0+cdh5.3.0+364-1.cdh5.3.0.p0.36.el6 @cdh
spark-python.noarch               1.2.0+cdh5.3.0+364-1.cdh5.3.0.p0.36.el6 @cdh
spark-worker.noarch               1.2.0+cdh5.3.0+364-1.cdh5.3.0.p0.36.el6 @cdh
hue-spark.x86_64                  3.7.0+cdh5.3.0+134-1.cdh5.3.0.p0.24.el6 cdh

# 在 cdh1 节点上运行
$ sudo yum install spark-core spark-master spark-worker spark-python spark-history-server -y
 
# 在 cdh2、cdh3 上运行
$ sudo yum install spark-core spark-worker spark-python -y

cdh1节点:  spark-master spark-history-server
cdh2节点:  spark-worker 
cdh3节点:  spark-worker 

export SPARK_HOME=/usr/lib/spark

# 设置 master 主机名称
export STANDALONE_SPARK_MASTER_HOST=cdh1
 
export SPARK_MASTER_IP=$STANDALONE_SPARK_MASTER_HOST
 
### Let's run everything with JVM runtime, instead of Scala
export SPARK_LAUNCH_WITH_SCALA=0
export SPARK_LIBRARY_PATH=${SPARK_HOME}/lib
export SCALA_LIBRARY_PATH=${SPARK_HOME}/lib
export SPARK_MASTER_WEBUI_PORT=18080
export SPARK_MASTER_PORT=7077
export SPARK_WORKER_PORT=7078
export SPARK_WORKER_WEBUI_PORT=18081
export SPARK_WORKER_DIR=/var/run/spark/work
export SPARK_LOG_DIR=/var/log/spark
 
if [ -n "$HADOOP_HOME" ]; then
  export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${HADOOP_HOME}/lib/native
fi
 
export HIVE_CONF_DIR=${HIVE_CONF_DIR:-/etc/hive/conf}
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop/conf}
 
### Comment above 2 lines and uncomment the following if
### you want to run with scala version, that is included with the package
#export SCALA_HOME=${SCALA_HOME:-/usr/lib/spark/scala}
#export PATH=$PATH:$SCALA_HOME/bin

export SPARK_DAEMON_MEMORY=256m

scp -r /etc/spark/conf  cdh2:/etc/spark
scp -r /etc/spark/conf  cdh3:/etc/spark

$ sudo -u hdfs hadoop fs -mkdir /user/spark
$ sudo -u hdfs hadoop fs -mkdir /user/spark/applicationHistory
$ sudo -u hdfs hadoop fs -chown -R spark:spark /user/spark
$ sudo -u hdfs hadoop fs -chmod 1777 /user/spark/applicationHistory

cp /etc/spark/conf/spark-defaults.conf.template /etc/spark/conf/spark-defaults.conf

spark.eventLog.dir=/user/spark/applicationHistory
spark.eventLog.enabled=true

spark.yarn.historyServer.address=http://HISTORY_HOST:HISTORY_PORT

spark.master=spark://cdh1:7077
spark.eventLog.dir=/user/spark/applicationHistory
spark.eventLog.enabled=true
 
spark.yarn.historyServer.address=http://cdh1:19888

#这里配置的是本地目录，也可以改为 hdfs 上的目录
export SPARK_HISTORY_OPTS="$SPARK_HISTORY_OPTS -Dspark.history.fs.logDirectory=/var/log/spark"

HOSTNAME=`hostname -f`
export SPARK_HISTORY_OPTS="$SPARK_HISTORY_OPTS -Dspark.history.kerberos.enabled=true -Dspark.history.kerberos.principal=spark/${HOSTNAME}@LASHOU.COM -Dspark.history.kerberos.keytab=/etc/spark/conf/spark.keytab -Dspark.history.ui.acls.enable=true"

# 在 cdh1 节点上运行
$ sudo service spark-master start
 
# 在 cdh1 节点上运行，如果 hadoop 集群配置了 kerberos，则运行之前需要先获取 spark 用户的凭证
# kinit -k -t /etc/spark/conf/spark.keytab spark/cdh1@JAVACHEN.COM
$ sudo service spark-history-server start
 
# 在 cdh1、cdh2、cdh3 节点上运行
$ sudo service spark-worker start

$ sudo service spark-master stop
$ sudo service spark-worker stop
$ sudo service spark-history-server stop

$ sudo chkconfig spark-master on
$ sudo chkconfig spark-worker on
$ sudo chkconfig spark-history-server on

15/04/02 11:31:56 INFO HistoryServer: Registered signal handlers for [TERM, HUP, INT]
15/04/02 11:31:57 INFO UserGroupInformation: Login successful for user spark/cdh1@JAVACHEN.COM using keytab file /etc/spark/conf/spark.keytab
15/04/02 11:31:57 INFO SecurityManager: Changing view acls to: spark
15/04/02 11:31:57 INFO SecurityManager: Changing modify acls to: spark
15/04/02 11:31:57 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(spark); users with modify permissions: Set(spark)
15/04/02 11:31:58 WARN Utils: Service could not bind on port 18080. Attempting port 18081.
15/04/02 11:31:58 INFO Utils: Successfully started service on port 18081.
15/04/02 11:31:58 INFO HistoryServer: Started HistoryServer at http://cdh1:18081

$ ls /usr/lib/spark/sbin
slaves.sh        spark-daemons.sh  start-master.sh  stop-all.sh
spark-config.sh  spark-executor    start-slave.sh   stop-master.sh
spark-daemon.sh  start-all.sh      start-slaves.sh  stop-slaves.sh

# A Spark Worker will be started on each of the machines listed below.
cdh1
cdh2
cdh3

$ cd /usr/lib/spark/sbin
$ ./start-master.sh

$ ./start-slaves.sh

$ ./bin/spark-class org.apache.spark.deploy.worker.Worker spark://cdh1:18080

$ ./bin/run-example SparkPi 10
 
15/04/02 11:45:50 INFO SecurityManager: Changing view acls to: root
15/04/02 11:45:50 INFO SecurityManager: Changing modify acls to: root
15/04/02 11:45:50 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(root); users with modify permissions: Set(root)
Pi is roughly 3.141808

# 使用2个 Worker 线程本地化运行 Spark（理想情况下，该值应该根据运行机器的 CPU 核数设定）
$ ./bin/pyspark --master local[2]
 
Python 2.6.6 (r266:84292, Nov 22 2013, 12:16:22)
[GCC 4.4.7 20120313 (Red Hat 4.4.7-4)] on linux2
Type "help", "copyright", "credits" or "license" for more information.
 
Welcome to
      ____              __
     / __/__  ___ _____/ /__
    _\ \/ _ \/ _ / __/  _/
   /__ / .__/\_,_/_/ /_/\_\   version 1.2.0
      /_/
 
Using Python version 2.6.6 (r266:84292, Nov 22 2013 12:16:22)
SparkContext available as sc.
>>>

$ ./bin/spark-shell --master local[2]
15/04/02 11:49:26 INFO SecurityManager: Changing view acls to: root
15/04/02 11:49:26 INFO SecurityManager: Changing modify acls to: root
15/04/02 11:49:26 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(root); users with modify permissions: Set(root)
15/04/02 11:49:26 INFO HttpServer: Starting HTTP Server
15/04/02 11:49:26 INFO Utils: Successfully started service 'HTTP class server' on port 47031.
Welcome to
      ____              __
     / __/__  ___ _____/ /__
    _\ \/ _ \/ _ / __/  _/
   /___/ .__/\_,_/_/ /_/\_\   version 1.2.0
      /_/
 
Using Scala version 2.10.4 (Java HotSpot(TM) 64-Bit Server VM, Java 1.6.0_45)
scala> val textFile = sc.textFile("README.md")
textFile: spark.RDD[String] = spark.MappedRDD@2ee9b6e3

scala> textFile.count() // Number of items in this RDD
res0: Long = 126
 
scala> textFile.first() // First item in this RDD
res1: String = # Apache Spark

scala> val linesWithSpark = textFile.filter(line => line.contains("Spark"))
linesWithSpark: spark.RDD[String] = spark.FilteredRDD@7dd4af09

scala> textFile.filter(line => line.contains("Spark")).count() // How many lines contain "Spark"?
res3: Long = 15

$ spark-shell --master spark://cdh1:7077 

$ spark-shell --master spark://cdh1:7077 --cores 2 --jars code.jar

$ spark-shell --deploy-mode client --master yarn

15/04/02 11:58:32 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool
org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 0.0 failed 4 times, most recent failure: Lost task 0.3 in stage 0.0 (TID 6, bj03-bi-pro-hdpnamenn): java.io.IOException: Failed on local exception: java.io.IOException: org.apache.hadoop.security.AccessControlException: Client cannot authenticate via:[TOKEN, KERBEROS]; Host Details : local host is: "cdh1/192.168.56.121"; destination host is: "192.168.56.121":8020;
        org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:764)
        org.apache.hadoop.ipc.Client.call(Client.java:1415)
        org.apache.hadoop.ipc.Client.call(Client.java:1364)
        org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:206)
        com.sun.proxy.$Proxy17.getBlockLocations(Unknown Source)

$ spark-shell --deploy-mode client --master yarn

$ mkdir -p /usr/lib/spark/examples/python
$ tar zxvf /usr/lib/spark/lib/python.tar.gz -C /usr/lib/spark/examples/python
 
$ ./bin/spark-submit examples/python/pi.py 10

$ spark-submit --class "SimpleApp" --master local[4] simple-project-1.0.jar

$ spark-shell --master spark://cdh1:7077

$ ./bin/spark-class org.apache.spark.deploy.Client kill <master url> <driver ID>

val conf = new SparkConf()
             .setMaster(...)
             .setAppName(...)
             .set("spark.cores.max", "10")
val sc = new SparkContext(conf)

export SPARK_MASTER_OPTS="-Dspark.deploy.defaultCores=<value>"

$ echo "hello world" >test.txt
$ hadoop fs -put test.txt /tmp
 
$ spark-shell --master spark://cdh1:7077
scala> val file = sc.textFile("hdfs://cdh1:8020/tmp/test.txt")
scala> file.count()

/usr/lib/spark/bin/spark-shell: line 48:  5385 Killed                  "$FWDIR"/bin/spark-submit --class org.apache.spark.repl.Main "${SUBMISSION_OPTS[@]}" spark-shell "${APPLICATION_OPTS[@]}"

14/10/24 14:51:40 WARN hdfs.BlockReaderLocal: The short-circuit local reads feature cannot be used because libhadoop cannot be loaded.
14/10/24 14:51:40 ERROR lzo.GPLNativeCodeLoader: Could not load native gpl library
java.lang.UnsatisfiedLinkError: no gplcompression in java.library.path
    at java.lang.ClassLoader.loadLibrary(ClassLoader.java:1738)
    at java.lang.Runtime.loadLibrary0(Runtime.java:823)
    at java.lang.System.loadLibrary(System.java:1028)
    at com.hadoop.compression.lzo.GPLNativeCodeLoader.<clinit>(GPLNativeCodeLoader.java:32)
    at com.hadoop.compression.lzo.LzoCodec.<clinit>(LzoCodec.java:71)
    at java.lang.Class.forName0(Native Method)
    at java.lang.Class.forName(Class.java:249)
    at org.apache.hadoop.conf.Configuration.getClassByNameOrNull(Configuration.java:1836)
    at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:1801)
    at org.apache.hadoop.io.compress.CompressionCodecFactory.getCodecClasses(CompressionCodecFactory.java:128)

cp /usr/lib/hadoop/lib/native/libgplcompression.so $JAVA_HOME/jre/lib/amd64/
cp /usr/lib/hadoop/lib/native/libhadoop.so $JAVA_HOME/jre/lib/amd64/
cp /usr/lib/hadoop/lib/native/libsnappy.so $JAVA_HOME/jre/lib/amd64/

$ spark-shell --master spark://cdh1:7077
scala> val file = sc.textFile("hdfs://cdh1:8020/tmp/data.txt")
scala> val counts = file.flatMap(line => line.split(" ")).map(word => (word, 1)).reduceByKey(_ + _)
scala> counts.saveAsTextFile("hdfs://cdh1:8020/tmp/output")

$ hadoop fs -cat /tmp/output/part-00000
(hello,1)
(world,1)

$ spark-submit --class org.apache.spark.examples.SparkPi  --master spark://cdh1:7077 /usr/lib/spark/lib/spark-examples-1.2.0-cdh5.3.0-hadoop2.5.0-cdh5.3.0.jar 10

$ spark-submit --class path.to.your.Class --deploy-mode cluster --master yarn [options] <app jar> [app options]

$ spark-submit --class path.to.your.Class --deploy-mode client --master yarn [options] <app jar> [app options]

$ spark-submit --class org.apache.spark.examples.SparkPi \
    --deploy-mode cluster  \
    --master yarn  \
    --num-executors 3 \
    --driver-memory 4g \
    --executor-memory 2g \
    --executor-cores 1 \
    --queue thequeue \
    /usr/lib/spark/lib/spark-examples-1.2.0-cdh5.3.0-hadoop2.5.0-cdh5.3.0.jar \
    10

$ hdfs dfs -mkdir -p /user/spark/share/lib
$ hdfs dfs -put $SPARK_HOME/lib/spark-assembly.jar  /user/spark/share/lib/spark-assembly.jar
 
$ SPARK_JAR=hdfs://<nn>:<port>/user/spark/share/lib/spark-assembly.jar

$ cd /usr/lib/spark/bin
$ ./spark-sql
java.lang.ClassNotFoundException: org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver
    at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
    at java.security.AccessController.doPrivileged(Native Method)
    at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:247)
    at java.lang.Class.forName0(Native Method)
    at java.lang.Class.forName(Class.java:247)
    at org.apache.spark.deploy.SparkSubmit$.launch(SparkSubmit.scala:319)
    at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:75)
    at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
 
Failed to load Spark SQL CLI main class org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.
You need to build Spark with -Phive.

Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.hive.cli.CliDriver
  at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
  at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
  at java.security.AccessController.doPrivileged(Native Method)
  at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
  at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
  at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
  at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
  ... 18 more

$ git clone git@github.com:cloudera/spark.git
$ cd spark
$ git checkout -b origin/cdh5-1.2.0_5.3.0

$ sbt/sbt -Dhadoop.version=2.5.0-cdh5.3.0 -Pyarn -Phive assembly

<modules>
    <module>core</module>
    <module>bagel</module>
    <module>graphx</module>
    <module>mllib</module>
    <module>tools</module>
    <module>streaming</module>
    <module>sql/catalyst</module>
    <module>sql/core</module>
    <module>sql/hive</module>
    <module>sql/hive-thriftserver</module> <!--添加的一行-->
    <module>repl</module>
    <module>assembly</module>
    <module>external/twitter</module>
    <module>external/kafka</module>
    <module>external/flume</module>
    <module>external/flume-sink</module>
    <module>external/zeromq</module>
    <module>external/mqtt</module>
    <module>examples</module>
  </modules>

$ export MAVEN_OPTS="-Xmx2g -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=512m"
$ mvn -Pyarn -Dhadoop.version=2.5.0-cdh5.3.0 -Phive -Phive-thriftserver -DskipTests clean package