我想使用scala示例创建我的第一个scala程序 HBaseTest2.scala ，见sparkd 1.4.1。目标是连接到hbase并执行一些基本操作，例如计算行数或扫描行。然而，当我试图执行程序时，我得到了一个错误。看来斯帕克找不到班级了 HBaseConfiguration . 假设我们位于项目hbasetest2的根路径 /usr/local/Cellar/spark/programs/HBaseTest2 . 以下是例外情况的一些详细信息：
./src/main/scala/com/orange/spark/examples/hbasetest2.scala

package com.orange.spark.examples

import org.apache.hadoop.hbase.client.HBaseAdmin
import org.apache.hadoop.hbase.{HBaseConfiguration, HTableDescriptor, TableName}
import org.apache.hadoop.hbase.mapreduce.TableInputFormat

import org.apache.spark._

object HBaseTest2 {
  def main(args: Array[String]) {
    val sparkConf = new SparkConf().setAppName("HBaseTest2")
    val sc = new SparkContext(sparkConf)
    val tableName = "personal-cloud-test"

    // please ensure HBASE_CONF_DIR is on classpath of spark driver
    // e.g: set it through spark.driver.extraClassPath property
    // in spark-defaults.conf or through --driver-class-path
    // command line option of spark-submit

    val conf = HBaseConfiguration.create()

    // Other options for configuring scan behavior are available. More information available at
    // http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/mapreduce/TableInputFormat.html
    conf.set(TableInputFormat.INPUT_TABLE, tableName)

    // Initialize hBase table if necessary
    val admin = new HBaseAdmin(conf)
    if (!admin.isTableAvailable(tableName)) {
      val tableDesc = new HTableDescriptor(TableName.valueOf(tableName))
      admin.createTable(tableDesc)
    }

    val hBaseRDD = sc.newAPIHadoopRDD(conf, classOf[TableInputFormat],
      classOf[org.apache.hadoop.hbase.io.ImmutableBytesWritable],
      classOf[org.apache.hadoop.hbase.client.Result])

    println("hbaseRDD.count()")
    println(hBaseRDD.count())

    sc.stop()
    admin.close()
  }
}

/构建.sbt
我在这个文件中添加了依赖项，以确保所有调用的类都包含在jar文件中。

name := "HBaseTest2"

version := "1.0"

scalaVersion := "2.11.7"

libraryDependencies += "org.apache.spark" %% "spark-core" % "1.4.1"

libraryDependencies ++= Seq(
    "org.apache.hadoop" % "hadoop-core" % "1.2.1",
    "org.apache.hbase" % "hbase" % "1.0.1.1",
    "org.apache.hbase" % "hbase-client" % "1.0.1.1",
    "org.apache.hbase" % "hbase-common" % "1.0.1.1",
    "org.apache.hbase" % "hbase-server" % "1.0.1.1"
)

运行应用程序

MacBook-Pro-de-Mincong:spark-1.4.1 minconghuang$ bin/spark-submit \
  --class "com.orange.spark.examples.HBaseTest2" \
  --master local[4] \ 
  ../programs/HBaseTest2/target/scala-2.11/hbasetest2_2.11-1.0.jar

例外

15/08/18 12:06:17 INFO storage.BlockManagerMaster: Registered BlockManager
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/hbase/HBaseConfiguration
    at com.orange.spark.examples.HBaseTest2$.main(HBaseTest2.scala:21)
    at com.orange.spark.examples.HBaseTest2.main(HBaseTest2.scala)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:497)
    at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:665)
    at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:170)
    at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:193)
    at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:112)
    at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.hbase.HBaseConfiguration
    at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
    ... 11 more
15/08/18 12:06:17 INFO spark.SparkContext: Invoking stop() from shutdown hook

问题可能来自中提到的hbase配置 HBaseTest2.scala 第16行：
//请确保hbase\u conf\u dir位于spark驱动程序的类路径上
//例如：通过spark.driver.extraclasspath属性进行设置
//在spark-defaults.conf中或通过--driver类路径
//spark submit的命令行选项
但我不知道如何配置它。。。我添加了 HBASE_CONF_DIR 至 CLASSPATH 在我的命令行中。类路径现在是 /usr/local/Cellar/hadoop/hbase-1.0.1.1/conf . 什么都没发生。。。那我该怎么做才能把它修好呢？如果需要，我可以添加/删除详细信息。谢谢！！