我正在用scala脚本编写简单的etl进程,并以'sparkshell-i rawetl.scala'的形式运行,但是出现了异常“table not found basetable”。我也检查了文件,它是正确的选择。
下面是示例代码
import java.io.File
import sqlContext.implicits._
import scala.io.Source
import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
import org.apache.spark.SparkConf
import org.apache.spark.sql.SQLContext
import sys.process._
case class pageRow(affiliateid : String , pageurl : String, alertedbrandsafety : String, blockedbrandsafety : String, grossimpressions : String, spider_bot : String, invalidbrowser : String ,outlieractivity : String , day : String)
object batch_nht {
def main() {
processRawNHT()
}
def processRawNHT() {
val rawFile = "hadoop fs -ls /tmp/XXX/rawDB/" #| "tail -1" !!
val fileName = rawFile.substring(rawFile.indexOf("/"))
val filePathName = "hdfs://AAAAA:8020" + fileName.trim()
println(filePathName)
val sc = new SparkContext(new SparkConf().setAppName("analyzeBlog"))
val sqlContext = new org.apache.spark.sql.SQLContext(sc)
val inviewraw = sc.textFile(filePathName).map(_.split(",")).map(x=>x.map(_.replace("\"","")))
val base_people = inviewraw.map{r => if(r(13) == null || r(13).trim.isEmpty) (r(5) ,r(32), r(48), r(49),r(14), r(71), r(72), r(73),r(0)) else (r(5) ,r(32), r(48), r(49),r(14), r(71), r(72), r(73),r(0))}
val logs_base_page_schemaRDD = base_people.map(p => pageRow(p._1, p._2, p._3, p._4,p._5, p._6, p._7, p._8,p._9)).toDF()
logs_base_page_schemaRDD.registerTempTable("baseTable")
sqlContext.sql("select * from baseTable").collect().foreach(println)
}
}
batch_nht.main()
注意:如果我在sparkshell中一个接一个地运行下面的命令(没有脚本),我会毫无例外地得到正确的输出。
val sqlContext = new org.apache.spark.sql.SQLContext(sc)
import sqlContext.implicits._
val inviewraw = sc.textFile("hdfs://AAAAA:8020/tmp/XXX/rawDB/rawFile.csv").map(_.split(",")).map(x=>x.map(_.replace("\"","")))
val base_people = inviewraw.map{r => if(r(13) == null || r(13).trim.isEmpty) (r(5) ,r(32), r(48), r(49),r(14), r(71), r(72), r(73),r(0)) else (r(5) ,r(32), r(48), r(49),r(14), r(71), r(72), r(73),r(0))}
case class pageRow(affiliateid : String , pageurl : String, alertedbrandsafety : String, blockedbrandsafety : String, grossimpressions : String, spider_bot : String, invalidbrowser : String ,outlieractivity : String , day : String)
val logs_base_page_schemaRDD = base_people.map(p => pageRow(p._1, p._2, p._3, p._4,p._5, p._6, p._7, p._8,p._9)).toDF()
--create table
logs_base_page_schemaRDD.registerTempTable("baseTable")
sqlContext.sql("select * from baseTable").collect().foreach(println)
请提出问题所在?在脚本中
1条答案
按热度按时间tnkciper1#
下面是测试的代码片段。您的方法不适用于使用scala的编程方法。