我已经在自己的ubuntulinux18.04机器上安装并配置了jdk1.8/hadoop2.8.4/scala2.10.6,wordcountjava应用程序可以用“hadoopjar”命令运行。
然后我在同一个intellij项目中使用java wordcount尝试了scala代码,代码如下:
import java.io.IOException
import java.util._
import org.apache.hadoop.fs.Path
import org.apache.hadoop.io._
import org.apache.hadoop.mapred._
object wc01 {
@throws[Exception]
def main(args: Array[String]) {
val conf: JobConf = new JobConf(this.getClass)
conf.setJobName("WordCountScala")
conf.setOutputKeyClass(classOf[Text])
conf.setOutputValueClass(classOf[IntWritable])
conf.setMapperClass(classOf[Map])
conf.setCombinerClass(classOf[Reduce])
conf.setReducerClass(classOf[Reduce])
conf.setInputFormat(classOf[TextInputFormat])
conf.setOutputFormat(classOf[TextOutputFormat[Text, IntWritable]])
FileInputFormat.setInputPaths(conf, new Path(args(0)))
FileOutputFormat.setOutputPath(conf, new Path(args(1)))
JobClient.runJob(conf)
}
class Map extends MapReduceBase with Mapper[LongWritable, Text, Text, IntWritable] {
private final val one = new IntWritable(1)
private val word = new Text()
@throws[IOException]
def map(key: LongWritable, value: Text, output: OutputCollector[Text, IntWritable], reporter: Reporter) {
val line: String = value.toString
line.split(" ").foreach { token =>
word.set(token)
output.collect(word, one)
}
}
}
class Reduce extends MapReduceBase with Reducer[Text, IntWritable, Text, IntWritable] {
@throws[IOException]
def reduce(key: Text, values: Iterator[IntWritable], output: OutputCollector[Text, IntWritable], reporter: Reporter) {
import scala.collection.JavaConversions._
val sum = values.toList.reduce((valueOne, valueTwo) => new IntWritable(valueOne.get() + valueTwo.get()))
output.collect(key, new IntWritable(sum.get()))
}
}
}
我编译并打包了它,hadoop jar来运行它,它给出了错误:
hdfs@ubuntu:$ hadoop jar my_java_scala_mr-1.0-SNAPSHOT.jar wc01 my-input my-output
18/08/26 01:30:58 INFO client.RMProxy: Connecting to ResourceManager at /0.0.0.0:8032
18/08/26 01:30:58 INFO client.RMProxy: Connecting to ResourceManager at /0.0.0.0:8032
18/08/26 01:30:58 WARN mapreduce.JobResourceUploader: Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this.
18/08/26 01:30:58 INFO mapred.FileInputFormat: Total input files to process : 1
18/08/26 01:30:58 INFO mapreduce.JobSubmitter: number of splits:2
18/08/26 01:30:58 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1535165327468_0012
18/08/26 01:30:59 INFO impl.YarnClientImpl: Submitted application application_1535165327468_0012
18/08/26 01:30:59 INFO mapreduce.Job: The url to track the job: http://ubuntu:8088/proxy/application_1535165327468_0012/
18/08/26 01:30:59 INFO mapreduce.Job: Running job: job_1535165327468_0012
18/08/26 01:31:04 INFO mapreduce.Job: Job job_1535165327468_0012 running in uber mode : false
18/08/26 01:31:04 INFO mapreduce.Job: map 0% reduce 0%
18/08/26 01:31:08 INFO mapreduce.Job: Task Id : attempt_1535165327468_0012_m_000000_0, Status : FAILED
Error: java.lang.ClassNotFoundException: scala.Function2
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
我想知道是否需要hadoop的额外java包来支持scala-mr?我没有在pom.xml中指定任何定制的包语句,我只是“mvnpackage”来生成我的jar,看起来还可以。
我怎样才能解决这个问题?
1条答案
按热度按时间zqdjd7g91#
听起来你好像错过了scala标准库。尝试将org.scala-lang/scala library/2.12.6添加到依赖项中。