当我从终端提交spark作业时,我发现下面的错误是文件不存在。虽然我已经把配置文件放在了本地。
spark提交命令:
spark submit--class com.yotpo.metorikku.metorikku/home/hadoopuser/ideaprojects/metorikku/target/scala-2.11/metorikku.jar-c/home/hadoopuser/ideaprojects/metorikku/examples/movies.yaml
代码
import com.yotpo.metorikku.configuration.job.{ConfigurationParser, Periodic}
import com.yotpo.metorikku.metric.MetricSet
import org.apache.log4j.LogManager
object Metorikku extends App {
val log = LogManager.getLogger(this.getClass)
log.info("Starting Metorikku - Parsing configuration")
val session = Job(ConfigurationParser.parse(args))
session.config.periodic match {
case Some(periodic) => {
executePeriodicTask(periodic)
}
case _ => runMetrics(session)
}
private def executePeriodicTask(periodic: Periodic) = {
val task = new Runnable {
def run() = {
session.sparkSession.catalog.clearCache()
runMetrics(session)
}
}
val ex = new ScheduledThreadPoolExecutor(1)
val initialDelay = 0
ex.scheduleAtFixedRate(task, initialDelay, periodic.getTriggerDurationInSeconds(), TimeUnit.SECONDS)
}
def runMetrics(job: Job): Unit = {
job.config.metrics match {
case Some(metrics) => metrics.foreach(metricSetPath => {
val metricSet = new MetricSet(metricSetPath)
metricSet.run(job)
})
case None => log.warn("No mertics were defined, exiting")
}
}
}
错误
org.apache.hadoop.ipc.remoteexception(java.io.filenotfoundexception):文件不存在:/home/hadoopuser/ideaprojects/metorikku/examples/movies.yaml
暂无答案!
目前还没有任何答案,快来回答吧!