我在kubernetes上用spark2.4.5/hadoop2.7.3开发了一个spark集群。一切正常。
现在我尝试将spark和hadoop的版本分别升级到3.0.0和3.2.0,但是当我尝试在aws eks pod上应用定制的映像时,我得到了以下消息。
我看到有人在Spark3.0.0上问了类似的问题,他发现这与 spark_uid
参数未在dockerfile上声明,但已设置为 ARG spark_uid=185
在dockerfile上。
在建立spark的定制形象方面有什么关键性的改变吗?
20/07/07 02:16:56 ERROR SparkUncaughtExceptionHandler: Uncaught exception in thread Thread[main,5,main]
org.apache.hadoop.security.KerberosAuthException: failure to login: javax.security.auth.login.LoginException: java.lang.NullPointerException: invalid null input: name
at com.sun.security.auth.UnixPrincipal.<init>(UnixPrincipal.java:71)
at com.sun.security.auth.module.UnixLoginModule.login(UnixLoginModule.java:133)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at javax.security.auth.login.LoginContext.invoke(LoginContext.java:755)
at javax.security.auth.login.LoginContext.access$000(LoginContext.java:195)
at javax.security.auth.login.LoginContext$4.run(LoginContext.java:682)
at javax.security.auth.login.LoginContext$4.run(LoginContext.java:680)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.login.LoginContext.invokePriv(LoginContext.java:680)
at javax.security.auth.login.LoginContext.login(LoginContext.java:587)
at org.apache.hadoop.security.UserGroupInformation.loginUserFromSubject(UserGroupInformation.java:858)
at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:820)
at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:689)
at org.apache.spark.util.Utils$.$anonfun$getCurrentUserName$1(Utils.scala:2412)
at scala.Option.getOrElse(Option.scala:189)
at org.apache.spark.util.Utils$.getCurrentUserName(Utils.scala:2412)
at org.apache.spark.SecurityManager.<init>(SecurityManager.scala:79)
at org.apache.spark.deploy.master.Master$.startRpcEnvAndEndpoint(Master.scala:1137)
at org.apache.spark.deploy.master.Master$.main(Master.scala:1122)
at org.apache.spark.deploy.master.Master.main(Master.scala)
at org.apache.hadoop.security.UserGroupInformation.loginUserFromSubject(UserGroupInformation.java:915)
at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:820)
at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:689)
at org.apache.spark.util.Utils$.$anonfun$getCurrentUserName$1(Utils.scala:2412)
at scala.Option.getOrElse(Option.scala:189)
at org.apache.spark.util.Utils$.getCurrentUserName(Utils.scala:2412)
at org.apache.spark.SecurityManager.<init>(SecurityManager.scala:79)
at org.apache.spark.deploy.master.Master$.startRpcEnvAndEndpoint(Master.scala:1137)
at org.apache.spark.deploy.master.Master$.main(Master.scala:1122)
at org.apache.spark.deploy.master.Master.main(Master.scala)
Caused by: javax.security.auth.login.LoginException: java.lang.NullPointerException: invalid null input: name
at com.sun.security.auth.UnixPrincipal.<init>(UnixPrincipal.java:71)
at com.sun.security.auth.module.UnixLoginModule.login(UnixLoginModule.java:133)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at javax.security.auth.login.LoginContext.invoke(LoginContext.java:755)
at javax.security.auth.login.LoginContext.access$000(LoginContext.java:195)
at javax.security.auth.login.LoginContext$4.run(LoginContext.java:682)
at javax.security.auth.login.LoginContext$4.run(LoginContext.java:680)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.login.LoginContext.invokePriv(LoginContext.java:680)
at javax.security.auth.login.LoginContext.login(LoginContext.java:587)
at org.apache.hadoop.security.UserGroupInformation.loginUserFromSubject(UserGroupInformation.java:858)
at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:820)
at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:689)
at org.apache.spark.util.Utils$.$anonfun$getCurrentUserName$1(Utils.scala:2412)
at scala.Option.getOrElse(Option.scala:189)
at org.apache.spark.util.Utils$.getCurrentUserName(Utils.scala:2412)
at org.apache.spark.SecurityManager.<init>(SecurityManager.scala:79)
at org.apache.spark.deploy.master.Master$.startRpcEnvAndEndpoint(Master.scala:1137)
at org.apache.spark.deploy.master.Master$.main(Master.scala:1122)
at org.apache.spark.deploy.master.Master.main(Master.scala)
暂无答案!
目前还没有任何答案,快来回答吧!