解决spark core和azure key vault依赖项之间的guava冲突

lvmkulzt  于 2021-05-16  发布在  Spark
关注(0)|答案(0)|浏览(382)

我正在试用一个java中的spark应用程序,它需要连接到azure密钥库,从blob存储读取数据并推送到sqlserver表之后获取一些机密和处理数据。这是我的pom:

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.my.package</groupId>
    <artifactId>hdinsightSample</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <packaging>jar</packaging>

    <name>hdinsightSample</name>
    <url>http://maven.apache.org</url>

    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>

    <dependencies>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>3.8.1</version>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.12</artifactId>
            <version>2.4.7</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql_2.12</artifactId>
            <version>2.4.7</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-azure</artifactId>
            <version>2.10.1</version>
        </dependency>
        <dependency>
            <groupId>com.microsoft.sqlserver</groupId>
            <artifactId>mssql-jdbc</artifactId>
            <version>6.2.2.jre8</version>
        </dependency>
        <dependency>
            <groupId>com.microsoft.azure</groupId>
            <artifactId>azure-keyvault</artifactId>
            <version>1.0.0</version>
        </dependency>
        <dependency>
            <groupId>com.microsoft.azure</groupId>
            <artifactId>azure</artifactId>
            <version>1.3.0</version>
        </dependency>
    </dependencies>
    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-shade-plugin</artifactId>
                <executions>
                    <execution>
                        <phase>package</phase>
                        <goals>
                            <goal>shade</goal>
                        </goals>
                        <configuration>
                            <transformers>
                                <transformer
                                    implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                                    <mainClass>com.my.package.hdinsightSample.App</mainClass>
                                </transformer>
                            </transformers>
                            <filters>
                                <filter>
                                    <artifact>*:*</artifact>
                                    <excludes>
                                        <exclude>META-INF/maven/**</exclude>
                                        <exclude>META-INF/*.SF</exclude>
                                        <exclude>META-INF/*.DSA</exclude>
                                        <exclude>META-INF/*.RSA</exclude>
                                    </excludes>
                                </filter>
                            </filters>
                            <relocations>
                                <relocation>
                                    <pattern>com.google.common</pattern>
                                    <shadedPattern>repackaged.com.google.common</shadedPattern>
                                    <includes>
                                        <include>com.google.guava:*</include>
                                    </includes>
                                </relocation>
                            </relocations>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>
</project>

在运行jar文件或从eclipse运行代码之后,我得到以下异常:

Exception in thread "main" java.lang.NoClassDefFoundError: com.google.common.reflect.TypeToken
    at java.lang.ClassLoader.defineClassImpl(Native Method)
    at java.lang.ClassLoader.defineClass(ClassLoader.java:379)
    at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
    at java.net.URLClassLoader.defineClass(URLClassLoader.java:682)
    at java.net.URLClassLoader.access$400(URLClassLoader.java:89)
    at java.net.URLClassLoader$ClassFinder.run(URLClassLoader.java:1086)
    at java.security.AccessController.doPrivileged(AccessController.java:739)
    at java.net.URLClassLoader.findClass(URLClassLoader.java:589)
    at java.lang.ClassLoader.loadClassHelper(ClassLoader.java:934)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:879)
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:862)
    at com.microsoft.azure.keyvault.KeyVaultClientImpl.getSecretDelegate(KeyVaultClientImpl.java:2938)
    at com.microsoft.azure.keyvault.KeyVaultClientImpl.access$1800(KeyVaultClientImpl.java:92)
    at com.microsoft.azure.keyvault.KeyVaultClientImpl$82.call(KeyVaultClientImpl.java:2928)
    at com.microsoft.azure.keyvault.KeyVaultClientImpl$82.call(KeyVaultClientImpl.java:2924)
    at rx.internal.operators.OnSubscribeMap$MapSubscriber.onNext(OnSubscribeMap.java:69)
    at retrofit2.adapter.rxjava.CallArbiter.deliverResponse(CallArbiter.java:120)
    at retrofit2.adapter.rxjava.CallArbiter.emitResponse(CallArbiter.java:102)
    at retrofit2.adapter.rxjava.CallExecuteOnSubscribe.call(CallExecuteOnSubscribe.java:46)
    at retrofit2.adapter.rxjava.CallExecuteOnSubscribe.call(CallExecuteOnSubscribe.java:24)
    at rx.Observable.unsafeSubscribe(Observable.java:10142)
    at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:48)
    at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:33)
    at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:48)
    at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:30)
    at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:48)
    at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:30)
    at rx.Observable.subscribe(Observable.java:10238)
    at rx.Observable.subscribe(Observable.java:10205)
    at rx.observables.BlockingObservable.blockForSingle(BlockingObservable.java:444)
    at rx.observables.BlockingObservable.single(BlockingObservable.java:341)
    at com.microsoft.azure.keyvault.KeyVaultClientImpl.getSecret(KeyVaultClientImpl.java:2868)
    at com.microsoft.azure.keyvault.KeyVaultClient.getSecret(KeyVaultClient.java:862)
    at com.my.package.hdsinsightSample.utils.KeyVaultUtils.getSecret(KeyVaultUtils.java:12)
    at com.my.package.hdinsightSample.App.main(App.java:23)
Caused by: java.lang.ClassNotFoundException: com.google.common.reflect.TypeToken
    at java.net.URLClassLoader.findClass(URLClassLoader.java:591)
    at java.lang.ClassLoader.loadClassHelper(ClassLoader.java:934)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:879)
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:862)
    ... 36 more

这是因为从它使用的密钥库中获取秘密 guava jar但是 guava jar的默认版本是11.0.2(存在于spark core dependecny中),而它应该使用的正确版本是20(存在于azure key vault dependency中)。我试图阴影jar如上面的pom所示,但问题仍然是一样的。
请帮忙。

暂无答案!

目前还没有任何答案,快来回答吧!

相关问题