hive单元测试在添加spark hive依赖项时不起作用

z9zf31ra  于 2021-06-27  发布在  Hive
关注(0)|答案(0)|浏览(161)

当我尝试执行单元测试时，hiverunner遇到一个错误，这是我的测试类：

import com.klarna.hiverunner.HiveShell;
import com.klarna.hiverunner.StandaloneHiveRunner;
import com.klarna.hiverunner.annotations.HiveSQL;
import org.junit.*;
import org.junit.rules.TestName;
import org.junit.runner.RunWith;

import java.util.Arrays;
import java.util.List;

@RunWith(StandaloneHiveRunner.class)
public class MiniClusterHiveTest {

    @Rule
    public TestName name = new TestName();

    @HiveSQL(files = {})
    private HiveShell shell;

    @Before
    public void createDatabaseAndTable() {
        shell.execute("CREATE DATABASE source_db");
        shell.execute("CREATE TABLE source_db.survey (" +
                "submittedTime STRING," +
                "age STRING," +
                "gender STRING," +
                "country STRING," +
                "state STRING," +
                "self_employed STRING," +
                "family_history STRING," +
                "treatment STRING," +
                "work_interfere STRING," +
                "no_employees STRING," +
                "remote_work STRING," +
                "tech_company STRING," +
                "benefits STRING," +
                "care_options STRING," +
                "wellness_program STRING," +
                "seek_help STRING," +
                "anonymity STRING," +
                "leave STRING," +
                "mental_health_consequence STRING," +
                "phys_health_consequence STRING," +
                "coworkers STRING," +
                "supervisor STRING," +
                "mental_health_interview STRING," +
                "phys_health_interview STRING," +
                "mental_vs_physical STRING," +
                "obs_consequence STRING," +
                "comments STRING) " +
                "ROW FORMAT DELIMITED " +
                "FIELDS TERMINATED BY ',' " +
                "STORED AS TEXTFILE " +
                "LOCATION '/home/datasets/'");
    }

    @Before
    public void createORCTable() {
        shell.execute("CREATE TABLE source_db.survey2 (" +
                "submittedTime STRING," +
                "age STRING," +
                "gender STRING," +
                "country STRING," +
                "state STRING," +
                "self_employed STRING," +
                "family_history STRING," +
                "treatment STRING," +
                "work_interfere STRING," +
                "no_employees STRING," +
                "remote_work STRING," +
                "tech_company STRING," +
                "benefits STRING," +
                "care_options STRING," +
                "wellness_program STRING," +
                "seek_help STRING," +
                "anonymity STRING," +
                "leave STRING," +
                "mental_health_consequence STRING," +
                "phys_health_consequence STRING," +
                "coworkers STRING," +
                "supervisor STRING," +
                "mental_health_interview STRING," +
                "phys_health_interview STRING," +
                "mental_vs_physical STRING," +
                "obs_consequence STRING," +
                "comments STRING) " +
                "ROW FORMAT DELIMITED " +
                "FIELDS TERMINATED BY ',' " +
                "STORED AS ORC tblproperties (\"orc.compress\"=\"ZLIB\"); ");

    }

    @Before
    public void createParquetTable() {
        shell.execute("CREATE TABLE source_db.survey3 " +
                "STORED AS PARQUET TBLPROPERTIES (\"parquet.compression\"=\"SNAPPY\")\n" +
                " AS SELECT * FROM source_db.survey;");
    }

    /**
     * We use temporary table survey to load the orc table survey2
     */
    @Test
    public void loadOrcSurvey2Table() {
        shell.execute("INSERT INTO TABLE source_db.survey2 SELECT * from source_db.survey");
    }

    @Ignore("Use for simple test without external file")
    public void simpleInit() {
        shell.execute("CREATE DATABASE source_db");
        shell.execute("CREATE TABLE source_db.test (" +
                "a STRING," +
                "b STRING ) ");

    }

    @Ignore
    public void simpleInsertionDataIntoTable() {
        shell.insertInto("source_db", "test")
                .withAllColumns()
                .addRow("bim", "bap")
                .commit();

        printResult(shell.executeStatement("select * from source_db.test"));
    }

    @Test
    public void executeQuery() {

        List<Object[]> result = shell.executeStatement("select * from source_db.survey where age=37");
        List<Object[]> result2 = shell.executeStatement("select * from source_db.survey where age=12");

        Assert.assertEquals(43L, result.size());
        Assert.assertEquals(0L, result2.size());
    }

    @Test
    public void insertDataIntoTable() {
        shell.insertInto("source_db", "survey")
                .withAllColumns()
                .addRow("2019-03-01 09:29:31",
                        17,
                        "Male",
                        "France",
                        "IL",
                        "NA",
                        "No",
                        "Yes",
                        "Often",
                        "6-25",
                        "No",
                        "Yes",
                        "Yes",
                        "Not sure",
                        "No",
                        "Yes",
                        "Yes",
                        "Somewhat easy",
                        "No",
                        "No",
                        "Some of them",
                        "Yes",
                        "No",
                        "Maybe",
                        "Yes",
                        "No",
                        "NA"
                )
                .commit();
        printResult(shell.executeStatement("select * from source_db.survey where age=17"));
    }

    private void printResult(List<Object[]> result) {
        System.out.println(String.format("Result from %s:", name.getMethodName()));
        result.stream().map(Arrays::asList).forEach(System.out::println);
    }
}

只要我不将此依赖项添加到pom.xml中，这个测试类就可以正常工作：

<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-hive_2.11</artifactId>
    <version>${spark.version}</version>
</dependency>

这是测试类工作时的pom.xml：

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.finaxys</groupId>
    <artifactId>MiniClusterTestBigData</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <java.version>1.8</java.version>
        <hadoop.version>2.7.3</hadoop.version>
        <spark.version>2.3.0</spark.version>
        <junit.version>4.12</junit.version>
        <maven.compiler.source>${java.version}</maven.compiler.source>
        <maven.compiler.target>${java.version}</maven.compiler.target>
    </properties>

    <dependencies>
        <!--hive dependencies-->
        <dependency>
            <groupId>com.klarna</groupId>
            <artifactId>hiverunner</artifactId>
            <version>4.1.0</version>
        </dependency>

        <!--hadoop dependencies-->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-minicluster</artifactId>
            <version>${hadoop.version}</version>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>${junit.version}</version>
            <scope>test</scope>
        </dependency>

        <!--Spark dependencies-->
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>

    </dependencies>
</project>

这是我在执行带有spark hive依赖项的测试类时遇到的错误：
java.lang.illegalstateexception:未能创建hiveserver:在配置单元配置上应用授权策略时出错：org.apache.hadoop.hive.ql.metadata.hiveexception:java.lang.runtimeexception:无法示例化org.apache.hadoop.hive.ql.metadata.sessionhivemetastoreclient
有人也有类似的问题，并在下面的帖子中解释：
https://github.com/klarna/hiverunner/issues/66
我尝试添加程序包，如所述，但仍无法解决问题：
org.apache.hadoop.hive.metastore.retryingmetastoreclient
你知道怎么修吗？

JavaMavenHiveunit-testing

暂无答案！

目前还没有任何答案，快来回答吧！

我来回答

相关问题

    查看更多

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新问答

    更多
    • 蜀ICP备2022004421号-2 NULL123 https://www.null123.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com