比较java中tsv文件和hbase表的值

polhcujo  于 2021-06-02  发布在  Hadoop
关注(0)|答案(1)|浏览(389)

我有一个hbase表,它有一个唯一的行键、一个列族和一个列。。我有一个tsv文件,它有大约300多列。此文件中的rowkey是两列的组合值。所以现在我需要比较表和文件中的rowkey,如果rowkey匹配,那么我需要插入表列值作为tsv文件中相应rowkey的最后一列。我已经编写了以下代码,但它总是执行else部分。

package mapReduce;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileReader;
import java.io.FileWriter;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;

public class Tsv_read{

    private static Configuration conf = null;

    static {
        conf = HBaseConfiguration.create();
    }

    @SuppressWarnings("resource")
    public static void main(String[] arg) throws Exception {

        BufferedReader TSVFile = 
                new BufferedReader(new FileReader("Path/to/file/.tsv"));

        String dataRow = TSVFile.readLine();
        List<String> list = new ArrayList<String>();

        while (dataRow != null){
            list.clear();
            String[] dataArray = dataRow.split("\t");

            for (String item:dataArray) { 

                HTable table = new HTable(conf, "Table name"); //Hbase table name
                Scan s = new Scan();
                ResultScanner ss = table.getScanner(s);
                for(Result r:ss){
                    for(KeyValue kv : r.raw()){
                        //System.out.println("Rowkey :" +dataArray[12]+"-"+dataArray[13]);
                        //System.out.print(new String(kv.getRow()) + " ");
                        if((dataArray[12]+"-"+dataArray[13]).equals(new String(kv.getRow()))){  //Comparing the rowkeys from file and table  (doesn't work)
                            System.out.println("File Rowkey :"+dataArray[12]+"-"+dataArray[13]);
                            System.out.println("Table Row key"+new String(kv.getRow()));
                            //dataArray[392]=new String(kv.getValue());
                            FileWriter fstream = new FileWriter("/path/to/the/file/*.tsv",true);
                            BufferedWriter fbw = new BufferedWriter(fstream);
                            fbw.write(new String(kv.getValue())); //inserting the value to the tsv file
                            fbw.newLine();
                            fbw.close();
                            System.out.println("Column value written succesfully");
                        }
                        else //always executes this part
                        {
                            System.out.println("RowKey not found :" +new String(kv.getRow()));
                        }
                        /*System.out.print(new String(kv.getFamily()) + ":");
                       System.out.print(new String(kv.getQualifier()) + " ");
                       System.out.print(kv.getTimestamp() + " ");*/
                        //System.out.println(new String(kv.getValue()));

                list.add(item);
                    }
                }
            } 
            Iterator<String> it = list.iterator();
            while (it.hasNext()) {
                String txt = it.next();
                System.out.print(txt+" ");
            } 
            System.out.println(); // Print the data line.
            dataRow = TSVFile.readLine(); 
        }

        TSVFile.close();

        System.out.println();

    } //main()
}

样品记录:
数据数组[12]+“-”+数据数组[13]=30496200139673452544-5172983457411783096
在hbase表中,rowkey也具有相同格式的值。
我无法共享整个记录,因为它有300多列。
tsv文件大小:大约10gb
hbase表:大约10254950行。
感谢您的帮助。提前谢谢。

56lgkhnf

56lgkhnf1#

而不是这样写
if((dataarray[12]+“-”+dataarray[13]).equals(new string(kv.getrow()){//比较文件和表中的行键(不起作用)
试试这个
if((dataarray[12]+“-”+dataarray[13]).equals(bytes.tostring(kv.getrow()){
您没有正确获取行值。
试试这个更新的代码,它使用get而不是hbase的扫描,运行时间更短

while (dataRow != null) {
        list.clear();
        String[] dataArray = dataRow.split("\t");

        for (String item : dataArray) {

            String key = dataArray[12] + "-" + dataArray[13];
            HTable table = new HTable(conf, "Table name"); // Hbase table
                                                            // name
            Get get = new Get(Bytes.toBytes(key));
            Result r = table.get(get);
            if (r != null && r.size() > 0) {
                for (KeyValue kv : r.raw()) {
                    System.out.println("File Rowkey :" + key);
                    System.out.println("Table Row key"
                            + Bytes.toString(kv.getRow()));
                    FileWriter fstream = new FileWriter(
                            "/path/to/the/file/*.tsv", true);
                    BufferedWriter fbw = new BufferedWriter(fstream);
                    fbw.write(new String(kv.getValue())); // inserting the
                                                            // value to the
                                                            // tsv file
                    fbw.newLine();
                    fbw.close();
                    System.out.println("Column value written succesfully");
                }
            } else {
                System.out.println("RowKey not found :" + key);
            }
            list.add(item);
        }
    }

相关问题