本文整理了Java中org.apache.hadoop.hive.ql.io.orc.Reader.hasMetadataValue
方法的一些代码示例,展示了Reader.hasMetadataValue
的具体用法。这些代码示例主要来源于Github
/Stackoverflow
/Maven
等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Reader.hasMetadataValue
方法的具体详情如下:
包路径:org.apache.hadoop.hive.ql.io.orc.Reader
类名称:Reader
方法名:hasMetadataValue
[英]Did the user set the given metadata value.
[中]用户是否设置了给定的元数据值。
代码示例来源:origin: apache/hive
public static boolean isOriginal(Reader file) {
return !file.hasMetadataValue(OrcRecordUpdater.ACID_KEY_INDEX_NAME);
}
代码示例来源:origin: apache/drill
public static boolean isOriginal(Reader file) {
return !file.hasMetadataValue(OrcRecordUpdater.ACID_KEY_INDEX_NAME);
}
代码示例来源:origin: apache/hive
static RecordIdentifier[] parseKeyIndex(Reader reader) {
String[] stripes;
try {
if (!reader.hasMetadataValue(OrcRecordUpdater.ACID_KEY_INDEX_NAME)) {
return null;
}
ByteBuffer val =
reader.getMetadataValue(OrcRecordUpdater.ACID_KEY_INDEX_NAME)
.duplicate();
stripes = utf8Decoder.decode(val).toString().split(";");
} catch (CharacterCodingException e) {
throw new IllegalArgumentException("Bad string encoding for " +
OrcRecordUpdater.ACID_KEY_INDEX_NAME, e);
}
RecordIdentifier[] result = new RecordIdentifier[stripes.length];
for(int i=0; i < stripes.length; ++i) {
if (stripes[i].length() != 0) {
String[] parts = stripes[i].split(",");
result[i] = new RecordIdentifier();
result[i].setValues(Long.parseLong(parts[0]),
Integer.parseInt(parts[1]), Long.parseLong(parts[2]));
}
}
return result;
}
代码示例来源:origin: apache/hive
private static boolean needsCompaction(FileStatus bucket, FileSystem fs) throws IOException {
//create reader, look at footer
//no need to check side file since it can only be in a streaming ingest delta
Reader orcReader = OrcFile.createReader(bucket.getPath(), OrcFile.readerOptions(fs.getConf()).filesystem(fs));
if (orcReader.hasMetadataValue(ACID_STATS)) {
try {
ByteBuffer val = orcReader.getMetadataValue(ACID_STATS).duplicate();
String acidStats = utf8Decoder.decode(val).toString();
String[] parts = acidStats.split(",");
long updates = Long.parseLong(parts[1]);
long deletes = Long.parseLong(parts[2]);
return deletes > 0 || updates > 0;
} catch (CharacterCodingException e) {
throw new IllegalArgumentException("Bad string encoding for " + ACID_STATS, e);
}
} else {
throw new IllegalStateException("AcidStats missing in " + bucket.getPath());
}
}
代码示例来源:origin: apache/hive
/**
* This is smart enough to handle streaming ingest where there could be a
* {@link OrcAcidUtils#DELTA_SIDE_FILE_SUFFIX} side file.
* @param dataFile - ORC acid data file
* @return version property from file if there,
* {@link #ORC_ACID_VERSION_DEFAULT} otherwise
*/
@VisibleForTesting
public static int getAcidVersionFromDataFile(Path dataFile, FileSystem fs) throws IOException {
FileStatus fileStatus = fs.getFileStatus(dataFile);
Reader orcReader = OrcFile.createReader(dataFile,
OrcFile.readerOptions(fs.getConf())
.filesystem(fs)
//make sure to check for side file in case streaming ingest died
.maxLength(getLogicalLength(fs, fileStatus)));
if (orcReader.hasMetadataValue(ACID_VERSION_KEY)) {
char[] versionChar = UTF8.decode(orcReader.getMetadataValue(ACID_VERSION_KEY)).array();
String version = new String(versionChar);
return Integer.valueOf(version);
}
return ORC_ACID_VERSION_DEFAULT;
}
/**
代码示例来源:origin: apache/hive
Mockito.when(recordReader.next(row3)).thenReturn(row5);
Mockito.when(reader.hasMetadataValue(OrcRecordUpdater.ACID_KEY_INDEX_NAME))
.thenReturn(true);
Mockito.when(reader.getMetadataValue(OrcRecordUpdater.ACID_KEY_INDEX_NAME))
代码示例来源:origin: com.facebook.presto.hive/hive-apache
public static boolean isOriginal(Reader file) {
return !file.hasMetadataValue(OrcRecordUpdater.ACID_KEY_INDEX_NAME);
}
代码示例来源:origin: com.hotels/corc-core
private boolean isAtomic(Reader orcReader) {
// Use org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.isOriginal(Reader) from hive-exec:1.1.0
boolean atomic = orcReader.hasMetadataValue(OrcRecordUpdater.ACID_KEY_INDEX_NAME);
LOG.debug("Atomic ORCFile: {}", atomic);
return atomic;
}
代码示例来源:origin: com.facebook.presto/presto-raptor
try (Closer<RecordReader, IOException> recordReader = closer(reader.rows(), RecordReader::close);
Closer<Writer, IOException> writer = closer(createWriter(path(output), writerOptions), Writer::close)) {
if (reader.hasMetadataValue(OrcFileMetadata.KEY)) {
ByteBuffer orcFileMetadata = reader.getMetadataValue(OrcFileMetadata.KEY);
writer.get().addUserMetadata(OrcFileMetadata.KEY, orcFileMetadata);
代码示例来源:origin: prestosql/presto
try (Closer<RecordReader, IOException> recordReader = closer(reader.rows(), RecordReader::close);
Closer<Writer, IOException> writer = closer(createWriter(path(output), writerOptions), Writer::close)) {
if (reader.hasMetadataValue(OrcFileMetadata.KEY)) {
ByteBuffer orcFileMetadata = reader.getMetadataValue(OrcFileMetadata.KEY);
writer.get().addUserMetadata(OrcFileMetadata.KEY, orcFileMetadata);
内容来源于网络,如有侵权,请联系作者删除!