parquet.io.api.Binary类的使用及代码示例

x33g5p2x  于2022-01-17 转载在 其他  
字(6.9k)|赞(0)|评价(0)|浏览(239)

本文整理了Java中parquet.io.api.Binary类的一些代码示例,展示了Binary类的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Binary类的具体详情如下:
包路径:parquet.io.api.Binary
类名称:Binary

Binary介绍

暂无

代码示例

代码示例来源:origin: prestodb/presto

case STRING:
  String v = ((StringObjectInspector) inspector).getPrimitiveJavaObject(value);
  recordConsumer.addBinary(Binary.fromString(v));
  break;
case CHAR:
  String vChar = ((HiveCharObjectInspector) inspector).getPrimitiveJavaObject(value).getStrippedValue();
  recordConsumer.addBinary(Binary.fromString(vChar));
  break;
case VARCHAR:
  String vVarchar = ((HiveVarcharObjectInspector) inspector).getPrimitiveJavaObject(value).getValue();
  recordConsumer.addBinary(Binary.fromString(vVarchar));
  break;
case BINARY:
  byte[] vBinary = ((BinaryObjectInspector) inspector).getPrimitiveJavaObject(value);
  recordConsumer.addBinary(Binary.fromByteArray(vBinary));
  break;
case TIMESTAMP:

代码示例来源:origin: prestodb/presto

/**
 * Returns GMT timestamp from binary encoded parquet timestamp (12 bytes - julian date + time of day nanos).
 *
 * @param timestampBinary INT96 parquet timestamp
 * @return timestamp in millis, GMT timezone
 */
public static long getTimestampMillis(Binary timestampBinary)
{
  if (timestampBinary.length() != 12) {
    throw new PrestoException(NOT_SUPPORTED, "Parquet timestamp must be 12 bytes, actual " + timestampBinary.length());
  }
  byte[] bytes = timestampBinary.getBytes();
  // little endian encoding - need to invert byte order
  long timeOfDayNanos = Longs.fromBytes(bytes[7], bytes[6], bytes[5], bytes[4], bytes[3], bytes[2], bytes[1], bytes[0]);
  int julianDay = Ints.fromBytes(bytes[11], bytes[10], bytes[9], bytes[8]);
  return julianDayToMillis(julianDay) + (timeOfDayNanos / NANOS_PER_MILLISECOND);
}

代码示例来源:origin: prestodb/presto

public BinaryDictionary(DictionaryPage dictionaryPage, Integer length)
    throws IOException
{
  super(dictionaryPage.getEncoding());
  byte[] dictionaryBytes = dictionaryPage.getSlice().getBytes();
  content = new Binary[dictionaryPage.getDictionarySize()];
  int offset = 0;
  if (length == null) {
    for (int i = 0; i < content.length; i++) {
      int len = readIntLittleEndian(dictionaryBytes, offset);
      offset += 4;
      content[i] = Binary.fromByteArray(dictionaryBytes, offset, len);
      offset += len;
    }
  }
  else {
    checkArgument(length > 0, "Invalid byte array length: %s", length);
    for (int i = 0; i < content.length; i++) {
      content[i] = Binary.fromByteArray(dictionaryBytes, offset, length);
      offset += length;
    }
  }
}

代码示例来源:origin: apache/incubator-gobblin

@Override
BinaryValue convertField(JsonElement value) {
 return new BinaryValue(Binary.fromString(value.getAsString()));
}

代码示例来源:origin: prestodb/presto

@Override
protected void readValue(BlockBuilder blockBuilder, Type type)
{
  if (definitionLevel == columnDescriptor.getMaxDefinitionLevel()) {
    Binary value = valuesReader.readBytes();
    type.writeSlice(blockBuilder, Decimals.encodeUnscaledValue(new BigInteger(value.getBytes())));
  }
  else if (isValueNull()) {
    blockBuilder.appendNull();
  }
}

代码示例来源:origin: com.facebook.presto.hive/hive-apache

protected static Binary copy(Binary binary) {
  return Binary.fromByteArray(
    Arrays.copyOf(binary.getBytes(), binary.length()));
 }
}

代码示例来源:origin: com.facebook.presto.hive/hive-apache

@Override
public String getString() {
 return binary.toStringUsingUTF8();
}

代码示例来源:origin: com.facebook.presto.hive/hive-apache

@Override
 public void writeBytes(Binary v) {
  int i = 0;
  byte[] vb = v.getBytes();
  int length = previous.length < vb.length ? previous.length : vb.length;
  for(i = 0; (i < length) && (previous[i] == vb[i]); i++);
  prefixLengthWriter.writeInteger(i);
  suffixWriter.writeBytes(Binary.fromByteArray(vb, i, vb.length - i));
  previous = vb;
 }
}

代码示例来源:origin: com.twitter/parquet-tools

public static String binaryToString(Binary value) {
  byte[] data = value.getBytes();
  if (data == null) return null;
  try {
    CharBuffer buffer = UTF8_DECODER.decode(value.toByteBuffer());
    return buffer.toString();
  } catch (Throwable th) {
  }
  return "<bytes...>";
}

代码示例来源:origin: com.facebook.presto.hive/hive-apache

public static NanoTime fromBinary(Binary bytes) {
 Preconditions.checkArgument(bytes.length() == 12, "Must be 12 bytes");
 ByteBuffer buf = bytes.toByteBuffer();
 buf.order(ByteOrder.LITTLE_ENDIAN);
 long timeOfDayNanos = buf.getLong();
 int julianDay = buf.getInt();
 return new NanoTime(julianDay, timeOfDayNanos);
}

代码示例来源:origin: com.facebook.presto.hive/hive-apache

public void writeBytes(Binary v) {
 //for rawdata, length(4 bytes int) is stored, followed by the binary content itself
 rawDataByteSize += v.length() + 4;
 currentWriter.writeBytes(v);
 checkFallback();
}

代码示例来源:origin: com.facebook.presto.hive/hive-apache

@Override
public final void writeBytes(Binary v) {
 if (v.length() != length) {
  throw new IllegalArgumentException("Fixed Binary size " + v.length() +
    " does not match field type length " + length);
 }
 try {
  v.writeTo(out);
 } catch (IOException e) {
  throw new ParquetEncodingException("could not write fixed bytes", e);
 }
}

代码示例来源:origin: asakusafw/asakusafw

@Override
public void addBinary(Binary value) {
  ByteBuffer bytes = value.toByteBuffer().order(ByteOrder.LITTLE_ENDIAN);
  long time = bytes.getLong();
  int day = bytes.getInt();
  addNanoTime(day, time);
}

代码示例来源:origin: prestodb/presto

@Test
public void testInvalidBinaryLength()
{
  try {
    byte[] invalidLengthBinaryTimestamp = new byte[8];
    getTimestampMillis(Binary.fromByteArray(invalidLengthBinaryTimestamp));
  }
  catch (PrestoException e) {
    assertEquals(e.getErrorCode(), NOT_SUPPORTED.toErrorCode());
    assertEquals(e.getMessage(), "Parquet timestamp must be 12 bytes, actual 8");
  }
}

代码示例来源:origin: prestodb/presto

private static BinaryStatistics stringColumnStats(String minimum, String maximum)
{
  BinaryStatistics statistics = new BinaryStatistics();
  statistics.setMinMax(Binary.fromString(minimum), Binary.fromString(maximum));
  return statistics;
}

代码示例来源:origin: prestodb/presto

List<Domain> domains = new ArrayList<>();
for (int i = 0; i < dictionarySize; i++) {
  domains.add(Domain.singleValue(type, Slices.wrappedBuffer(dictionary.decodeToBinary(i).getBytes())));

代码示例来源:origin: com.facebook.presto.hive/hive-apache

@Override
 public Binary readBytes() {
  int prefixLength = prefixLengthReader.readInteger();
  // This does not copy bytes
  Binary suffix = suffixReader.readBytes();
  int length = prefixLength + suffix.length();
  
  // We have to do this to materialize the output
  if(prefixLength != 0) {
   byte[] out = new byte[length];
   System.arraycopy(previous.getBytes(), 0, out, 0, prefixLength);
   System.arraycopy(suffix.getBytes(), 0, out, prefixLength, suffix.length());
   previous =  Binary.fromByteArray(out);
  } else {
   previous = suffix;
  }
  return previous;
 }
}

代码示例来源:origin: com.twitter/parquet-pig

@Override
final public void addBinary(Binary value) {
 currentKey = value.toStringUsingUTF8();
}

代码示例来源:origin: com.facebook.presto.hive/hive-apache

public static NanoTime fromBinary(Binary bytes) {
 Preconditions.checkArgument(bytes.length() == 12, "Must be 12 bytes");
 ByteBuffer buf = bytes.toByteBuffer();
 buf.order(ByteOrder.LITTLE_ENDIAN);
 long timeOfDayNanos = buf.getLong();
 int julianDay = buf.getInt();
 return new NanoTime(julianDay, timeOfDayNanos);
}

代码示例来源:origin: com.facebook.presto.hive/hive-apache

@Override
public void writeBytes(Binary v) {
 int id = binaryDictionaryContent.getInt(v);
 if (id == -1) {
  id = binaryDictionaryContent.size();
  binaryDictionaryContent.put(copy(v), id);
  // length as int (4 bytes) + actual bytes
  dictionaryByteSize += 4 + v.length();
 }
 encodedValues.add(id);
}

相关文章