com.clearspring.analytics.stream.cardinality.HyperLogLog.cardinality()方法的使用及代码示例

x33g5p2x  于2022-01-20 转载在 其他  
字(9.6k)|赞(0)|评价(0)|浏览(161)

本文整理了Java中com.clearspring.analytics.stream.cardinality.HyperLogLog.cardinality()方法的一些代码示例，展示了HyperLogLog.cardinality()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。HyperLogLog.cardinality()方法的具体详情如下：
包路径：com.clearspring.analytics.stream.cardinality.HyperLogLog
类名称：HyperLogLog
方法名：cardinality

HyperLogLog.cardinality介绍

暂无

代码示例

代码示例来源：origin: apache/incubator-pinot

@Nonnull
@Override
public Long extractFinalResult(@Nonnull HyperLogLog intermediateResult) {
 return intermediateResult.cardinality();
}

代码示例来源：origin: apache/incubator-pinot

@Nonnull
@Override
public Long extractFinalResult(@Nonnull HyperLogLog intermediateResult) {
 return intermediateResult.cardinality();
}

代码示例来源：origin: apache/incubator-pinot

@Override
 void assertAggregatedValue(HyperLogLog starTreeResult, HyperLogLog nonStarTreeResult) {
  assertEquals(starTreeResult.cardinality(), nonStarTreeResult.cardinality());
 }
}

代码示例来源：origin: apache/incubator-pinot

@Override
 void assertAggregatedValue(HyperLogLog starTreeResult, HyperLogLog nonStarTreeResult) {
  assertEquals(starTreeResult.cardinality(), nonStarTreeResult.cardinality());
 }
}

代码示例来源：origin: apache/incubator-pinot

/**
 * To display a row with hll fields properly,
 * instead of directly invoking {@link GenericRow#toString()},
 * hll fields should be inspected and transformed.
 *
 * @param row GenericRow
 * @param hllDeriveColumnSuffix column with this suffix will be treated as hll column
 * @return string representation of row
 */
public static String inspectGenericRow(GenericRow row, String hllDeriveColumnSuffix) {
 StringBuilder b = new StringBuilder();
 for (String name : row.getFieldNames()) {
  b.append(name);
  b.append(" : ");
  Object value = row.getValue(name);
  if (value instanceof String && name.endsWith(hllDeriveColumnSuffix)) {
   // hll field
   b.append(convertStringToHll((String) value).cardinality());
  } else if (value instanceof Object[]) {
   b.append(Arrays.toString((Object[]) value));
  } else {
   b.append(value);
  }
  b.append(", ");
 }
 return b.toString();
}

代码示例来源：origin: apache/incubator-pinot

@Test
public void testHyperLogLog() {
 for (int i = 0; i < NUM_ITERATIONS; i++) {
  HyperLogLog expected = new HyperLogLog(7);
  byte[] bytes = ObjectSerDeUtils.serialize(expected);
  HyperLogLog actual = ObjectSerDeUtils.deserialize(bytes, ObjectSerDeUtils.ObjectType.HyperLogLog);
  assertEquals(actual.cardinality(), expected.cardinality(), ERROR_MESSAGE);
 }
}

代码示例来源：origin: addthis/stream-lib

@Test
public void testHighCardinality() {
  long start = System.currentTimeMillis();
  HyperLogLog hyperLogLog = new HyperLogLog(10);
  int size = 10000000;
  for (int i = 0; i < size; i++) {
    hyperLogLog.offer(TestICardinality.streamElement(i));
  }
  System.out.println("time: " + (System.currentTimeMillis() - start));
  long estimate = hyperLogLog.cardinality();
  double err = Math.abs(estimate - size) / (double) size;
  System.out.println(err);
  assertTrue(err < .1);
}

代码示例来源：origin: addthis/stream-lib

@Test
public void testHighCardinality_withDefinedRSD() {
  long start = System.currentTimeMillis();
  HyperLogLog hyperLogLog = new HyperLogLog(0.01);
  int size = 100000000;
  for (int i = 0; i < size; i++) {
    hyperLogLog.offer(TestICardinality.streamElement(i));
  }
  System.out.println("time: " + (System.currentTimeMillis() - start));
  long estimate = hyperLogLog.cardinality();
  double err = Math.abs(estimate - size) / (double) size;
  System.out.println(err);
  assertTrue(err < .1);
}

代码示例来源：origin: apache/incubator-pinot

@Test
 public void testHllFieldSerializedSize()
   throws Exception {
  for (int i = 5; i < 10; i++) {
   HyperLogLog hll = new HyperLogLog(i);
   Assert.assertEquals(HllSizeUtils.getHllFieldSizeFromLog2m(i), hll.getBytes().length);
   LOGGER.info("Estimated: " + hll.cardinality());
   for (int j = 0; j < 100; j++) {
    hll.offer(rand.nextLong());
   }
   Assert.assertEquals(HllSizeUtils.getHllFieldSizeFromLog2m(i), hll.getBytes().length);
   LOGGER.info("Estimated: " + hll.cardinality());
   for (int j = 0; j < 9900; j++) {
    hll.offer(rand.nextLong());
   }
   Assert.assertEquals(HllSizeUtils.getHllFieldSizeFromLog2m(i), hll.getBytes().length);
   LOGGER.info("Estimated: " + hll.cardinality());
  }
 }
}

代码示例来源：origin: addthis/stream-lib

@Test
public void testMerge() throws CardinalityMergeException {
  int numToMerge = 5;
  int bits = 16;
  int cardinality = 1000000;
  HyperLogLog[] hyperLogLogs = new HyperLogLog[numToMerge];
  HyperLogLog baseline = new HyperLogLog(bits);
  for (int i = 0; i < numToMerge; i++) {
    hyperLogLogs[i] = new HyperLogLog(bits);
    for (int j = 0; j < cardinality; j++) {
      double val = Math.random();
      hyperLogLogs[i].offer(val);
      baseline.offer(val);
    }
  }
  long expectedCardinality = numToMerge * cardinality;
  HyperLogLog hll = hyperLogLogs[0];
  hyperLogLogs = Arrays.asList(hyperLogLogs).subList(1, hyperLogLogs.length).toArray(new HyperLogLog[0]);
  long mergedEstimate = hll.merge(hyperLogLogs).cardinality();
  long baselineEstimate = baseline.cardinality();
  double se = expectedCardinality * (1.04 / Math.sqrt(Math.pow(2, bits)));
  System.out.println("Baseline estimate: " + baselineEstimate);
  System.out.println("Expect estimate: " + mergedEstimate + " is between " + (expectedCardinality - (3 * se)) + " and " + (expectedCardinality + (3 * se)));
  assertTrue(mergedEstimate >= expectedCardinality - (3 * se));
  assertTrue(mergedEstimate <= expectedCardinality + (3 * se));
  assertEquals(mergedEstimate, baselineEstimate);
}

代码示例来源：origin: apache/incubator-pinot

List<Double> finalResult = new ArrayList<>();
for (HyperLogLog hyperLogLog : hyperLogLogs) {
 finalResult.add((double) hyperLogLog.cardinality());

代码示例来源：origin: addthis/stream-lib

long baselineEstimate = baseline.cardinality();
long g128Estimate = guava128.cardinality();
double se = cardinality * (1.04 / Math.sqrt(Math.pow(2, b)));
double baselineError = (baselineEstimate - cardinality) / (double) cardinality;

代码示例来源：origin: apache/incubator-pinot

QueriesTestUtils.testInnerSegmentExecutionStatistics(executionStatistics, 1L, 0L, 2L, 30000L);
List<Object> aggregationResult = resultsBlock.getAggregationResult();
Assert.assertEquals(((HyperLogLog) aggregationResult.get(0)).cardinality(), 21L);
Assert.assertEquals(((HyperLogLog) aggregationResult.get(1)).cardinality(), 1762L);
QueriesTestUtils.testInnerSegmentExecutionStatistics(executionStatistics, 6129L, 112472L, 12258L, 30000L);
aggregationResult = resultsBlock.getAggregationResult();
Assert.assertEquals(((HyperLogLog) aggregationResult.get(0)).cardinality(), 17L);
Assert.assertEquals(((HyperLogLog) aggregationResult.get(1)).cardinality(), 1197L);
GroupKeyGenerator.GroupKey firstGroupKey = aggregationGroupByResult.getGroupKeyIterator().next();
Assert.assertEquals(firstGroupKey._stringKey, "");
Assert.assertEquals(((HyperLogLog) aggregationGroupByResult.getResultForKey(firstGroupKey, 0)).cardinality(), 21L);
Assert.assertEquals(((HyperLogLog) aggregationGroupByResult.getResultForKey(firstGroupKey, 1)).cardinality(), 691L);

代码示例来源：origin: apache/incubator-pinot

QueriesTestUtils.testInnerSegmentExecutionStatistics(executionStatistics, 30000L, 0L, 60000L, 30000L);
List<Object> aggregationResult = resultsBlock.getAggregationResult();
Assert.assertEquals(((HyperLogLog) aggregationResult.get(0)).cardinality(), 21L);
Assert.assertEquals(((HyperLogLog) aggregationResult.get(1)).cardinality(), 1762L);
QueriesTestUtils.testInnerSegmentExecutionStatistics(executionStatistics, 6129L, 84134L, 12258L, 30000L);
aggregationResult = resultsBlock.getAggregationResult();
Assert.assertEquals(((HyperLogLog) aggregationResult.get(0)).cardinality(), 17L);
Assert.assertEquals(((HyperLogLog) aggregationResult.get(1)).cardinality(), 1197L);
GroupKeyGenerator.GroupKey firstGroupKey = aggregationGroupByResult.getGroupKeyIterator().next();
Assert.assertEquals(firstGroupKey._stringKey, "");
Assert.assertEquals(((HyperLogLog) aggregationGroupByResult.getResultForKey(firstGroupKey, 0)).cardinality(), 21L);
Assert.assertEquals(((HyperLogLog) aggregationGroupByResult.getResultForKey(firstGroupKey, 1)).cardinality(), 691L);

代码示例来源：origin: addthis/stream-lib

@Test
public void testSerialization() throws IOException, ClassNotFoundException {
  HyperLogLog hll = new HyperLogLog(8);
  hll.offer("a");
  hll.offer("b");
  hll.offer("c");
  hll.offer("d");
  hll.offer("e");
  HyperLogLog hll2 = (HyperLogLog) TestUtils.deserialize(TestUtils.serialize(hll));
  assertEquals(hll.cardinality(), hll2.cardinality());
}

代码示例来源：origin: addthis/stream-lib

@Test
public void testSerializationUsingBuilder() throws IOException {
  HyperLogLog hll = new HyperLogLog(8);
  hll.offer("a");
  hll.offer("b");
  hll.offer("c");
  hll.offer("d");
  hll.offer("e");
  HyperLogLog hll2 = HyperLogLog.Builder.build(hll.getBytes());
  assertEquals(hll.cardinality(), hll2.cardinality());
}

代码示例来源：origin: apache/incubator-pinot

assertApproximation(HllUtil.convertStringToHll((String) lastRow.getValue(hllMetricName)).cardinality(),
  preciseCardinality, 0.1);

代码示例来源：origin: addthis/stream-lib

@Test
public void testComputeCount() {
  HyperLogLog hyperLogLog = new HyperLogLog(16);
  hyperLogLog.offer(0);
  hyperLogLog.offer(1);
  hyperLogLog.offer(2);
  hyperLogLog.offer(3);
  hyperLogLog.offer(16);
  hyperLogLog.offer(17);
  hyperLogLog.offer(18);
  hyperLogLog.offer(19);
  hyperLogLog.offer(19);
  assertEquals(8, hyperLogLog.cardinality());
}

代码示例来源：origin: com.github.ddth/ddth-simplehll-core

/**
 * {@inheritDoc}
 */
@Override
public long count() {
  if (hll == null) {
    throw new IllegalStateException();
  }
  return hll.cardinality();
}

代码示例来源：origin: LiveRamp/cascading_ext

@Override
public void cleanup(FlowProcess flowProcess, OperationCall operationCall) {
 JobConf conf = (JobConf) flowProcess.getConfigCopy();
 try {
  LOG.info("HLL counter found " + approxCounter.cardinality() + " distinct keys");
  Hfs tap = new Hfs(new SequenceFile(new Fields("bytes")), BloomProps.getApproxCountsDir(conf));
  TupleEntryCollector out = tap.openForWrite(new HadoopFlowProcess(conf));
  out.add(new Tuple(new BytesWritable(approxCounter.getBytes())));
  out.close();
 } catch (IOException e) {
  throw new RuntimeException("couldn't write approximate counts to side bucket", e);
 }
}

内容来源于网络，如有侵权，请联系作者删除！

HyperLogLog

相关文章

    查看更多

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备2022004421号-2 NULL123 https://www.null123.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com