org.apache.hadoop.mapreduce.lib.output.MultipleOutputs.write()方法的使用及代码示例

x33g5p2x  于2022-01-25 转载在 其他  
字(10.2k)|赞(0)|评价(0)|浏览(134)

本文整理了Java中org.apache.hadoop.mapreduce.lib.output.MultipleOutputs.write()方法的一些代码示例，展示了MultipleOutputs.write()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。MultipleOutputs.write()方法的具体详情如下：
包路径：org.apache.hadoop.mapreduce.lib.output.MultipleOutputs
类名称：MultipleOutputs
方法名：write

MultipleOutputs.write介绍

[英]Write key value to an output file name. Gets the record writer from job's output format. Job's output format should be a FileOutputFormat.
[中]将键值写入输出文件名。从作业的输出格式获取记录编写器。作业的输出格式应为FileOutputFormat。

代码示例

代码示例来源：origin: apache/kylin

private void outputDimRangeInfo() throws IOException, InterruptedException {
  if (col != null && minValue != null) {
    // output written to baseDir/colName/colName.dci-r-00000 (etc)
    String dimRangeFileName = col.getIdentity() + "/" + col.getName() + DIMENSION_COL_INFO_FILE_POSTFIX;
    mos.write(BatchConstants.CFG_OUTPUT_PARTITION, NullWritable.get(),
        new Text(minValue.getBytes(StandardCharsets.UTF_8)), dimRangeFileName);
    mos.write(BatchConstants.CFG_OUTPUT_PARTITION, NullWritable.get(),
        new Text(maxValue.getBytes(StandardCharsets.UTF_8)), dimRangeFileName);
    logger.info("write dimension range info for col : " + col.getName() + "  minValue:" + minValue
        + " maxValue:" + maxValue);
  }
}

代码示例来源：origin: apache/kylin

@Override
public void doMap(Text key, Text value, Context context) throws IOException, InterruptedException {
  long cuboidID = rowKeySplitter.split(key.getBytes());
  if (cuboidID != baseCuboid && !recommendCuboids.contains(cuboidID)) {
    return;
  }
  String baseOutputPath = PathNameCuboidOld;
  if (cuboidID == baseCuboid) {
    baseOutputPath = PathNameCuboidBase;
  }
  mos.write(key, value, generateFileName(baseOutputPath));
}

代码示例来源：origin: apache/kylin

private void outputDict(TblColRef col, Dictionary<String> dict) throws IOException, InterruptedException {
  // output written to baseDir/colName/colName.rldict-r-00000 (etc)
  String dictFileName = col.getIdentity() + "/" + col.getName() + DICT_FILE_POSTFIX;
  try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
      DataOutputStream outputStream = new DataOutputStream(baos);) {
    outputStream.writeUTF(dict.getClass().getName());
    dict.write(outputStream);
    mos.write(BatchConstants.CFG_OUTPUT_DICT, NullWritable.get(),
        new ArrayPrimitiveWritable(baos.toByteArray()), dictFileName);
  }
}

代码示例来源：origin: apache/kylin

private void outputDict(TblColRef col, Dictionary<String> dict) throws IOException, InterruptedException {
    // output written to baseDir/colName/colName.rldict-r-00000 (etc)
    String dictFileName = col.getIdentity() + "/" + col.getName() + DICT_FILE_POSTFIX;

    try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream outputStream = new DataOutputStream(baos);) {
      outputStream.writeUTF(dict.getClass().getName());
      dict.write(outputStream);

      mos.write(BatchConstants.CFG_OUTPUT_DICT, NullWritable.get(), new ArrayPrimitiveWritable(baos.toByteArray()), dictFileName);
    }
    mos.close();
  }
}

代码示例来源：origin: apache/kylin

private void outputStatistics(List<Long> allCuboids) throws IOException, InterruptedException {
  // output written to baseDir/statistics/statistics-r-00000 (etc)
  String statisticsFileName = BatchConstants.CFG_OUTPUT_STATISTICS + "/" + BatchConstants.CFG_OUTPUT_STATISTICS;
  ByteBuffer valueBuf = ByteBuffer.allocate(BufferedMeasureCodec.DEFAULT_BUFFER_SIZE);
  // mapper overlap ratio at key -1
  long grandTotal = 0;
  for (HLLCounter hll : cuboidHLLMap.values()) {
    grandTotal += hll.getCountEstimate();
  }
  double mapperOverlapRatio = grandTotal == 0 ? 0 : (double) totalRowsBeforeMerge / grandTotal;
  mos.write(BatchConstants.CFG_OUTPUT_STATISTICS, new LongWritable(-1),
      new BytesWritable(Bytes.toBytes(mapperOverlapRatio)), statisticsFileName);
  // mapper number at key -2
  mos.write(BatchConstants.CFG_OUTPUT_STATISTICS, new LongWritable(-2),
      new BytesWritable(Bytes.toBytes(baseCuboidRowCountInMappers.size())), statisticsFileName);
  // sampling percentage at key 0
  mos.write(BatchConstants.CFG_OUTPUT_STATISTICS, new LongWritable(0L),
      new BytesWritable(Bytes.toBytes(samplingPercentage)), statisticsFileName);
  for (long i : allCuboids) {
    valueBuf.clear();
    cuboidHLLMap.get(i).writeRegisters(valueBuf);
    valueBuf.flip();
    mos.write(BatchConstants.CFG_OUTPUT_STATISTICS, new LongWritable(i),
        new BytesWritable(valueBuf.array(), valueBuf.limit()), statisticsFileName);
  }
}

代码示例来源：origin: apache/kylin

@Override
public void doMap(Text key, Text value, Context context) throws IOException, InterruptedException {
  long cuboidID = rowKeySplitter.split(key.getBytes());
  Cuboid cuboid = Cuboid.findForMandatory(cubeDesc, cuboidID);
  int fullKeySize = buildKey(cuboid, rowKeySplitter.getSplitBuffers());
  outputKey.set(newKeyBuf.array(), 0, fullKeySize);
  String baseOutputPath = PathNameCuboidOld;
  if (cuboidID == baseCuboid) {
    baseOutputPath = PathNameCuboidBase;
  }
  mos.write(outputKey, value, generateFileName(baseOutputPath));
}

代码示例来源：origin: apache/kylin

mos.write(BatchConstants.CFG_OUTPUT_COLUMN, NullWritable.get(), new Text(keyBytes), fileName);

代码示例来源：origin: thinkaurelius/faunus

public void write(final String type, final Writable key, final Writable value) throws IOException, InterruptedException {
  if (this.testing) {
    if (type.equals(Tokens.SIDEEFFECT))
      this.context.write(key, value);
  } else
    this.outputs.write(type, key, value);
}

代码示例来源：origin: apache/incubator-rya

@Override
  public void reduce(Text key, Iterable<Text> values, Context context)
      throws IOException, InterruptedException {
    String out = key.toString();
    for (Text value : values) {
      mout.write(out, NullWritable.get(), value);
    }
  }
}

代码示例来源：origin: thinkaurelius/faunus

public void write(final String type, final Writable key, final Writable value) throws IOException, InterruptedException {
  if (this.testing) {
    if (type.equals(Tokens.SIDEEFFECT))
      this.context.write(key, value);
  } else
    this.outputs.write(type, key, value);
}

代码示例来源：origin: org.apache.hadoop/hadoop-mapred-test

public void map(LongWritable key, Text value, Context context)
  throws IOException, InterruptedException {
 context.write(key, value);
 if (value.toString().equals("a")) {
  mos.write(TEXT, key, new Text(TEXT));
  mos.write(SEQUENCE, new IntWritable(1), new Text(SEQUENCE),
   (SEQUENCE + "_A"));
  mos.write(SEQUENCE, new IntWritable(2), new Text(SEQUENCE),
   (SEQUENCE + "_B"));
 }
}

代码示例来源：origin: com.conversantmedia/mara-test

protected void verifyNamedOutput(MultipleOutputs multiOut, String name, Object key, Object value, VerificationMode mode) {
    ArgumentCaptor keyArg = ArgumentCaptor.forClass(key.getClass());
    ArgumentCaptor valueArg = ArgumentCaptor.forClass(value.getClass());
    try {
      verify(multiOut, mode).write(name, keyArg.capture(), valueArg.capture());
    } catch (IOException | InterruptedException e) {
      fail(e.getMessage());
    }
  }
}

代码示例来源：origin: org.apache.kylin/kylin-engine-mr

private void outputDimRangeInfo() throws IOException, InterruptedException {
  if (col != null && minValue != null) {
    // output written to baseDir/colName/colName.dci-r-00000 (etc)
    String dimRangeFileName = col.getIdentity() + "/" + col.getName() + DIMENSION_COL_INFO_FILE_POSTFIX;
    mos.write(BatchConstants.CFG_OUTPUT_PARTITION, NullWritable.get(),
        new Text(minValue.getBytes(StandardCharsets.UTF_8)), dimRangeFileName);
    mos.write(BatchConstants.CFG_OUTPUT_PARTITION, NullWritable.get(),
        new Text(maxValue.getBytes(StandardCharsets.UTF_8)), dimRangeFileName);
    logger.info("write dimension range info for col : " + col.getName() + "  minValue:" + minValue
        + " maxValue:" + maxValue);
  }
}

代码示例来源：origin: org.apache.kylin/kylin-engine-mr

@Override
public void doMap(Text key, Text value, Context context) throws IOException, InterruptedException {
  long cuboidID = rowKeySplitter.split(key.getBytes());
  if (cuboidID != baseCuboid && !recommendCuboids.contains(cuboidID)) {
    return;
  }
  String baseOutputPath = PathNameCuboidOld;
  if (cuboidID == baseCuboid) {
    baseOutputPath = PathNameCuboidBase;
  }
  mos.write(key, value, generateFileName(baseOutputPath));
}

代码示例来源：origin: pl.edu.icm.coansys/coansys-io-input

void mergeProjects(String key, Iterable<BytesWritable> values) throws IOException, InterruptedException {
  List<ProjectProtos.ProjectWrapper> pwList = new ArrayList<>();
  values.forEach((bw) -> {
    try {
      pwList.add(ProjectProtos.ProjectWrapper.parseFrom(bw.copyBytes()));
    } catch (InvalidProtocolBufferException ex) {
      Logger.getLogger(HBaseToProtosReducer.class.getName()).log(Level.SEVERE, null, ex);
    }
  });
  ProjectProtos.ProjectWrapper merged = projectMerger.merge(pwList);
  mos.write(new Text(key), new BytesWritable(merged.toByteArray()), Type.PROJECT.name()+"/");
}

代码示例来源：origin: pl.edu.icm.coansys/coansys-io-input

void mergeDocuments(String key, Iterable<BytesWritable> values) throws IOException, InterruptedException {
  List<DocumentProtos.DocumentWrapper> dwList = new ArrayList<>();
  values.forEach((bw) -> {
    try {
      dwList.add(DocumentProtos.DocumentWrapper.parseFrom(bw.copyBytes()));
    } catch (InvalidProtocolBufferException ex) {
      Logger.getLogger(HBaseToProtosReducer.class.getName()).log(Level.SEVERE, null, ex);
    }
  });
  DocumentProtos.DocumentWrapper merged = docDuplicatesMerger.merge(dwList);
  mos.write(new Text(key), new BytesWritable(merged.toByteArray()), Type.DOCUMENT.name()+"/");
}

代码示例来源：origin: ch.cern.hadoop/hadoop-mapreduce-client-jobclient

public void reduce(Long key, Iterable<String> values, 
  Context context) throws IOException, InterruptedException {
 for (String value : values) {
  mos.write(key, value, value.toString());
  if (!value.toString().equals("b")) {
   context.write(key, value);
  } else {
   mos.write(TEXT, key, new Text(TEXT));
  }
 }
}

代码示例来源：origin: ch.cern.hadoop/hadoop-mapreduce-client-jobclient

public void map(LongWritable key, Text value, Context context)
  throws IOException, InterruptedException {
 context.write(key, value);
 if (value.toString().equals("a")) {
  mos.write(TEXT, key, new Text(TEXT));
  mos.write(SEQUENCE, new IntWritable(1), new Text(SEQUENCE),
   (SEQUENCE + "_A"));
  mos.write(SEQUENCE, new IntWritable(2), new Text(SEQUENCE),
   (SEQUENCE + "_B"));
 }
}

代码示例来源：origin: pl.edu.icm.coansys/coansys-io-input

protected void writeDecision(Writable oldKey, Decision decision) throws IOException, InterruptedException {
  FilterContstants.OUTPUT out = getStringForDecision(decision);
  if (out == FilterContstants.OUTPUT.UNKNOWN) {
    mos.write(out.getName(), oldKey, decision,undecidedDir+"/undecided");
  } else {
    
    mos.write(out.getName(), new Text(getNameOfFilter()), decision, mainOutputsDir + "/" + out.getName() + "/out");
  }
}

代码示例来源：origin: ch.cern.hadoop/hadoop-mapreduce-client-jobclient

public void map(LongWritable key, Text value, Context context)
  throws IOException, InterruptedException {
 context.write(key.get(), value.toString());
 if (value.toString().equals("a")) {
  mos.write(TEXT, key.get(), TEXT);
 }
}

内容来源于网络，如有侵权，请联系作者删除！

MultipleOutputs

相关文章

    查看更多

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备2022004421号-2 NULL123 https://www.null123.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com