org.apache.beam.sdk.Pipeline.getOptions()方法的使用及代码示例

x33g5p2x  于2022-01-26 转载在 其他  
字(12.6k)|赞(0)|评价(0)|浏览(96)

本文整理了Java中org.apache.beam.sdk.Pipeline.getOptions()方法的一些代码示例，展示了Pipeline.getOptions()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。Pipeline.getOptions()方法的具体详情如下：
包路径：org.apache.beam.sdk.Pipeline
类名称：Pipeline
方法名：getOptions

Pipeline.getOptions介绍

暂无

代码示例

代码示例来源：origin: org.apache.beam/beam-runners-flink_2.10-examples

/**
 * Gets KafkaOptions from the Pipeline.
 * @param p the pipeline
 * @return KafkaOptions
 */
private static KafkaOptions getOptions(Pipeline p) {
 return p.getOptions().as(KafkaOptions.class);
}

代码示例来源：origin: org.apache.beam/beam-runners-core-construction-java

public static RunnerApi.Pipeline toProto(Pipeline pipeline, boolean useDeprecatedViewTransforms) {
 return toProto(
   pipeline, SdkComponents.create(pipeline.getOptions()), useDeprecatedViewTransforms);
}

代码示例来源：origin: org.apache.beam/beam-runners-core-construction-java

public static RunnerApi.Pipeline toProto(Pipeline pipeline) {
 return toProto(pipeline, SdkComponents.create(pipeline.getOptions()));
}

代码示例来源：origin: org.apache.beam/beam-runners-core-construction-java

private static <T> ReadPayload getReadPayload(
  AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>> transform)
  throws IOException {
 SdkComponents components = SdkComponents.create(transform.getPipeline().getOptions());
 return ReadPayload.parseFrom(
   PTransformTranslation.toProto(transform, Collections.emptyList(), components)
     .getSpec()
     .getPayload());
}

代码示例来源：origin: org.apache.beam/beam-runners-core-construction-java

private static <T> ParDoPayload getParDoPayload(AppliedPTransform<?, ?, ?> transform)
  throws IOException {
 SdkComponents components = SdkComponents.create(transform.getPipeline().getOptions());
 RunnerApi.PTransform parDoPTransform =
   PTransformTranslation.toProto(transform, Collections.emptyList(), components);
 return ParDoPayload.parseFrom(parDoPTransform.getSpec().getPayload());
}

代码示例来源：origin: org.apache.beam/beam-runners-core-construction-java

public static PCollection.IsBounded sourceIsBounded(AppliedPTransform<?, ?, ?> transform) {
 try {
  SdkComponents components = SdkComponents.create(transform.getPipeline().getOptions());
  return PCollectionTranslation.fromProto(
    ReadPayload.parseFrom(
        PTransformTranslation.toProto(transform, Collections.emptyList(), components)
          .getSpec()
          .getPayload())
      .getIsBounded());
 } catch (IOException e) {
  throw new RuntimeException("Internal error determining boundedness of Read", e);
 }
}

代码示例来源：origin: org.apache.beam/beam-runners-core-construction-java

private static <T, DestinationT> WriteFilesPayload getWriteFilesPayload(
  AppliedPTransform<
      PCollection<T>, WriteFilesResult<DestinationT>,
      ? extends PTransform<PCollection<T>, WriteFilesResult<DestinationT>>>
    transform)
  throws IOException {
 SdkComponents components = SdkComponents.create(transform.getPipeline().getOptions());
 return WriteFilesPayload.parseFrom(
   PTransformTranslation.toProto(transform, Collections.emptyList(), components)
     .getSpec()
     .getPayload());
}

代码示例来源：origin: org.apache.beam/beam-runners-core-construction-java

public static WindowIntoPayload getWindowIntoPayload(AppliedPTransform<?, ?, ?> application) {
 RunnerApi.PTransform transformProto;
 try {
  SdkComponents components = SdkComponents.create(application.getPipeline().getOptions());
  transformProto =
    PTransformTranslation.toProto(application, Collections.emptyList(), components);
 } catch (IOException exc) {
  throw new RuntimeException(exc);
 }
 checkArgument(
   PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN.equals(
     transformProto.getSpec().getUrn()),
   "Illegal attempt to extract %s from transform %s with name \"%s\" and URN \"%s\"",
   Window.Assign.class.getSimpleName(),
   application.getTransform(),
   application.getFullName(),
   transformProto.getSpec().getUrn());
 try {
  return WindowIntoPayload.parseFrom(transformProto.getSpec().getPayload());
 } catch (InvalidProtocolBufferException exc) {
  throw new IllegalStateException(
    String.format(
      "%s translated %s with URN '%s' but payload was not a %s",
      PTransformTranslation.class.getSimpleName(),
      application,
      PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN,
      WindowIntoPayload.class.getSimpleName()),
    exc);
 }
}

代码示例来源：origin: org.apache.beam/beam-runners-core-construction-java

public static <UserT, DestinationT> List<PCollectionView<?>> getDynamicDestinationSideInputs(
  AppliedPTransform<
      PCollection<UserT>, WriteFilesResult<DestinationT>,
      ? extends PTransform<PCollection<UserT>, WriteFilesResult<DestinationT>>>
    transform)
  throws IOException {
 SdkComponents sdkComponents = SdkComponents.create(transform.getPipeline().getOptions());
 RunnerApi.PTransform transformProto = PTransformTranslation.toProto(transform, sdkComponents);
 List<PCollectionView<?>> views = Lists.newArrayList();
 Map<String, SideInput> sideInputs = getWriteFilesPayload(transform).getSideInputsMap();
 for (Map.Entry<String, SideInput> entry : sideInputs.entrySet()) {
  PCollection<?> originalPCollection =
    checkNotNull(
      (PCollection<?>) transform.getInputs().get(new TupleTag<>(entry.getKey())),
      "no input with tag %s",
      entry.getKey());
  views.add(
    PCollectionViewTranslation.viewFromProto(
      entry.getValue(),
      entry.getKey(),
      originalPCollection,
      transformProto,
      RehydratedComponents.forComponents(sdkComponents.toComponents())));
 }
 return views;
}

代码示例来源：origin: org.apache.beam/beam-runners-flink_2.11

private JobGraph getJobGraph(Pipeline pipeline) {
 FlinkRunner flinkRunner = FlinkRunner.fromOptions(pipeline.getOptions());
 return flinkRunner.getJobGraph(pipeline);
}

代码示例来源：origin: org.apache.beam/beam-runners-core-construction-java

public static List<PCollectionView<?>> getSideInputs(AppliedPTransform<?, ?, ?> application)
  throws IOException {
 PTransform<?, ?> transform = application.getTransform();
 if (transform instanceof ParDo.MultiOutput) {
  return ((ParDo.MultiOutput<?, ?>) transform).getSideInputs();
 }
 SdkComponents sdkComponents = SdkComponents.create(application.getPipeline().getOptions());
 RunnerApi.PTransform parDoProto = PTransformTranslation.toProto(application, sdkComponents);
 ParDoPayload payload = ParDoPayload.parseFrom(parDoProto.getSpec().getPayload());
 List<PCollectionView<?>> views = new ArrayList<>();
 RehydratedComponents components =
   RehydratedComponents.forComponents(sdkComponents.toComponents());
 for (Map.Entry<String, SideInput> sideInputEntry : payload.getSideInputsMap().entrySet()) {
  String sideInputTag = sideInputEntry.getKey();
  RunnerApi.SideInput sideInput = sideInputEntry.getValue();
  PCollection<?> originalPCollection =
    checkNotNull(
      (PCollection<?>) application.getInputs().get(new TupleTag<>(sideInputTag)),
      "no input with tag %s",
      sideInputTag);
  views.add(
    PCollectionViewTranslation.viewFromProto(
      sideInput, sideInputTag, originalPCollection, parDoProto, components));
 }
 return views;
}

代码示例来源：origin: org.apache.beam/beam-runners-core-construction-java

/** Extracts an {@link ExecutableStagePayload} from the given transform. */
 public static ExecutableStagePayload getExecutableStagePayload(
   AppliedPTransform<?, ?, ?> appliedTransform) throws IOException {
  RunnerApi.PTransform transform =
    PTransformTranslation.toProto(
      appliedTransform, SdkComponents.create(appliedTransform.getPipeline().getOptions()));
  checkArgument(ExecutableStage.URN.equals(transform.getSpec().getUrn()));
  return ExecutableStagePayload.parseFrom(transform.getSpec().getPayload());
 }
}

代码示例来源：origin: org.apache.beam/beam-runners-flink_2.11

private void restoreFromSavepointPortable(Pipeline pipeline, String savepointDir)
  throws Exception {
 FlinkPipelineOptions flinkOptions = pipeline.getOptions().as(FlinkPipelineOptions.class);
 flinkOptions.setSavepointPath(savepointDir);
 executePortable(pipeline);
}

代码示例来源：origin: org.apache.beam/beam-runners-core-construction-java

public static TupleTagList getAdditionalOutputTags(AppliedPTransform<?, ?, ?> application)
  throws IOException {
 PTransform<?, ?> transform = application.getTransform();
 if (transform instanceof ParDo.MultiOutput) {
  return ((ParDo.MultiOutput<?, ?>) transform).getAdditionalOutputTags();
 }
 RunnerApi.PTransform protoTransform =
   PTransformTranslation.toProto(
     application, SdkComponents.create(application.getPipeline().getOptions()));
 ParDoPayload payload = ParDoPayload.parseFrom(protoTransform.getSpec().getPayload());
 TupleTag<?> mainOutputTag = getMainOutputTag(payload);
 Set<String> outputTags =
   Sets.difference(
     protoTransform.getOutputsMap().keySet(), Collections.singleton(mainOutputTag.getId()));
 ArrayList<TupleTag<?>> additionalOutputTags = new ArrayList<>();
 for (String outputTag : outputTags) {
  additionalOutputTags.add(new TupleTag<>(outputTag));
 }
 return TupleTagList.of(additionalOutputTags);
}

代码示例来源：origin: org.apache.beam/beam-runners-core-construction-java

private static Coder<?> getAccumulatorCoder(AppliedPTransform<?, ?, ?> transform)
  throws IOException {
 SdkComponents sdkComponents = SdkComponents.create(transform.getPipeline().getOptions());
 String id =
   getCombinePayload(transform, sdkComponents)
     .map(CombinePayload::getAccumulatorCoderId)
     .orElseThrow(() -> new IOException("Transform does not contain an AccumulatorCoder"));
 Components components = sdkComponents.toComponents();
 return CoderTranslation.fromProto(
   components.getCodersOrThrow(id), RehydratedComponents.forComponents(components));
}

代码示例来源：origin: org.apache.beam/beam-sdks-java-io-amazon-web-services

@Override
 public PDone expand(PCollection<SendMessageRequest> input) {
  input.apply(
    ParDo.of(
      new SqsWriteFn(
        new SqsConfiguration(input.getPipeline().getOptions().as(AwsOptions.class)))));
  return PDone.in(input.getPipeline());
 }
}

代码示例来源：origin: org.apache.beam/beam-runners-core-construction-java

/**
 * Converts an {@link AppliedPTransform}, which may be a rehydrated transform or an original
 * {@link TestStream}, to a {@link TestStream}.
 */
public static <T> TestStream<T> getTestStream(
  AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>> application)
  throws IOException {
 // For robustness, we don't take this shortcut:
 // if (application.getTransform() instanceof TestStream) {
 //   return application.getTransform()
 // }
 SdkComponents sdkComponents = SdkComponents.create(application.getPipeline().getOptions());
 RunnerApi.PTransform transformProto = PTransformTranslation.toProto(application, sdkComponents);
 checkArgument(
   TEST_STREAM_TRANSFORM_URN.equals(transformProto.getSpec().getUrn()),
   "Attempt to get %s from a transform with wrong URN %s",
   TestStream.class.getSimpleName(),
   transformProto.getSpec().getUrn());
 RunnerApi.TestStreamPayload testStreamPayload =
   RunnerApi.TestStreamPayload.parseFrom(transformProto.getSpec().getPayload());
 return (TestStream<T>)
   testStreamFromProtoPayload(
     testStreamPayload, RehydratedComponents.forComponents(sdkComponents.toComponents()));
}

代码示例来源：origin: org.apache.beam/beam-runners-flink_2.11

private JobID executePortable(Pipeline pipeline) throws Exception {
 pipeline
   .getOptions()
   .as(PortablePipelineOptions.class)
   .setDefaultEnvironmentType(Environments.ENVIRONMENT_EMBEDDED);
 pipeline
   .getOptions()
   .as(FlinkPipelineOptions.class)
   .setFlinkMaster(
      executorService,
      pipelineProto,
      pipeline.getOptions().as(FlinkPipelineOptions.class),
      null,
      Collections.emptyList());

代码示例来源：origin: org.apache.beam/beam-sdks-java-io-amazon-web-services

@Override
 public PCollection<Message> expand(PBegin input) {
  org.apache.beam.sdk.io.Read.Unbounded<Message> unbounded =
    org.apache.beam.sdk.io.Read.from(
      new SqsUnboundedSource(
        this,
        new SqsConfiguration(input.getPipeline().getOptions().as(AwsOptions.class))));
  PTransform<PBegin, PCollection<Message>> transform = unbounded;
  if (maxNumRecords() < Long.MAX_VALUE || maxReadTime() != null) {
   transform = unbounded.withMaxReadTime(maxReadTime()).withMaxNumRecords(maxNumRecords());
  }
  return input.getPipeline().apply(transform);
 }
}

代码示例来源：origin: org.apache.beam/beam-runners-core-construction-java

@Test
public void toAndFromProto() throws IOException {
 SdkComponents components = SdkComponents.create(spec.getTransform().getPipeline().getOptions());
 RunnerApi.PTransform converted = convert(spec, components);
 Components protoComponents = components.toComponents();
 // Sanity checks
 assertThat(converted.getInputsCount(), equalTo(spec.getTransform().getInputs().size()));
 assertThat(converted.getOutputsCount(), equalTo(spec.getTransform().getOutputs().size()));
 assertThat(converted.getSubtransformsCount(), equalTo(spec.getChildren().size()));
 assertThat(converted.getUniqueName(), equalTo(spec.getTransform().getFullName()));
 for (PValue inputValue : spec.getTransform().getInputs().values()) {
  PCollection<?> inputPc = (PCollection<?>) inputValue;
  protoComponents.getPcollectionsOrThrow(components.registerPCollection(inputPc));
 }
 for (PValue outputValue : spec.getTransform().getOutputs().values()) {
  PCollection<?> outputPc = (PCollection<?>) outputValue;
  protoComponents.getPcollectionsOrThrow(components.registerPCollection(outputPc));
 }
}

内容来源于网络，如有侵权，请联系作者删除！

Pipeline

相关文章

    查看更多

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备2022004421号-2 NULL123 https://www.null123.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com