org.apache.gobblin.configuration.State.addAll()方法的使用及代码示例

x33g5p2x  于2022-01-30 转载在 其他  
字(10.3k)|赞(0)|评价(0)|浏览(108)

本文整理了Java中org.apache.gobblin.configuration.State.addAll()方法的一些代码示例,展示了State.addAll()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。State.addAll()方法的具体详情如下:
包路径:org.apache.gobblin.configuration.State
类名称:State
方法名:addAll

State.addAll介绍

[英]Populates this instance with values of a Properties instance.
[中]用属性实例的值填充此实例。

代码示例

代码示例来源:origin: apache/incubator-gobblin

public void setJobProps(State jobProps) {
 this.jobProps.addAll(jobProps);
}

代码示例来源:origin: apache/incubator-gobblin

/**
 * Constructor.
 *
 * @param properties job configuration properties
 */
public SourceState(State properties) {
 super.addAll(properties);
 this.previousWorkUnitStates = new ArrayList<>();
 this.previousDatasetStatesByUrns = ImmutableMap.of();
}

代码示例来源:origin: apache/incubator-gobblin

/**
 * Copy constructor.
 *
 * @param other the other {@link WorkUnit} instance
 *
 * @deprecated Use {@link #copyOf(WorkUnit)}
 */
@Deprecated
public WorkUnit(WorkUnit other) {
 super.addAll(other);
 this.extract = other.getExtract();
}

代码示例来源:origin: apache/incubator-gobblin

/**
 * Populates this instance with properties of the other instance.
 *
 * @param otherState the other {@link State} instance
 */
public void addAll(State otherState) {
 Properties diffCommonProps = new Properties();
 diffCommonProps.putAll(Maps.difference(this.commonProperties, otherState.commonProperties).entriesOnlyOnRight());
 addAll(diffCommonProps);
 addAll(otherState.specProperties);
}

代码示例来源:origin: apache/incubator-gobblin

/**
 * Deep copy constructor.
 *
 * @param extract the other {@link Extract} instance
 */
public Extract(Extract extract) {
 super.addAll(extract.getProperties());
}

代码示例来源:origin: apache/incubator-gobblin

Builder withState(State state) {
 this.state = new State();
 this.state.addAll(state);
 return this;
}

代码示例来源:origin: apache/incubator-gobblin

/**
 * Constructor.
 *
 * @param properties job configuration properties
 * @param previousWorkUnitStates an {@link Iterable} of {@link WorkUnitState}s of the previous job run
 */
public SourceState(State properties, Iterable<WorkUnitState> previousWorkUnitStates) {
 super.addAll(properties);
 this.previousDatasetStatesByUrns = ImmutableMap.of();
 for (WorkUnitState workUnitState : previousWorkUnitStates) {
  this.previousWorkUnitStates.add(new ImmutableWorkUnitState(workUnitState));
 }
}

代码示例来源:origin: apache/incubator-gobblin

HiveRegistrationUnit(Builder<?> builder) {
 Preconditions.checkArgument(!Strings.isNullOrEmpty(builder.dbName));
 Preconditions.checkArgument(!Strings.isNullOrEmpty(builder.tableName));
 this.dbName = builder.dbName;
 this.tableName = builder.tableName;
 this.columns.addAll(builder.columns);
 this.props.addAll(builder.props);
 this.storageProps.addAll(builder.storageProps);
 this.serDeProps.addAll(builder.serDeProps);
 this.serDeManager = builder.serDeManager;
 populateTablePartitionFields(this.props);
 populateStorageFields(this.storageProps);
 populateSerDeFields(this.serDeProps);
}

代码示例来源:origin: apache/incubator-gobblin

@Override
public State getFinalState() {
 State state = new State();
 if (this.writer instanceof FinalState) {
  state.addAll(((FinalState)this.writer).getFinalState());
 } else {
  LOG.warn("Wrapped writer does not implement FinalState: " + this.writer.getClass());
 }
 return state;
}

代码示例来源:origin: apache/incubator-gobblin

/**
 * Constructor.
 *
 * @param state a {@link SourceState} the properties of which will be copied into this {@link WorkUnit} instance
 * @param extract an {@link Extract}
 *
 * @deprecated Properties in {@link SourceState} should not be added to a {@link WorkUnit}. Having each
 * {@link WorkUnit} contain a copy of {@link SourceState} is a waste of memory. Use {@link #create(Extract)}.
 */
@Deprecated
public WorkUnit(SourceState state, Extract extract) {
 // Values should only be null for deserialization
 if (state != null) {
  super.addAll(state);
 }
 if (extract != null) {
  this.extract = extract;
 } else {
  this.extract = new Extract(null, null, null, null);
 }
}

代码示例来源:origin: apache/incubator-gobblin

/**
 * Constructor.
 *
 * @param properties job configuration properties
 * @param previousDatasetStatesByUrns {@link SourceState} of the previous job run
 * @param previousWorkUnitStates an {@link Iterable} of {@link WorkUnitState}s of the previous job run
 */
public SourceState(State properties, Map<String, ? extends SourceState> previousDatasetStatesByUrns,
  Iterable<WorkUnitState> previousWorkUnitStates) {
 super.addAll(properties.getProperties());
 this.previousDatasetStatesByUrns = ImmutableMap.copyOf(previousDatasetStatesByUrns);
 for (WorkUnitState workUnitState : previousWorkUnitStates) {
  this.previousWorkUnitStates.add(new ImmutableWorkUnitState(workUnitState));
 }
}

代码示例来源:origin: apache/incubator-gobblin

@Override
public State getFinalState() {
 State state = new State();
 if (this.writer instanceof FinalState) {
  state.addAll(((FinalState)this.writer).getFinalState());
 } else {
  LOG.warn("Wrapped writer does not implement FinalState: " + this.writer.getClass());
 }
 state.setProp(THROTTLED_TIME_KEY, this.throttledTime);
 return state;
}

代码示例来源:origin: apache/incubator-gobblin

@Override
public State getFinalState() {
 State state = new State();
 if (this.writer instanceof FinalState) {
  state.addAll(((FinalState)this.writer).getFinalState());
 } else {
  LOG.warn("Wrapped writer does not implement FinalState: " + this.writer.getClass());
 }
 state.setProp(FAILED_WRITES_KEY, this.failedWrites);
 return state;
}

代码示例来源:origin: apache/incubator-gobblin

/**
 * Get final state for this object, obtained by merging the final states of the
 * {@link org.apache.gobblin.qualitychecker.row.RowLevelPolicy}s used by this object.
 * @return Merged {@link org.apache.gobblin.configuration.State} of final states for
 *                {@link org.apache.gobblin.qualitychecker.row.RowLevelPolicy} used by this checker.
 */
@Override
public State getFinalState() {
 State state = new State();
 for (RowLevelPolicy policy : this.list) {
  state.addAll(policy.getFinalState());
 }
 return state;
}

代码示例来源:origin: apache/incubator-gobblin

@Override
public State getFinalState() {
 State state = new State();
 try {
  for (Map.Entry<GenericRecord, DataWriter<D>> entry : this.partitionWriters.asMap().entrySet()) {
   if (entry.getValue() instanceof FinalState) {
    State partitionFinalState = ((FinalState) entry.getValue()).getFinalState();
    if (this.shouldPartition) {
     for (String key : partitionFinalState.getPropertyNames()) {
      // Prevent overwriting final state across writers
      partitionFinalState.setProp(key + "_" + AvroUtils.serializeAsPath(entry.getKey(), false, true),
        partitionFinalState.getProp(key));
     }
    }
    state.addAll(partitionFinalState);
   }
  }
  state.setProp("RecordsWritten", recordsWritten());
  state.setProp("BytesWritten", bytesWritten());
 } catch (Exception exception) {
  log.warn("Failed to get final state." + exception.getMessage());
  // If Writer fails to return bytesWritten, it might not be implemented, or implemented incorrectly.
  // Omit property instead of failing.
 }
 return state;
}

代码示例来源:origin: apache/incubator-gobblin

super.addAll(state);
super.setProp(ConfigurationKeys.EXTRACT_TABLE_TYPE_KEY, type.toString());
super.setProp(ConfigurationKeys.EXTRACT_NAMESPACE_NAME_KEY, namespace);
 Extract previousExtract = pre.getWorkunit().getExtract();
 if (previousExtract.getNamespace().equals(namespace) && previousExtract.getTable().equals(table)) {
  this.previousTableState.addAll(pre);

代码示例来源:origin: apache/incubator-gobblin

jobProps.addAll(this.state);
jobProps.setProp(MRCompactor.COMPACTION_ENABLE_SUCCESS_FILE, false);
jobProps.setProp(MRCompactor.COMPACTION_INPUT_DEDUPLICATED, this.inputDeduplicated);

代码示例来源:origin: apache/incubator-gobblin

public MRCompactor(Properties props, List<? extends Tag<?>> tags, Optional<CompactorListener> compactorListener)
  throws IOException {
 this.state = new State();
 this.state.addAll(props);
 this.initilizeTime = getCurrentTime();
 this.tags = tags;
 this.conf = HadoopUtils.getConfFromState(this.state);
 this.tmpOutputDir = getTmpOutputDir();
 this.fs = getFileSystem();
 this.datasets = getDatasetsFinder().findDistinctDatasets();
 this.jobExecutor = createJobExecutor();
 this.jobRunnables = Maps.newConcurrentMap();
 this.closer = Closer.create();
 this.stopwatch = Stopwatch.createStarted();
 this.gobblinMetrics = initializeMetrics();
 this.eventSubmitter = new EventSubmitter.Builder(
   GobblinMetrics.get(this.state.getProp(ConfigurationKeys.JOB_NAME_KEY)).getMetricContext(),
   MRCompactor.COMPACTION_TRACKING_EVENTS_NAMESPACE).build();
 this.compactorListener = compactorListener;
 this.dataVerifTimeoutMinutes = getDataVerifTimeoutMinutes();
 this.compactionTimeoutMinutes = getCompactionTimeoutMinutes();
 this.shouldVerifDataCompl = shouldVerifyDataCompleteness();
 this.compactionCompleteListener = getCompactionCompleteListener();
 this.verifier =
   this.shouldVerifDataCompl ? Optional.of(this.closer.register(new DataCompletenessVerifier(this.state)))
     : Optional.<DataCompletenessVerifier> absent();
 this.shouldPublishDataIfCannotVerifyCompl = shouldPublishDataIfCannotVerifyCompl();
}

代码示例来源:origin: apache/incubator-gobblin

jobPropsState.addAll(jobProps);

代码示例来源:origin: apache/incubator-gobblin

private PurgeableHivePartitionDataset createPurgeableHivePartitionDataset(State state)
  throws IOException {
 HivePartitionDataset hivePartitionDataset =
   HivePartitionFinder.findDataset(state.getProp(ComplianceConfigurationKeys.PARTITION_NAME), state);
 Preconditions.checkArgument(state.contains(ComplianceConfigurationKeys.COMPLIANCEID_KEY),
   "Missing property " + ComplianceConfigurationKeys.COMPLIANCEID_KEY);
 Preconditions.checkArgument(state.contains(ComplianceConfigurationKeys.COMPLIANCE_ID_TABLE_KEY),
   "Missing property " + ComplianceConfigurationKeys.COMPLIANCE_ID_TABLE_KEY);
 Preconditions.checkArgument(state.contains(ComplianceConfigurationKeys.TIMESTAMP),
   "Missing table property " + ComplianceConfigurationKeys.TIMESTAMP);
 Boolean simulate = state.getPropAsBoolean(ComplianceConfigurationKeys.COMPLIANCE_JOB_SIMULATE,
   ComplianceConfigurationKeys.DEFAULT_COMPLIANCE_JOB_SIMULATE);
 String complianceIdentifier = state.getProp(ComplianceConfigurationKeys.COMPLIANCEID_KEY);
 String complianceIdTable = state.getProp(ComplianceConfigurationKeys.COMPLIANCE_ID_TABLE_KEY);
 String timeStamp = state.getProp(ComplianceConfigurationKeys.TIMESTAMP);
 Boolean specifyPartitionFormat = state.getPropAsBoolean(ComplianceConfigurationKeys.SPECIFY_PARTITION_FORMAT,
   ComplianceConfigurationKeys.DEFAULT_SPECIFY_PARTITION_FORMAT);
 State datasetState = new State();
 datasetState.addAll(state.getProperties());
 PurgeableHivePartitionDataset dataset = new PurgeableHivePartitionDataset(hivePartitionDataset);
 dataset.setComplianceId(complianceIdentifier);
 dataset.setComplianceIdTable(complianceIdTable);
 dataset.setComplianceField(getComplianceField(state, hivePartitionDataset));
 dataset.setTimeStamp(timeStamp);
 dataset.setState(datasetState);
 dataset.setSimulate(simulate);
 dataset.setSpecifyPartitionFormat(specifyPartitionFormat);
 return dataset;
}

相关文章