本文整理了Java中org.apache.gobblin.configuration.State.getProp()
方法的一些代码示例,展示了State.getProp()
的具体用法。这些代码示例主要来源于Github
/Stackoverflow
/Maven
等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。State.getProp()
方法的具体详情如下:
包路径:org.apache.gobblin.configuration.State
类名称:State
方法名:getProp
[英]Get the value of a property.
[中]获取属性的值。
代码示例来源:origin: apache/incubator-gobblin
@Override
public String datasetURN() {
return state.getProp(SERIALIZE_COMPACTION_FILE_PATH_NAME);
}
};
代码示例来源:origin: apache/incubator-gobblin
public CachedInstanceKey(State state) {
this.numOfEncryptionKeys = state.getPropAsInt(ConfigurationKeys.NUMBER_OF_ENCRYPT_KEYS, ConfigurationKeys.DEFAULT_NUMBER_OF_MASTER_PASSWORDS);
this.useStrongEncryptor = shouldUseStrongEncryptor(state);
this.fsURI = state.getProp(ConfigurationKeys.ENCRYPT_KEY_LOC);
this.masterPasswordFile = state.getProp(ConfigurationKeys.ENCRYPT_KEY_LOC);
}
代码示例来源:origin: apache/incubator-gobblin
public HiveBaseExtractor(WorkUnitState state) throws IOException {
if (Boolean.valueOf(state.getPropAsBoolean(PartitionLevelWatermarker.IS_WATERMARK_WORKUNIT_KEY))) {
return;
}
this.hiveWorkUnit = new HiveWorkUnit(state.getWorkunit());
this.hiveDataset = hiveWorkUnit.getHiveDataset();
this.dbName = hiveDataset.getDbAndTable().getDb();
this.tableName = hiveDataset.getDbAndTable().getTable();
this.pool = HiveMetastoreClientPool.get(state.getJobState().getProperties(),
Optional.fromNullable(state.getJobState().getProp(HiveDatasetFinder.HIVE_METASTORE_URI_KEY)));
}
代码示例来源:origin: apache/incubator-gobblin
public AvroSchemaManager(FileSystem fs, State state) {
this.fs = fs;
this.schemaPaths = Maps.newHashMap();
this.schemaDir = new Path(state.getProp(HIVE_SCHEMA_TEMP_DIR_PATH_KEY, DEFAULT_HIVE_SCHEMA_TEMP_DIR_PATH_KEY),
state.getProp(ConfigurationKeys.JOB_ID_KEY));
}
代码示例来源:origin: apache/incubator-gobblin
/**
* Check whether metrics collection and reporting are enabled or not.
*
* @param state a {@link State} object containing configuration properties
* @return whether metrics collection and reporting are enabled
*/
public static boolean isEnabled(State state) {
return Boolean
.valueOf(state.getProp(ConfigurationKeys.METRICS_ENABLED_KEY, ConfigurationKeys.DEFAULT_METRICS_ENABLED));
}
代码示例来源:origin: apache/incubator-gobblin
/**
* Get the value of a property as an short, using the given default value if the property is not set.
*
* @param key property key
* @param def default value
* @return short value associated with the key or the default value if the property is not set
*/
public short getPropAsShort(String key, short def) {
return Short.parseShort(getProp(key, String.valueOf(def)));
}
代码示例来源:origin: apache/incubator-gobblin
/**
* Get the value of a property as a double, using the given default value if the property is not set.
*
* @param key property key
* @param def default value
* @return double value associated with the key or the default value if the property is not set
*/
public double getPropAsDouble(String key, double def) {
return Double.parseDouble(getProp(key, String.valueOf(def)));
}
代码示例来源:origin: apache/incubator-gobblin
/**
* @return the {@link Split} object contained in the {@link WorkUnit}.
*/
public static Optional<Split> getSplit(State workUnit) {
return workUnit.contains(SPLIT_KEY) ? Optional.of(GSON.fromJson(workUnit.getProp(SPLIT_KEY), Split.class))
: Optional.<Split>absent();
}
代码示例来源:origin: apache/incubator-gobblin
private String getDstBaseDir(State state) {
Preconditions.checkArgument(state.contains(MRCompactor.COMPACTION_DEST_DIR),
"Missing required property " + MRCompactor.COMPACTION_DEST_DIR);
return state.getProp(MRCompactor.COMPACTION_DEST_DIR);
}
代码示例来源:origin: apache/incubator-gobblin
private String getDstSubDir(State state) {
Preconditions.checkArgument(state.contains(MRCompactor.COMPACTION_DEST_SUBDIR),
"Missing required property " + MRCompactor.COMPACTION_DEST_SUBDIR);
return state.getProp(MRCompactor.COMPACTION_DEST_SUBDIR);
}
}
代码示例来源:origin: apache/incubator-gobblin
/**
* Read execution count from a specific directory.
* File name is {@link InputRecordCountHelper#STATE_FILE}
* @param dir directory where a state file is located
* @return record count
*/
public long readExecutionCount (Path dir) throws IOException {
State state = loadState(fs, dir);
return Long.parseLong(state.getProp(CompactionSlaEventHelper.EXEC_COUNT_TOTAL, "0"));
}
代码示例来源:origin: apache/incubator-gobblin
/**
* Get the full lineage event name from a state
*/
public static String getFullEventName(State state) {
return Joiner.on('.').join(LineageEventBuilder.LIENAGE_EVENT_NAMESPACE, state.getProp(getKey(NAME_KEY)));
}
代码示例来源:origin: apache/incubator-gobblin
private FsRenameCommitStep(Builder<? extends Builder<?>> builder) throws IOException {
super(builder);
this.srcPath = builder.srcPath;
this.dstPath = builder.dstPath;
this.srcFs = builder.srcFs != null ? builder.srcFs
: getFileSystem(this.props.getProp(ConfigurationKeys.FS_URI_KEY, ConfigurationKeys.LOCAL_FS_URI));
this.srcFsUri = this.srcFs.getUri().toString();
this.dstFs = builder.dstFs != null ? builder.dstFs
: getFileSystem(this.props.getProp(ConfigurationKeys.FS_URI_KEY, ConfigurationKeys.LOCAL_FS_URI));
this.dstFsUri = this.dstFs.getUri().toString();
this.overwrite = builder.overwrite;
}
代码示例来源:origin: apache/incubator-gobblin
private HiveConnection getHiveConnection(Optional<String> proxyUser)
throws ClassNotFoundException, SQLException {
Class.forName("org.apache.hive.jdbc.HiveDriver");
Preconditions.checkArgument(this.state.contains(ComplianceConfigurationKeys.HIVE_JDBC_URL), "Missing required property " + ComplianceConfigurationKeys.HIVE_JDBC_URL);
String url = this.state.getProp(ComplianceConfigurationKeys.HIVE_JDBC_URL);
if (proxyUser.isPresent()) {
url = url + ComplianceConfigurationKeys.HIVE_SERVER2_PROXY_USER + proxyUser.get();
}
return (HiveConnection) DriverManager.getConnection(url);
}
代码示例来源:origin: apache/incubator-gobblin
private String getBackUpTableLocation(HivePartitionVersion version) {
Preconditions.checkArgument(this.state.contains(ComplianceConfigurationKeys.TRASH_DIR),
"Missing required property " + ComplianceConfigurationKeys.TRASH_DIR);
return StringUtils
.join(Arrays.asList(this.state.getProp(ComplianceConfigurationKeys.TRASH_DIR), getVersionTableName(version)),
'/');
}
代码示例来源:origin: apache/incubator-gobblin
private static DatePartitionType getGranularity(State state, int numBranches, int branchId) {
String propName = ForkOperatorUtils.getPropertyNameForBranch(WRITER_PARTITION_GRANULARITY, numBranches, branchId);
String granularityValue = state.getProp(propName, DEFAULT_WRITER_PARTITION_GRANULARITY.toString());
Optional<DatePartitionType> granularity =
Enums.getIfPresent(DatePartitionType.class, granularityValue.toUpperCase());
Preconditions.checkState(granularity.isPresent(),
granularityValue + " is not a valid writer partition granularity");
return granularity.get();
}
代码示例来源:origin: apache/incubator-gobblin
private CombineOperation getConditionOperation (Dataset dataset) {
String oprName = dataset.jobProps().getProp (MRCompactor.COMPACTION_RECOMPACT_COMBINE_CONDITIONS_OPERATION,
MRCompactor.DEFAULT_COMPACTION_RECOMPACT_COMBINE_CONDITIONS_OPERATION);
try {
CombineOperation opr = CombineOperation.valueOf (oprName.toUpperCase());
return opr;
} catch (Exception e) {
return CombineOperation.OR;
}
}
代码示例来源:origin: apache/incubator-gobblin
private static List<HiveDataset> getHiveDatasets(FileSystem fs, State state)
throws IOException {
Preconditions.checkArgument(state.contains(ComplianceConfigurationKeys.COMPLIANCE_DATASET_WHITELIST),
"Missing required property " + ComplianceConfigurationKeys.COMPLIANCE_DATASET_WHITELIST);
Properties prop = new Properties();
prop.setProperty(ComplianceConfigurationKeys.HIVE_DATASET_WHITELIST,
state.getProp(ComplianceConfigurationKeys.COMPLIANCE_DATASET_WHITELIST));
HiveDatasetFinder finder = new HiveDatasetFinder(fs, prop);
return finder.findDatasets();
}
代码示例来源:origin: apache/incubator-gobblin
@VisibleForTesting
public Connection createConnection() throws SQLException {
DataSource dataSource = DataSourceBuilder.builder().url(this.state.getProp(JdbcPublisher.JDBC_PUBLISHER_URL))
.driver(this.state.getProp(JdbcPublisher.JDBC_PUBLISHER_DRIVER))
.userName(this.state.getProp(JdbcPublisher.JDBC_PUBLISHER_USERNAME))
.passWord(this.state.getProp(JdbcPublisher.JDBC_PUBLISHER_PASSWORD))
.cryptoKeyLocation(this.state.getProp(JdbcPublisher.JDBC_PUBLISHER_ENCRYPTION_KEY_LOC)).maxActiveConnections(1)
.maxIdleConnections(1).state(this.state).build();
return dataSource.getConnection();
}
代码示例来源:origin: apache/incubator-gobblin
private DateTime getEarliestAllowedFolderTime(DateTime currentTime, PeriodFormatter periodFormatter) {
String maxTimeAgoStr =
this.state.getProp(COMPACTION_TIMEBASED_MAX_TIME_AGO, DEFAULT_COMPACTION_TIMEBASED_MAX_TIME_AGO);
Period maxTimeAgo = periodFormatter.parsePeriod(maxTimeAgoStr);
return currentTime.minus(maxTimeAgo);
}
内容来源于网络,如有侵权,请联系作者删除!