本文整理了Java中org.apache.hadoop.hive.ql.metadata.Hive.getQualifiedPathWithoutSchemeAndAuthority()
方法的一些代码示例,展示了Hive.getQualifiedPathWithoutSchemeAndAuthority()
的具体用法。这些代码示例主要来源于Github
/Stackoverflow
/Maven
等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Hive.getQualifiedPathWithoutSchemeAndAuthority()
方法的具体详情如下:
包路径:org.apache.hadoop.hive.ql.metadata.Hive
类名称:Hive
方法名:getQualifiedPathWithoutSchemeAndAuthority
暂无
代码示例来源:origin: apache/hive
private static boolean isSubDir(Path srcf, Path destf, FileSystem srcFs, FileSystem destFs, boolean isSrcLocal) {
if (srcf == null) {
LOG.debug("The source path is null for isSubDir method.");
return false;
}
String fullF1 = getQualifiedPathWithoutSchemeAndAuthority(srcf, srcFs).toString() + Path.SEPARATOR;
String fullF2 = getQualifiedPathWithoutSchemeAndAuthority(destf, destFs).toString() + Path.SEPARATOR;
boolean isInTest = HiveConf.getBoolVar(srcFs.getConf(), ConfVars.HIVE_IN_TEST);
// In the automation, the data warehouse is the local file system based.
LOG.debug("The source path is " + fullF1 + " and the destination path is " + fullF2);
if (isInTest) {
return fullF1.startsWith(fullF2);
}
// schema is diff, return false
String schemaSrcf = srcf.toUri().getScheme();
String schemaDestf = destf.toUri().getScheme();
// if the schemaDestf is null, it means the destination is not in the local file system
if (schemaDestf == null && isSrcLocal) {
LOG.debug("The source file is in the local while the dest not.");
return false;
}
// If both schema information are provided, they should be the same.
if (schemaSrcf != null && schemaDestf != null && !schemaSrcf.equals(schemaDestf)) {
LOG.debug("The source path's schema is " + schemaSrcf +
" and the destination path's schema is " + schemaDestf + ".");
return false;
}
LOG.debug("The source path is " + fullF1 + " and the destination path is " + fullF2);
return fullF1.startsWith(fullF2);
}
代码示例来源:origin: apache/hive
public static void clearDestForSubDirSrc(final HiveConf conf, Path dest,
Path src, boolean isSrcLocal) throws IOException {
FileSystem destFS = dest.getFileSystem(conf);
FileSystem srcFS = src.getFileSystem(conf);
if (isSubDir(src, dest, srcFS, destFS, isSrcLocal)) {
final Path fullSrcPath = getQualifiedPathWithoutSchemeAndAuthority(src, srcFS);
final Path fullDestPath = getQualifiedPathWithoutSchemeAndAuthority(dest, destFS);
if (fullSrcPath.equals(fullDestPath)) {
return;
}
Path parent = fullSrcPath;
while (!parent.getParent().equals(fullDestPath)) {
parent = parent.getParent();
}
FileStatus[] existingFiles = destFS.listStatus(
dest, FileUtils.HIDDEN_FILES_PATH_FILTER);
for (FileStatus fileStatus : existingFiles) {
if (!fileStatus.getPath().getName().equals(parent.getName())) {
destFS.delete(fileStatus.getPath(), true);
}
}
}
}
代码示例来源:origin: apache/drill
private static boolean isSubDir(Path srcf, Path destf, FileSystem srcFs, FileSystem destFs, boolean isSrcLocal) {
if (srcf == null) {
LOG.debug("The source path is null for isSubDir method.");
return false;
}
String fullF1 = getQualifiedPathWithoutSchemeAndAuthority(srcf, srcFs).toString() + Path.SEPARATOR;
String fullF2 = getQualifiedPathWithoutSchemeAndAuthority(destf, destFs).toString() + Path.SEPARATOR;
boolean isInTest = HiveConf.getBoolVar(srcFs.getConf(), ConfVars.HIVE_IN_TEST);
// In the automation, the data warehouse is the local file system based.
LOG.debug("The source path is " + fullF1 + " and the destination path is " + fullF2);
if (isInTest) {
return fullF1.startsWith(fullF2);
}
// schema is diff, return false
String schemaSrcf = srcf.toUri().getScheme();
String schemaDestf = destf.toUri().getScheme();
// if the schemaDestf is null, it means the destination is not in the local file system
if (schemaDestf == null && isSrcLocal) {
LOG.debug("The source file is in the local while the dest not.");
return false;
}
// If both schema information are provided, they should be the same.
if (schemaSrcf != null && schemaDestf != null && !schemaSrcf.equals(schemaDestf)) {
LOG.debug("The source path's schema is " + schemaSrcf +
" and the destination path's schema is " + schemaDestf + ".");
return false;
}
LOG.debug("The source path is " + fullF1 + " and the destination path is " + fullF2);
return fullF1.startsWith(fullF2);
}
代码示例来源:origin: apache/drill
public static void clearDestForSubDirSrc(final HiveConf conf, Path dest,
Path src, boolean isSrcLocal) throws IOException {
FileSystem destFS = dest.getFileSystem(conf);
FileSystem srcFS = src.getFileSystem(conf);
if (isSubDir(src, dest, srcFS, destFS, isSrcLocal)) {
final Path fullSrcPath = getQualifiedPathWithoutSchemeAndAuthority(src, srcFS);
final Path fullDestPath = getQualifiedPathWithoutSchemeAndAuthority(dest, destFS);
if (fullSrcPath.equals(fullDestPath)) {
return;
}
Path parent = fullSrcPath;
while (!parent.getParent().equals(fullDestPath)) {
parent = parent.getParent();
}
FileStatus[] existingFiles = destFS.listStatus(
dest, FileUtils.HIDDEN_FILES_PATH_FILTER);
for (FileStatus fileStatus : existingFiles) {
if (!fileStatus.getPath().getName().equals(parent.getName())) {
destFS.delete(fileStatus.getPath(), true);
}
}
}
}
代码示例来源:origin: com.facebook.presto.hive/hive-apache
private static boolean isSubDir(Path srcf, Path destf, FileSystem fs, boolean isSrcLocal){
if (srcf == null) {
LOG.debug("The source path is null for isSubDir method.");
return false;
}
String fullF1 = getQualifiedPathWithoutSchemeAndAuthority(srcf, fs);
String fullF2 = getQualifiedPathWithoutSchemeAndAuthority(destf, fs);
boolean isInTest = Boolean.valueOf(HiveConf.getBoolVar(fs.getConf(), ConfVars.HIVE_IN_TEST));
// In the automation, the data warehouse is the local file system based.
LOG.debug("The source path is " + fullF1 + " and the destination path is " + fullF2);
if (isInTest) {
return fullF1.startsWith(fullF2);
}
// schema is diff, return false
String schemaSrcf = srcf.toUri().getScheme();
String schemaDestf = destf.toUri().getScheme();
// if the schemaDestf is null, it means the destination is not in the local file system
if (schemaDestf == null && isSrcLocal) {
LOG.debug("The source file is in the local while the dest not.");
return false;
}
// If both schema information are provided, they should be the same.
if (schemaSrcf != null && schemaDestf != null && !schemaSrcf.equals(schemaDestf)) {
LOG.debug("The source path's schema is " + schemaSrcf +
" and the destination path's schema is " + schemaDestf + ".");
return false;
}
LOG.debug("The source path is " + fullF1 + " and the destination path is " + fullF2);
return fullF1.startsWith(fullF2);
}
内容来源于网络,如有侵权,请联系作者删除!