org.apache.hadoop.hive.common.FileUtils.copy()方法的使用及代码示例

x33g5p2x  于2022-01-19 转载在 其他  
字(6.0k)|赞(0)|评价(0)|浏览(155)

本文整理了Java中org.apache.hadoop.hive.common.FileUtils.copy()方法的一些代码示例,展示了FileUtils.copy()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。FileUtils.copy()方法的具体详情如下:
包路径:org.apache.hadoop.hive.common.FileUtils
类名称:FileUtils
方法名:copy

FileUtils.copy介绍

[英]Copies files between filesystems.
[中]在文件系统之间复制文件。

代码示例

代码示例来源:origin: apache/drill

destFs.copyFromLocalFile(sourcePath, destFilePath);
} else {
 FileUtils.copy(sourceFs, sourcePath, destFs, destFilePath,
   true,   // delete source
   false,  // overwrite destination

代码示例来源:origin: apache/hive

destFs.copyFromLocalFile(sourcePath, destFilePath);
} else {
 if (!FileUtils.copy(sourceFs, sourcePath, destFs, destFilePath,
   false,   // delete source
   false,  // overwrite destination

代码示例来源:origin: apache/hive

console.printInfo("Copying file: " + oneSrcPathStr);
Utilities.FILE_OP_LOGGER.debug("Copying file {} to {}", oneSrcPathStr, toPath);
if (!FileUtils.copy(srcFs, oneSrc.getPath(), dstFs, toPath,
  false, // delete source
  true, // overwrite destination

代码示例来源:origin: apache/hive

return FileUtils.copy(srcf.getFileSystem(conf), srcf, destf.getFileSystem(conf), destf,
  true,    // delete source

代码示例来源:origin: apache/drill

console.printInfo("Copying file: " + oneSrc.getPath().toString());
LOG.debug("Copying file: " + oneSrc.getPath().toString());
if (!FileUtils.copy(srcFs, oneSrc.getPath(), dstFs, toPath,
  false, // delete source
  true, // overwrite destination

代码示例来源:origin: apache/hive

@Test
public void testCopyWithDistcp() throws IOException {
 Path copySrc = new Path("copySrc");
 Path copyDst = new Path("copyDst");
 HiveConf conf = new HiveConf(TestFileUtils.class);
 FileSystem mockFs = mock(FileSystem.class);
 when(mockFs.getUri()).thenReturn(URI.create("hdfs:///"));
 ContentSummary mockContentSummary = mock(ContentSummary.class);
 when(mockContentSummary.getFileCount()).thenReturn(Long.MAX_VALUE);
 when(mockContentSummary.getLength()).thenReturn(Long.MAX_VALUE);
 when(mockFs.getContentSummary(any(Path.class))).thenReturn(mockContentSummary);
 HadoopShims shims = mock(HadoopShims.class);
 when(shims.runDistCp(Collections.singletonList(copySrc), copyDst, conf)).thenReturn(true);
 Assert.assertTrue(FileUtils.copy(mockFs, copySrc, mockFs, copyDst, false, false, conf, shims));
 verify(shims).runDistCp(Collections.singletonList(copySrc), copyDst, conf);
}

代码示例来源:origin: apache/hive

@VisibleForTesting
static boolean copy(FileSystem srcFS, Path src,
 FileSystem dstFS, Path dst,
 boolean deleteSource,
 boolean overwrite,
 HiveConf conf, HadoopShims shims) throws IOException {
 boolean copied = false;
 boolean triedDistcp = false;
 /* Run distcp if source file/dir is too big */
 if (srcFS.getUri().getScheme().equals("hdfs")) {
  ContentSummary srcContentSummary = srcFS.getContentSummary(src);
  if (srcContentSummary.getFileCount() > conf.getLongVar(HiveConf.ConfVars.HIVE_EXEC_COPYFILE_MAXNUMFILES)
      && srcContentSummary.getLength() > conf.getLongVar(HiveConf.ConfVars.HIVE_EXEC_COPYFILE_MAXSIZE)) {
   LOG.info("Source is " + srcContentSummary.getLength() + " bytes. (MAX: " + conf.getLongVar(
       HiveConf.ConfVars.HIVE_EXEC_COPYFILE_MAXSIZE) + ")");
   LOG.info("Source is " + srcContentSummary.getFileCount() + " files. (MAX: " + conf.getLongVar(
       HiveConf.ConfVars.HIVE_EXEC_COPYFILE_MAXNUMFILES) + ")");
   LOG.info("Launch distributed copy (distcp) job.");
   triedDistcp = true;
   copied = distCp(srcFS, Collections.singletonList(src), dst, deleteSource, null, conf, shims);
  }
 }
 if (!triedDistcp) {
  // Note : Currently, this implementation does not "fall back" to regular copy if distcp
  // is tried and it fails. We depend upon that behaviour in cases like replication,
  // wherein if distcp fails, there is good reason to not plod along with a trivial
  // implementation, and fail instead.

代码示例来源:origin: apache/drill

if (!FileUtils.copy(actualSrcFs, oneSrc.getPath(), dstFs, toPath,
 false, // delete source
 true, // overwrite destination

代码示例来源:origin: apache/drill

return FileUtils.copy(srcf.getFileSystem(conf), srcf, destf.getFileSystem(conf), destf,
  true,    // delete source

代码示例来源:origin: apache/hive

ReplChangeManager.FileInfo sourceInfo = ReplChangeManager
  .getFileInfo(new Path(result[0]), result[1], result[2], result[3], conf);
if (FileUtils.copy(
  sourceInfo.getSrcFs(), sourceInfo.getSourcePath(),
  dstFs, toPath, false, false, conf)) {

代码示例来源:origin: com.facebook.presto.hive/hive-apache

console.printInfo("Copying file: " + oneSrc.getPath().toString());
LOG.debug("Copying file: " + oneSrc.getPath().toString());
if (!FileUtils.copy(srcFs, oneSrc.getPath(), dstFs, toPath,
  false, // delete source
  true, // overwrite destination

代码示例来源:origin: org.apache.hive/hive-common

@VisibleForTesting
static boolean copy(FileSystem srcFS, Path src,
 FileSystem dstFS, Path dst,
 boolean deleteSource,
 boolean overwrite,
 HiveConf conf, HadoopShims shims) throws IOException {
 boolean copied = false;
 boolean triedDistcp = false;
 /* Run distcp if source file/dir is too big */
 if (srcFS.getUri().getScheme().equals("hdfs")) {
  ContentSummary srcContentSummary = srcFS.getContentSummary(src);
  if (srcContentSummary.getFileCount() > conf.getLongVar(HiveConf.ConfVars.HIVE_EXEC_COPYFILE_MAXNUMFILES)
      && srcContentSummary.getLength() > conf.getLongVar(HiveConf.ConfVars.HIVE_EXEC_COPYFILE_MAXSIZE)) {
   LOG.info("Source is " + srcContentSummary.getLength() + " bytes. (MAX: " + conf.getLongVar(
       HiveConf.ConfVars.HIVE_EXEC_COPYFILE_MAXSIZE) + ")");
   LOG.info("Source is " + srcContentSummary.getFileCount() + " files. (MAX: " + conf.getLongVar(
       HiveConf.ConfVars.HIVE_EXEC_COPYFILE_MAXNUMFILES) + ")");
   LOG.info("Launch distributed copy (distcp) job.");
   triedDistcp = true;
   copied = distCp(srcFS, Collections.singletonList(src), dst, deleteSource, null, conf, shims);
  }
 }
 if (!triedDistcp) {
  // Note : Currently, this implementation does not "fall back" to regular copy if distcp
  // is tried and it fails. We depend upon that behaviour in cases like replication,
  // wherein if distcp fails, there is good reason to not plod along with a trivial
  // implementation, and fail instead.

代码示例来源:origin: com.facebook.presto.hive/hive-apache

success = FileUtils.copy(srcf.getFileSystem(conf), srcf, destf.getFileSystem(conf), destf,
  true,    // delete source
  success = FileUtils.copy(srcf.getFileSystem(conf), status.getPath(), destf.getFileSystem(conf), destf,
    true,     // delete source

相关文章