org.apache.hadoop.tools.DistCp.<init>()方法的使用及代码示例

x33g5p2x  于2022-01-18 转载在 其他  
字(10.4k)|赞(0)|评价(0)|浏览(156)

本文整理了Java中org.apache.hadoop.tools.DistCp.<init>()方法的一些代码示例，展示了DistCp.<init>()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。DistCp.<init>()方法的具体详情如下：
包路径：org.apache.hadoop.tools.DistCp
类名称：DistCp
方法名：<init>

DistCp.<init>介绍

[英]To be used with the ToolRunner. Not for public consumption.
[中]与ToolRunner一起使用。不供公众消费。

代码示例

代码示例来源：origin: apache/hive

public static boolean runDistCp(List<Path> srcPaths, Path dst, Configuration conf)
  throws IOException {
 DistCpOptions options = new DistCpOptions.Builder(srcPaths, dst)
   .withSyncFolder(true)
   .withCRC(true)
   .preserve(FileAttribute.BLOCKSIZE)
   .build();
 // Creates the command-line parameters for distcp
 List<String> params = constructDistCpParams(srcPaths, dst, conf);
 try {
  conf.setBoolean("mapred.mapper.new-api", true);
  DistCp distcp = new DistCp(conf, options);
  // HIVE-13704 states that we should use run() instead of execute() due to a hadoop known issue
  // added by HADOOP-10459
  if (distcp.run(params.toArray(new String[params.size()])) == 0) {
   return true;
  } else {
   return false;
  }
 } catch (Exception e) {
  throw new IOException("Cannot execute DistCp process: " + e, e);
 } finally {
  conf.setBoolean("mapred.mapper.new-api", false);
 }
}

代码示例来源：origin: apache/hive

@Override
public boolean runDistCp(List<Path> srcPaths, Path dst, Configuration conf) throws IOException {
   DistCpOptions options = new DistCpOptions.Builder(srcPaths, dst)
   .withSyncFolder(true)
   .withCRC(true)
   .preserve(FileAttribute.BLOCKSIZE)
   .build();
 // Creates the command-line parameters for distcp
 List<String> params = constructDistCpParams(srcPaths, dst, conf);
 try {
  conf.setBoolean("mapred.mapper.new-api", true);
  DistCp distcp = new DistCp(conf, options);
  // HIVE-13704 states that we should use run() instead of execute() due to a hadoop known issue
  // added by HADOOP-10459
  if (distcp.run(params.toArray(new String[0])) == 0) {
   return true;
  } else {
   return false;
  }
 } catch (Exception e) {
  throw new IOException("Cannot execute DistCp process: " + e, e);
 } finally {
  conf.setBoolean("mapred.mapper.new-api", false);
 }
}

代码示例来源：origin: org.apache.hadoop/hadoop-mapred-test

public DistCp run() {
  return new DistCp(userConf);
 }
});

代码示例来源：origin: com.hotels/circus-train-distcp-copier

@Override
 public Job exec(Configuration conf, DistCpOptions options) throws Exception {
  return new DistCp(conf, options).execute();
 }
};

代码示例来源：origin: HotelsDotCom/circus-train

@Override
 public Job exec(Configuration conf, DistCpOptions options) throws Exception {
  return new DistCp(conf, options).execute();
 }
};

代码示例来源：origin: org.apache.hadoop/hadoop-mapred-test

/** copy files from local file system to local file system */
public void testCopyFromLocalToLocal() throws Exception {
 Configuration conf = new Configuration();
 FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
 MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR+"/srcdat");
 ToolRunner.run(new DistCp(new Configuration()),
             new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
                    "file:///"+TEST_ROOT_DIR+"/destdat"});
 assertTrue("Source and destination directories do not match.",
       checkFiles(localfs, TEST_ROOT_DIR+"/destdat", files));
 deldir(localfs, TEST_ROOT_DIR+"/destdat");
 deldir(localfs, TEST_ROOT_DIR+"/srcdat");
}

代码示例来源：origin: org.apache.hadoop/hadoop-distcp

private static void assertRunDistCp(int exitCode, String src, String dst,
   String[] options, Configuration conf)
   throws Exception {
  DistCp distCp = new DistCp(conf, null);
  String[] optsArr = new String[options.length + 2];
  System.arraycopy(options, 0, optsArr, 0, options.length);
  optsArr[optsArr.length - 2] = src;
  optsArr[optsArr.length - 1] = dst;

  assertEquals(exitCode,
    ToolRunner.run(conf, distCp, optsArr));
 }
}

代码示例来源：origin: io.hops/hadoop-distcp

private static void assertRunDistCp(int exitCode, String src, String dst,
   String[] options, Configuration conf)
   throws Exception {
  DistCp distCp = new DistCp(conf, null);
  String[] optsArr = new String[options.length + 2];
  System.arraycopy(options, 0, optsArr, 0, options.length);
  optsArr[optsArr.length - 2] = src;
  optsArr[optsArr.length - 1] = dst;

  assertEquals(exitCode,
    ToolRunner.run(conf, distCp, optsArr));
 }
}

代码示例来源：origin: org.springframework.data/spring-data-hadoop-core

private static void invokeCopy(Configuration config, String[] parsedArgs) {
  try {
    log.info("Running DistCp with arguments [" + StringUtils.arrayToCommaDelimitedString(parsedArgs) + "]");
    DistCpOptions inputOptions = OptionsParser.parse(parsedArgs);
    org.apache.hadoop.tools.DistCp distCp = new org.apache.hadoop.tools.DistCp(config, inputOptions);
    distCp.execute();
  } catch (Exception e) {
    throw new HadoopException("Error running DistCp job", e);
  }
}

代码示例来源：origin: org.apache.hadoop/hadoop-distcp

/**
 * Runs distcp from /src to specified destination, preserving ACLs.  Asserts
 * expected exit code.
 *
 * @param int exitCode expected exit code
 * @param dst String distcp destination
 * @throws Exception if there is any error
 */
private static void assertRunDistCp(int exitCode, String dst)
  throws Exception {
 DistCp distCp = new DistCp(conf, null);
 assertEquals(exitCode, ToolRunner.run(
  conf, distCp, new String[] { "-pa", "/src", dst }));
}

代码示例来源：origin: org.apache.hadoop/hadoop-distcp

@Test
 public void testSourceRoot() throws Exception {
  FileSystem fs = cluster.getFileSystem();

  String rootStr = fs.makeQualified(new Path("/")).toString();

  String testRoot = "/testdir." + getMethodName();

  // Case 1. The target does not exist.

  Path tgtPath = new Path(testRoot + "/nodir");
  String tgtStr = fs.makeQualified(tgtPath).toString();
  String[] args = new String[]{rootStr, tgtStr};
  Assert.assertThat(ToolRunner.run(conf, new DistCp(), args), is(0));

  // Case 2. The target exists.

  Path tgtPath2 = new Path(testRoot + "/dir");
  assertTrue(fs.mkdirs(tgtPath2));
  String tgtStr2 = fs.makeQualified(tgtPath2).toString();
  String[] args2 = new String[]{rootStr, tgtStr2};
  Assert.assertThat(ToolRunner.run(conf, new DistCp(), args2), is(0));
 }
}

代码示例来源：origin: org.apache.hadoop/hadoop-distcp

private void copyAndVerify(final DistributedFileSystem fs,
  final FileEntry[] srcFiles, final FileStatus[] srcStats,
  final String testDst,
  final String[] args) throws Exception {
 final String testRoot = "/testdir";
 FsShell shell = new FsShell(fs.getConf());
 LOG.info("ls before distcp");
 LOG.info(execCmd(shell, "-lsr", testRoot));
 LOG.info("_____ running distcp: " + args[0] + " " + args[1]);
 ToolRunner.run(conf, new DistCp(), args);
 LOG.info("ls after distcp");
 LOG.info(execCmd(shell, "-lsr", testRoot));
 FileStatus[] dstStat = getFileStatus(fs, testDst, srcFiles);
 for (int i=0; i< dstStat.length; i++) {
  compareFiles(fs, srcStats[i], dstStat[i]);
 }
}

代码示例来源：origin: io.hops/hadoop-distcp

private void runTest(Path listFile, Path target, boolean targetExists, 
  boolean sync) throws IOException {
 DistCpOptions options = new DistCpOptions(listFile, target);
 options.setSyncFolder(sync);
 options.setTargetPathExists(targetExists);
 try {
  new DistCp(getConf(), options).execute();
 } catch (Exception e) {
  LOG.error("Exception encountered ", e);
  throw new IOException(e);
 }
}

代码示例来源：origin: org.apache.hadoop/hadoop-distcp

/**
 * test methods run end execute of DistCp class. silple copy file
 * @throws Exception 
 */
 @Test
 public void testCleanup() throws Exception {

   Configuration conf = getConf();

   Path stagingDir = JobSubmissionFiles.getStagingDir(new Cluster(conf),
     conf);
   stagingDir.getFileSystem(conf).mkdirs(stagingDir);
   Path soure = createFile("tmp.txt");
   Path target = createFile("target.txt");

   DistCp distcp = new DistCp(conf, null);
   String[] arg = { soure.toString(), target.toString() };

   distcp.run(arg);
   Assert.assertTrue(fs.exists(target));

 
 }

代码示例来源：origin: io.hops/hadoop-distcp

/**
 * test methods run end execute of DistCp class. silple copy file
 * @throws Exception 
 */
 @Test
 public void testCleanup() throws Exception {

   Configuration conf = getConf();

   Path stagingDir = JobSubmissionFiles.getStagingDir(new Cluster(conf),
     conf);
   stagingDir.getFileSystem(conf).mkdirs(stagingDir);
   Path soure = createFile("tmp.txt");
   Path target = createFile("target.txt");

   DistCp distcp = new DistCp(conf, null);
   String[] arg = { soure.toString(), target.toString() };

   distcp.run(arg);
   Assert.assertTrue(fs.exists(target));

 
 }

代码示例来源：origin: org.apache.hadoop/hadoop-distcp

private void runTest(Path listFile, Path target, boolean targetExists, 
  boolean sync) throws IOException {
 final DistCpOptions options = new DistCpOptions.Builder(listFile, target)
   .withSyncFolder(sync)
   .build();
 try {
  final DistCp distcp = new DistCp(getConf(), options);
  distcp.context.setTargetPathExists(targetExists);
  distcp.execute();
 } catch (Exception e) {
  LOG.error("Exception encountered ", e);
  throw new IOException(e);
 }
}

代码示例来源：origin: io.hops/hadoop-distcp

private void runTest(Path listFile, Path target, boolean targetExists, 
  boolean sync, boolean delete,
  boolean overwrite) throws IOException {
 DistCpOptions options = new DistCpOptions(listFile, target);
 options.setSyncFolder(sync);
 options.setDeleteMissing(delete);
 options.setOverwrite(overwrite);
 options.setTargetPathExists(targetExists);
 options.setNumListstatusThreads(numListstatusThreads);
 try {
  new DistCp(getConf(), options).execute();
 } catch (Exception e) {
  LOG.error("Exception encountered ", e);
  throw new IOException(e);
 }
}

代码示例来源：origin: io.hops/hadoop-distcp

/**
 * Executes DistCp and asserts that the job finished successfully.
 *
 * @param src source path
 * @param dst destination path
 * @throws Exception if there is a failure
 */
private void runDistCp(Path src, Path dst) throws Exception {
 DistCpOptions options = new DistCpOptions(Arrays.asList(src), dst);
 Job job = new DistCp(conf, options).execute();
 assertNotNull("Unexpected null job returned from DistCp execution.", job);
 assertTrue("DistCp job did not complete.", job.isComplete());
 assertTrue("DistCp job did not complete successfully.", job.isSuccessful());
}

代码示例来源：origin: org.apache.hadoop/hadoop-distcp

private void runTest(Path listFile, Path target, boolean targetExists, 
  boolean sync, boolean delete,
  boolean overwrite) throws IOException {
 final DistCpOptions options = new DistCpOptions.Builder(listFile, target)
   .withSyncFolder(sync)
   .withDeleteMissing(delete)
   .withOverwrite(overwrite)
   .withNumListstatusThreads(numListstatusThreads)
   .build();
 try {
  final DistCp distCp = new DistCp(getConf(), options);
  distCp.context.setTargetPathExists(targetExists);
  distCp.execute();
 } catch (Exception e) {
  LOG.error("Exception encountered ", e);
  throw new IOException(e);
 }
}

代码示例来源：origin: org.apache.hadoop/hadoop-distcp

/**
 * Run the distcp job.
 * @param optons distcp options
 * @return the job. It will have already completed.
 * @throws Exception failure
 */
private Job runDistCp(final DistCpOptions options) throws Exception {
 Job job = new DistCp(conf, options).execute();
 assertNotNull("Unexpected null job returned from DistCp execution.", job);
 assertTrue("DistCp job did not complete.", job.isComplete());
 assertTrue("DistCp job did not complete successfully.", job.isSuccessful());
 return job;
}

内容来源于网络，如有侵权，请联系作者删除！

DistCp

相关文章

    查看更多

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备2022004421号-2 NULL123 https://www.null123.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com