Java Code Examples for org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

The following are Jave code examples for showing how to use copyFromLocalFile() of the org.apache.hadoop.fs.FileSystem class. You can vote up the examples you like. Your votes will be used in our system to get more good examples.
Example 1
Project: circus-train   File: AvroSerDeTransformation.java   Source Code and License Vote up 6 votes
private boolean copy(String source, String destination) throws IOException {
  if (source == null || destination == null) {
    return false;
  }

  if (temporaryDirectory == null) {
    temporaryDirectory = Files.createTempDirectory("avro-schema-download-folder");
    temporaryDirectory.toFile().deleteOnExit();
  }

  FileSystem sourceFileSystem = new Path(source).getFileSystem(sourceHiveConf);
  String tempPath = temporaryDirectory.toString();
  sourceFileSystem.copyToLocalFile(false, new Path(source), new Path(tempPath));

  FileSystem destinationFileSystem = new Path(destination).getFileSystem(replicaHiveConf);
  destinationFileSystem.copyFromLocalFile(true, new Path(tempPath), new Path(destination));
  LOG.info("Avro schema has been copied from '{}' to '{}'", source, destination);

  return destinationFileSystem.exists(new Path(destination));
}
 
Example 2
Project: Hadoop-Codes   File: putting.java   Source Code and License Vote up 6 votes
public static void main(String[] args) throws IOException{
	String uri="hdfs://localhost:9000";
	Configuration conf = new Configuration();
	FileSystem fs = FileSystem.get(URI.create(uri), conf);
	fs.copyFromLocalFile(new Path(uri+"/haha.txt"), new Path("/home/aadish"));

}
 
Example 3
Project: TensorFlowOnYARN   File: Utils.java   Source Code and License Vote up 6 votes
public static Path copyLocalFileToDfs(FileSystem fs, String appId,
    Path srcPath, String dstFileName) throws IOException {
  Path dstPath = new Path(fs.getHomeDirectory(),
      Constants.DEFAULT_APP_NAME + Path.SEPARATOR + appId + Path.SEPARATOR + dstFileName);
  LOG.info("Copying " + srcPath + " to " + dstPath);
  fs.copyFromLocalFile(srcPath, dstPath);
  return dstPath;
}
 
Example 4
Project: hadoop-oss   File: Client.java   Source Code and License Vote up 5 votes
private void addToLocalResources(FileSystem fs, String fileSrcPath,
																 String fileDstPath, String appId, Map<String, LocalResource> localResources,
																 String resources) throws IOException {
	String suffix =
			"prkeyrotation" + "/" + appId + "/" + fileDstPath;
	Path dst =
			new Path(fs.getHomeDirectory(), suffix);
	if (fileSrcPath == null) {
		FSDataOutputStream ostream = null;
		try {
			ostream = FileSystem
					.create(fs, dst, new FsPermission((short) 0710));
			ostream.writeUTF(resources);
		} finally {
			IOUtils.closeQuietly(ostream);
		}
	} else {
		fs.copyFromLocalFile(new Path(fileSrcPath), dst);
	}
	FileStatus scFileStatus = fs.getFileStatus(dst);
	LocalResource scRsrc =
			LocalResource.newInstance(
					ConverterUtils.getYarnUrlFromPath(dst),
					LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
					scFileStatus.getLen(), scFileStatus.getModificationTime());
	localResources.put(fileDstPath, scRsrc);
}
 
Example 5
Project: circus-train   File: SchemaCopier.java   Source Code and License Vote up 5 votes
private void copyToRemote(Path localLocation, Path remoteDestinationLocation) {
  FileSystem destinationFileSystem;
  try {
    destinationFileSystem = remoteDestinationLocation.getFileSystem(replicaHiveConf);
    destinationFileSystem.copyFromLocalFile(localLocation, remoteDestinationLocation);
  } catch (IOException e) {
    throw new CircusTrainException("Couldn't copy file from " + localLocation + " to " + remoteDestinationLocation,
        e);
  }
}
 
Example 6
Project: hadoop   File: Client.java   Source Code and License Vote up 5 votes
private void addToLocalResources(FileSystem fs, String fileSrcPath,
    String fileDstPath, String appId, Map<String, LocalResource> localResources,
    String resources) throws IOException {
  String suffix =
      appName + "/" + appId + "/" + fileDstPath;
  Path dst =
      new Path(fs.getHomeDirectory(), suffix);
  if (fileSrcPath == null) {
    FSDataOutputStream ostream = null;
    try {
      ostream = FileSystem
          .create(fs, dst, new FsPermission((short) 0710));
      ostream.writeUTF(resources);
    } finally {
      IOUtils.closeQuietly(ostream);
    }
  } else {
    fs.copyFromLocalFile(new Path(fileSrcPath), dst);
  }
  FileStatus scFileStatus = fs.getFileStatus(dst);
  LocalResource scRsrc =
      LocalResource.newInstance(
          ConverterUtils.getYarnUrlFromURI(dst.toUri()),
          LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
          scFileStatus.getLen(), scFileStatus.getModificationTime());
  localResources.put(fileDstPath, scRsrc);
}
 
Example 7
Project: ditb   File: TestImportExport.java   Source Code and License Vote up 5 votes
/**
 * Test import data from 0.94 exported file
 * @throws Exception
 */
@Test
public void testImport94Table() throws Exception {
  final String name = "exportedTableIn94Format";
  URL url = TestImportExport.class.getResource(name);
  File f = new File(url.toURI());
  if (!f.exists()) {
    LOG.warn("FAILED TO FIND " + f + "; skipping out on test");
    return;
  }
  assertTrue(f.exists());
  LOG.info("FILE=" + f);
  Path importPath = new Path(f.toURI());
  FileSystem fs = FileSystem.get(UTIL.getConfiguration());
  fs.copyFromLocalFile(importPath, new Path(FQ_OUTPUT_DIR + Path.SEPARATOR + name));
  String IMPORT_TABLE = name;
  try (Table t = UTIL.createTable(TableName.valueOf(IMPORT_TABLE), Bytes.toBytes("f1"), 3);) {
    String[] args = new String[] {
        "-Dhbase.import.version=0.94" ,
        IMPORT_TABLE, FQ_OUTPUT_DIR
    };
    assertTrue(runImport(args));
    /* exportedTableIn94Format contains 5 rows
    ROW         COLUMN+CELL
    r1          column=f1:c1, timestamp=1383766761171, value=val1
    r2          column=f1:c1, timestamp=1383766771642, value=val2
    r3          column=f1:c1, timestamp=1383766777615, value=val3
    r4          column=f1:c1, timestamp=1383766785146, value=val4
    r5          column=f1:c1, timestamp=1383766791506, value=val5
    */
   assertEquals(5, UTIL.countRows(t));
  }
}
 
Example 8
Project: ditb   File: TestClassLoading.java   Source Code and License Vote up 4 votes
void loadingClassFromLibDirInJar(String libPrefix) throws Exception {
  FileSystem fs = cluster.getFileSystem();

  File innerJarFile1 = buildCoprocessorJar(cpName1);
  File innerJarFile2 = buildCoprocessorJar(cpName2);
  File outerJarFile = new File(TEST_UTIL.getDataTestDir().toString(), "outer.jar");

  ClassLoaderTestHelper.addJarFilesToJar(
    outerJarFile, libPrefix, innerJarFile1, innerJarFile2);

  // copy the jars into dfs
  fs.copyFromLocalFile(new Path(outerJarFile.getPath()),
    new Path(fs.getUri().toString() + Path.SEPARATOR));
  String jarFileOnHDFS = fs.getUri().toString() + Path.SEPARATOR +
    outerJarFile.getName();
  assertTrue("Copy jar file to HDFS failed.",
    fs.exists(new Path(jarFileOnHDFS)));
  LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS);

  // create a table that references the coprocessors
  HTableDescriptor htd = new HTableDescriptor(tableName);
  htd.addFamily(new HColumnDescriptor("test"));
    // without configuration values
  htd.setValue("COPROCESSOR$1", jarFileOnHDFS.toString() + "|" + cpName1 +
    "|" + Coprocessor.PRIORITY_USER);
    // with configuration values
  htd.setValue("COPROCESSOR$2", jarFileOnHDFS.toString() + "|" + cpName2 +
    "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
  Admin admin = TEST_UTIL.getHBaseAdmin();
  if (admin.tableExists(tableName)) {
    if (admin.isTableEnabled(tableName)) {
      admin.disableTable(tableName);
    }
    admin.deleteTable(tableName);
  }
  admin.createTable(htd);
  waitForTable(htd.getTableName());

  // verify that the coprocessors were loaded
  boolean found1 = false, found2 = false, found2_k1 = false,
      found2_k2 = false, found2_k3 = false;
  MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
  for (Region region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
    if (region.getRegionInfo().getRegionNameAsString().startsWith(tableName.getNameAsString())) {
      CoprocessorEnvironment env;
      env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
      if (env != null) {
        found1 = true;
      }
      env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
      if (env != null) {
        found2 = true;
        Configuration conf = env.getConfiguration();
        found2_k1 = conf.get("k1") != null;
        found2_k2 = conf.get("k2") != null;
        found2_k3 = conf.get("k3") != null;
      }
    }
  }
  assertTrue("Class " + cpName1 + " was missing on a region", found1);
  assertTrue("Class " + cpName2 + " was missing on a region", found2);
  assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
  assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
  assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
}