Java Code Examples for org.apache.hadoop.fs.FileSystem#copyToLocalFile()

public String copyToLocalFile(String remoteFilename) {
  try {
    loadProperties();
    Configuration conf = getConfiguration();
    login(conf);

    Path remotePath = new Path(remoteFilename);
    Path localPath = new Path(Paths.get(getAndCreateLocalDir(), new File(remoteFilename).getName()).toString());
    FileSystem fs = remotePath.getFileSystem(conf);
    // It is important to pass the custom configuration object to FileSystemUtils since we load some extra
    // properties from etc/**.xml in getConfiguration() for Presto
    String resolvedRemoteFilename = FileSystemUtils.resolveLatest(remoteFilename, conf);
    Path resolvedRemotePath = new Path(resolvedRemoteFilename);
    fs.copyToLocalFile(resolvedRemotePath, localPath);
    return localPath.toString();
  } catch (Exception e) {
    throw new RuntimeException("Error downloading HDFS file: " + remoteFilename, e);
  }
}
 

public void generateBaselinePyramid(final Configuration conf, final String testName,
    final String ex) throws IOException, JobFailedException, JobCancelledException,
    ParserException
{

  runMapAlgebraExpression(conf, testName, ex);

  final Path src = new Path(outputHdfs, testName);
  final MrsPyramid pyramid = MrsPyramid.open(src.toString(), (ProviderProperties) null);
  if (pyramid != null)
  {
    final Path dst = new Path(inputLocal, testName);
    final FileSystem fs = dst.getFileSystem(conf);
    fs.copyToLocalFile(src, dst);
  }
}
 

private static Path downloadResource(String destName, URI uri, Configuration conf, String destDir)
    throws IOException {
  FileSystem fs = FileSystem.get(uri, conf);
  Path cwd = new Path(destDir);
  Path dFile = new Path(cwd, destName);
  Path srcPath = new Path(uri);
  fs.copyToLocalFile(srcPath, dFile);
  return dFile.makeQualified(FileSystem.getLocal(conf).getUri(), cwd);
}
 

public StramAppLauncher(FileSystem fs, Path path, Configuration conf) throws Exception
{
  File jarsDir = new File(StramClientUtils.getUserDTDirectory(), "jars");
  jarsDir.mkdirs();
  File localJarFile = new File(jarsDir, path.getName());
  this.fs = fs;
  fs.copyToLocalFile(path, new Path(localJarFile.getAbsolutePath()));
  this.jarFile = localJarFile;
  this.conf = conf;
  this.propertiesBuilder = new LogicalPlanConfiguration(conf);
  init(this.jarFile.getName());
}
 

/**
 * 下载文件
 *
 * @param sourcePath 原文件路径（HDFS 中的路径）
 * @param targetPath 目标路径
 * @throws Exception
 */
public void downloadFile(@NotBlank String sourcePath, @NotBlank String targetPath) throws Exception {
    FileSystem fileSystem = null;
    try {
        fileSystem = this.hdfsPool.borrowObject();
        // 调用文件系统的文件复制方法，第一个参数为是否删除原文件（true为删除），默认为 false
        fileSystem.copyToLocalFile(false, new Path(sourcePath), new Path(targetPath));
    } finally {
        if (fileSystem != null) { this.hdfsPool.returnObject(fileSystem); }
    }
}
 

public void setup(Context context) throws IOException, InterruptedException {
    FileSystem fs = FileSystem.get(context.getConfiguration());
    Path ipFile = new Path("/user/hadoop/lib/17monipdb.dat");
    Path localPath = new Path(this.getClass().getResource("/").getPath());
    fs.copyToLocalFile(ipFile, localPath);
    IPUtil.load("17monipdb.dat");
}
 

@Override
public String retrieve(Config config, String targetDir) throws IOException {
  URI uri = URI.create(this.fsUri);
  FileSystem fs = FileSystem.get(uri, HadoopUtils.getConfFromState(ConfigUtils.configToState(config)));

  final Path sourceFile = new Path(path);
  final String zipFile = appendSlash(targetDir) +
          StringUtils.substringAfterLast(this.path, File.separator);
  LOGGER.debug("Downloading to zip: " + zipFile + " from uri: " + sourceFile);
  fs.copyToLocalFile(sourceFile, new Path(zipFile));
  return zipFile;
}
 

public void generateBaselineVector(final Configuration conf, final String testName,
    final String ex)
    throws IOException, ParserException, JobFailedException, JobCancelledException
{
  runMapAlgebraExpression(conf, testName, ex);

  final Path src = new Path(outputHdfs, testName);
  final FileSystem srcfs = src.getFileSystem(conf);
  if (srcfs.exists(src))
  {
    final Path dst = new Path(inputLocal, testName);
    final FileSystem fs = dst.getFileSystem(conf);
    fs.copyToLocalFile(src, dst);
  }
}
 

/**
 * @return returns 0 if successfull, -1 if filesize is incorrect and -2 if an exception occurred
 */
protected static int privateDownloadFileFromHDFS(TaskInputOutputContext context, FileSystem fs, String from, String to) {
    try {
        // check if file is present on local scratch
        File f = new File(to);
        if(!f.exists()) {
            Logger.DEBUG("attempting download of \"" + to + "\"");
            fs.copyToLocalFile(new Path(from), new Path(to));
            context.getCounter(HalvadeCounters.FIN_FROM_HDFS).increment(fs.getFileStatus(new Path(from)).getLen());
        } else {
            // check if filesize is correct
            if(fs.getFileStatus(new Path(from)).getLen() != f.length()) {
                // incorrect filesize, remove and download again
                Logger.DEBUG("incorrect filesize: " + f.length() + " =/= " + 
                        fs.getFileStatus(new Path(from)).getLen());
                f.delete();
                fs.copyToLocalFile(new Path(from), new Path(to));
                context.getCounter(HalvadeCounters.FIN_FROM_HDFS).increment(fs.getFileStatus(new Path(from)).getLen());
        
            } else {
                Logger.DEBUG("file \"" + to + "\" exists");
            }
        }
        if(fs.getFileStatus(new Path(from)).getLen() != f.length())
            return -1;
        else
            return 0;
    } catch (IOException ex) {
        Logger.DEBUG("failed to download " + from + " from HDFS: " + ex.getLocalizedMessage());
        Logger.EXCEPTION(ex);
        return -2;
    }
}
 

@Override
public String copyToLocal(String resourcePath) throws IOException {
    java.nio.file.Path localPath = Files.createTempDirectory("localized-keytab", PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------")));

    Path to = new Path(localPath.toString(), "keytab");
    Path from = new Path(resourcePath);

    FileSystem fs = FileSystem.get(from.toUri(), HConfiguration.unwrapDelegate());
    fs.copyToLocalFile(from, to);
    return to.toString();
}
 

/**
 * Copy file in DFS from p to newPath
 * @param fs file system
 * @param p old path
 * @param newPath new path
 * @throws IOException exception
 */
protected void copyFile(FileSystem fs, Path p, Path newPath) throws IOException {
  File f = File.createTempFile("data", "meta");
  Path localPath = new Path(f.getAbsolutePath());
  fs.copyToLocalFile(p, localPath);
  fs.copyFromLocalFile(localPath, newPath);
  boolean exists = fs.exists(newPath);
  if (!exists) {
    throw new IOException("Failed to copy meta file to: "+ newPath);
  }
}
 

private void copyResult(Configuration conf, String outputDirPath, String workerDirPath) throws IOException {
	FileSystem fileSystem = FileSystem.get(conf);
	Path path = new Path(
			outputDirPath + DataSourceCompConstants.SLASH + DataSourceCompConstants.TRANSFORMATION_VIOLATION);
	if (fileSystem.exists(path)) {
		fileSystem.copyToLocalFile(path, new Path(workerDirPath));
	}
}
 

/**
 * Copies the files from remote to local filesystem.
 * When 'multipleFiles' is set the path could point to multiple files
 * through globs or a directory. In this case, return array contains multiple
 * files, otherwise a single file is returned.
 *
 * If pig.jars.relative.to.dfs is true then a relative path is assumed to be
 * relative to the default filesystem's active directory.
 * Else they are assumed to be relative to the local working directory.
 *
 * @param properties
 * @param filePath
 * @param multipleFiles
 * @return
 */
private static FetchFileRet[] fetchFilesInternal(Properties properties,
                                        String filePath,
                                        boolean multipleFiles) throws IOException {

    Path path = new Path(filePath);
    if (path.getName().isEmpty()) {
        return new FetchFileRet[0];
    }
    URI uri = path.toUri();
    Configuration conf = new Configuration();
    ConfigurationUtil.mergeConf(conf, ConfigurationUtil.toConfiguration(properties));

    // if there is no schema or if the schema is "local", then it is
    // expected to be a local path.

    FileSystem localFs = FileSystem.getLocal(conf);
    FileSystem srcFs;
    if ( (!"true".equals(properties.getProperty("pig.jars.relative.to.dfs"))
            && uri.getScheme() == null )||
            // For Windows local files
            (uri.getScheme() == null && uri.getPath().matches("^/[A-Za-z]:.*")) ||
            (uri.getScheme() != null && uri.getScheme().equals("local"))
        ) {
        srcFs = localFs;
    } else {
        srcFs = path.getFileSystem(conf);
    }

    FileStatus[] files;

    if (multipleFiles) {
        files = srcFs.globStatus(path);
    } else {
        files = new FileStatus[]{ srcFs.getFileStatus(path) };
    }
    if (files == null || files.length == 0) {
        throw new ExecException("file '" + filePath + "' does not exist.", 101, PigException.INPUT);
    }

    FetchFileRet[] fetchFiles = new FetchFileRet[files.length];
    int idx = 0;

    for(FileStatus file : files) {
        // should throw an exception if this is not a file?

        String pathname = file.getPath().toUri().getPath();
        String filename = file.getPath().getName();

        if (srcFs == localFs) {
            fetchFiles[idx++] = new FetchFileRet(new File(pathname), false);
        } else {
            // fetch from remote:
            File dest = new File(localTempDir, filename);
            dest.deleteOnExit();
            try {
                srcFs.copyToLocalFile(file.getPath(), new Path(dest.getAbsolutePath()));
            } catch (IOException e) {
                throw new ExecException("Could not copy " + filePath + " to local destination " + dest, 101, PigException.INPUT, e);
            }
            fetchFiles[idx++] = new FetchFileRet(dest, true);
        }
    }

    return fetchFiles;
}
 

private void testReplayWorksWithMemoryCompactionPolicy(MemoryCompactionPolicy policy) throws
  IOException {
  Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
  // Set it so we flush every 1M or so.  Thats a lot.
  conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024*1024);
  conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, String.valueOf(policy).toLowerCase());
  // The file of recovered edits has a column family of 'meta'.
  final String columnFamily = "meta";
  byte[][] columnFamilyAsByteArray = new byte[][] { Bytes.toBytes(columnFamily) };
  TableDescriptor tableDescriptor = TableDescriptorBuilder
    .newBuilder(TableName.valueOf(testName.getMethodName())).setColumnFamily(
      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(columnFamily)).build())
    .build();
  RegionInfo hri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
  final String encodedRegionName = hri.getEncodedName();
  Path hbaseRootDir = TEST_UTIL.getDataTestDir();
  FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
  Path tableDir = CommonFSUtils.getTableDir(hbaseRootDir, tableDescriptor.getTableName());
  HRegionFileSystem hrfs =
      new HRegionFileSystem(TEST_UTIL.getConfiguration(), fs, tableDir, hri);
  if (fs.exists(hrfs.getRegionDir())) {
    LOG.info("Region directory already exists. Deleting.");
    fs.delete(hrfs.getRegionDir(), true);
  }
  HRegion region = HBaseTestingUtility
      .createRegionAndWAL(hri, hbaseRootDir, conf, tableDescriptor, blockCache);
  assertEquals(encodedRegionName, region.getRegionInfo().getEncodedName());
  List<String> storeFiles = region.getStoreFileList(columnFamilyAsByteArray);
  // There should be no store files.
  assertTrue(storeFiles.isEmpty());
  region.close();
  Path regionDir = FSUtils.getRegionDirFromRootDir(hbaseRootDir, hri);
  Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regionDir);
  // This is a little fragile getting this path to a file of 10M of edits.
  Path recoveredEditsFile = new Path(
    System.getProperty("test.build.classes", "target/test-classes"),
      "0000000000000016310");
  // Copy this file under the region's recovered.edits dir so it is replayed on reopen.
  Path destination = new Path(recoveredEditsDir, recoveredEditsFile.getName());
  fs.copyToLocalFile(recoveredEditsFile, destination);
  assertTrue(fs.exists(destination));
  // Now the file 0000000000000016310 is under recovered.edits, reopen the region to replay.
  region = HRegion.openHRegion(region, null);
  assertEquals(encodedRegionName, region.getRegionInfo().getEncodedName());
  storeFiles = region.getStoreFileList(columnFamilyAsByteArray);
  // Our 0000000000000016310 is 10MB. Most of the edits are for one region. Lets assume that if
  // we flush at 1MB, that there are at least 3 flushed files that are there because of the
  // replay of edits.
  if(policy == MemoryCompactionPolicy.EAGER || policy == MemoryCompactionPolicy.ADAPTIVE) {
    assertTrue("Files count=" + storeFiles.size(), storeFiles.size() >= 1);
  } else {
    assertTrue("Files count=" + storeFiles.size(), storeFiles.size() > 10);
  }
  // Now verify all edits made it into the region.
  int count = verifyAllEditsMadeItIn(fs, conf, recoveredEditsFile, region);
  LOG.info("Checked " + count + " edits made it in");
}
 

public void getFileFromHdfs(String hdfsPath, File dst) throws IllegalArgumentException, IOException {
  FileSystem fileSystem = this.getHadoopFileSystem();
  Path src = new Path(hdfsPath);
  fileSystem.copyToLocalFile(src, new Path(dst.getAbsolutePath()));
}
 

public void copyFiles(){
	LOG.info("Inside listFiles()");
	//Configuration conf = new Configuration();
	JobConf conf = new JobConf();
	// _____________ TO TEST THAT CORRECT HADOOP JARs ARE INCLUDED __________________
	ClassLoader cl = ClassLoader.getSystemClassLoader();
       URL[] urls = ((URLClassLoader)cl).getURLs();
       if(LOG.isDebugEnabled()) {
		for (URL url : urls) {
			LOG.debug(url.getFile());
		}
	}
	// _________________________________________
       String hdfsConnectionStr = configContext.getString("dataSourceConfig.hdfsConnection");
       LOG.info("HDFS connection string: " + hdfsConnectionStr);
      
	String hdfsPath = configContext.getString("dataSourceConfig.hdfsPath");
	LOG.info("HDFS path: " + hdfsPath);
	 
	String copyToPath = configContext.getString("dataSourceConfig.copyToPath");
	LOG.info("copyToPath: " + copyToPath);
	String srcPathStr = new String("hdfs://" + hdfsConnectionStr + hdfsPath);
	Path srcPath = new Path(srcPathStr); 
	LOG.info("listFiles called");
	LOG.info("srcPath: " + srcPath);
	try {
		FileSystem fs = srcPath.getFileSystem(conf);
		/*CompressionCodecFactory codecFactory = new CompressionCodecFactory(conf); 
		CompressionCodec codec = codecFactory.getCodec(srcPath);
		DataInputStream inputStream = new DataInputStream(codec.createInputStream(fs.open(srcPath)));
		*/
		
		Path destPath = new Path(copyToPath);
		LOG.info("Destination path: " + destPath);
		String userListFileName = configContext.getString("dataSourceConfig.userList");
		//loggerHDFSSpout.info("userListFileName: " + userListFileName);
		List<String> userList = getUser(userListFileName);
		for(String user:userList){
			Path finalSrcPath = new Path(srcPath.getName() + "/" + user);
			fs.copyToLocalFile(finalSrcPath, destPath);
		}
		LOG.info("Copy to local succeed");
		fs.close();
						
	} catch (IOException e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
	}
	
}
 

public JobInProgress(JobID jobid, JobTracker jobtracker, 
                     JobConf default_conf, int rCount) throws IOException {
  this.restartCount = rCount;
  this.jobId = jobid;
  String url = "http://" + jobtracker.getJobTrackerMachine() + ":" 
      + jobtracker.getInfoPort() + "/jobdetails.jsp?jobid=" + jobid;
  this.jobtracker = jobtracker;
  this.status = new JobStatus(jobid, 0.0f, 0.0f, JobStatus.PREP);
  this.startTime = System.currentTimeMillis();
  status.setStartTime(startTime);
  this.localFs = FileSystem.getLocal(default_conf);

  JobConf default_job_conf = new JobConf(default_conf);
  this.localJobFile = default_job_conf.getLocalPath(JobTracker.SUBDIR 
                                                    +"/"+jobid + ".xml");
  this.localJarFile = default_job_conf.getLocalPath(JobTracker.SUBDIR
                                                    +"/"+ jobid + ".jar");
  Path jobDir = jobtracker.getSystemDirectoryForJob(jobId);
  FileSystem fs = jobDir.getFileSystem(default_conf);
  jobFile = new Path(jobDir, "job.xml");
  fs.copyToLocalFile(jobFile, localJobFile);
  conf = new JobConf(localJobFile);
  this.priority = conf.getJobPriority();
  this.status.setJobPriority(this.priority);
  this.profile = new JobProfile(conf.getUser(), jobid, 
                                jobFile.toString(), url, conf.getJobName(),
                                conf.getQueueName());
  String jarFile = conf.getJar();
  if (jarFile != null) {
    fs.copyToLocalFile(new Path(jarFile), localJarFile);
    conf.setJar(localJarFile.toString());
  }

  this.numMapTasks = conf.getNumMapTasks();
  this.numReduceTasks = conf.getNumReduceTasks();
  this.taskCompletionEvents = new ArrayList<TaskCompletionEvent>
     (numMapTasks + numReduceTasks + 10);

  this.mapFailuresPercent = conf.getMaxMapTaskFailuresPercent();
  this.reduceFailuresPercent = conf.getMaxReduceTaskFailuresPercent();
      
  MetricsContext metricsContext = MetricsUtil.getContext("mapred");
  this.jobMetrics = MetricsUtil.createRecord(metricsContext, "job");
  this.jobMetrics.setTag("user", conf.getUser());
  this.jobMetrics.setTag("sessionId", conf.getSessionId());
  this.jobMetrics.setTag("jobName", conf.getJobName());
  this.jobMetrics.setTag("jobId", jobid.toString());
  hasSpeculativeMaps = conf.getMapSpeculativeExecution();
  hasSpeculativeReduces = conf.getReduceSpeculativeExecution();
  this.maxLevel = jobtracker.getNumTaskCacheLevels();
  this.anyCacheLevel = this.maxLevel+1;
  this.nonLocalMaps = new LinkedList<TaskInProgress>();
  this.nonLocalRunningMaps = new LinkedHashSet<TaskInProgress>();
  this.runningMapCache = new IdentityHashMap<Node, Set<TaskInProgress>>();
  this.nonRunningReduces = new LinkedList<TaskInProgress>();    
  this.runningReduces = new LinkedHashSet<TaskInProgress>();
  this.resourceEstimator = new ResourceEstimator(this);
}
 

/**
 * Publish job execution results into local file system, so you can view them.
 *
 * @param fs Distributed file sytem used in job.
 * @throws IOException If failed.
 */
private void publishResults(FileSystem fs) throws IOException {
    X.println(">>> Cleaning up DFS input directory: " + BOOKS_DFS_DIR);

    fs.delete(BOOKS_DFS_DIR, true);

    X.println(">>> Cleaning up LOCAL result directory: " + RESULT_LOCAL_DIR);

    fs.delete(RESULT_LOCAL_DIR, true);

    X.println(">>> Moving job results into LOCAL result directory: " + RESULT_LOCAL_DIR);

    fs.copyToLocalFile(true, RESULT_DFS_DIR, RESULT_LOCAL_DIR);
}
 

/**
 * Copies the file from HDFS into the local file system.
 * @param hdfsInputPath the HDFS input {@link Path}.
 * @param localOutputPath the local system output {@link Path}.
 * @param configuration the {@link Configuration} to use.
 * @throws IOException
 */
public static void copyHdfsToLocal(final Path hdfsInputPath, final Path localOutputPath, final Configuration configuration) throws IOException {
    final FileSystem fs = FileSystem.get(configuration);
    fs.copyToLocalFile(hdfsInputPath, localOutputPath);
}
 

/**
 * 上传至hdfs
 *
 * @return
 * @throws IOException
 */
public static String download(String fsFilePath, String localFilePath) throws IOException {
    FileSystem fileSystem = getFileSystem();
    fileSystem.copyToLocalFile(new Path(fsFilePath), new Path(localFilePath));
    return localFilePath;
}