Java Code Examples for org.apache.hadoop.fs.FileSystem#copyToLocalFile()
The following examples show how to use
org.apache.hadoop.fs.FileSystem#copyToLocalFile() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FileSystemClient.java From transport with BSD 2-Clause "Simplified" License | 6 votes |
public String copyToLocalFile(String remoteFilename) { try { loadProperties(); Configuration conf = getConfiguration(); login(conf); Path remotePath = new Path(remoteFilename); Path localPath = new Path(Paths.get(getAndCreateLocalDir(), new File(remoteFilename).getName()).toString()); FileSystem fs = remotePath.getFileSystem(conf); // It is important to pass the custom configuration object to FileSystemUtils since we load some extra // properties from etc/**.xml in getConfiguration() for Presto String resolvedRemoteFilename = FileSystemUtils.resolveLatest(remoteFilename, conf); Path resolvedRemotePath = new Path(resolvedRemoteFilename); fs.copyToLocalFile(resolvedRemotePath, localPath); return localPath.toString(); } catch (Exception e) { throw new RuntimeException("Error downloading HDFS file: " + remoteFilename, e); } }
Example 2
Source File: MapOpTestUtils.java From mrgeo with Apache License 2.0 | 6 votes |
public void generateBaselinePyramid(final Configuration conf, final String testName, final String ex) throws IOException, JobFailedException, JobCancelledException, ParserException { runMapAlgebraExpression(conf, testName, ex); final Path src = new Path(outputHdfs, testName); final MrsPyramid pyramid = MrsPyramid.open(src.toString(), (ProviderProperties) null); if (pyramid != null) { final Path dst = new Path(inputLocal, testName); final FileSystem fs = dst.getFileSystem(conf); fs.copyToLocalFile(src, dst); } }
Example 3
Source File: RelocalizationUtils.java From tez with Apache License 2.0 | 5 votes |
private static Path downloadResource(String destName, URI uri, Configuration conf, String destDir) throws IOException { FileSystem fs = FileSystem.get(uri, conf); Path cwd = new Path(destDir); Path dFile = new Path(cwd, destName); Path srcPath = new Path(uri); fs.copyToLocalFile(srcPath, dFile); return dFile.makeQualified(FileSystem.getLocal(conf).getUri(), cwd); }
Example 4
Source File: StramAppLauncher.java From Bats with Apache License 2.0 | 5 votes |
public StramAppLauncher(FileSystem fs, Path path, Configuration conf) throws Exception { File jarsDir = new File(StramClientUtils.getUserDTDirectory(), "jars"); jarsDir.mkdirs(); File localJarFile = new File(jarsDir, path.getName()); this.fs = fs; fs.copyToLocalFile(path, new Path(localJarFile.getAbsolutePath())); this.jarFile = localJarFile; this.conf = conf; this.propertiesBuilder = new LogicalPlanConfiguration(conf); init(this.jarFile.getName()); }
Example 5
Source File: HdfsUtil.java From spring-boot-tutorial with Creative Commons Attribution Share Alike 4.0 International | 5 votes |
/** * 下载文件 * * @param sourcePath 原文件路径(HDFS 中的路径) * @param targetPath 目标路径 * @throws Exception */ public void downloadFile(@NotBlank String sourcePath, @NotBlank String targetPath) throws Exception { FileSystem fileSystem = null; try { fileSystem = this.hdfsPool.borrowObject(); // 调用文件系统的文件复制方法,第一个参数为是否删除原文件(true为删除),默认为 false fileSystem.copyToLocalFile(false, new Path(sourcePath), new Path(targetPath)); } finally { if (fileSystem != null) { this.hdfsPool.returnObject(fileSystem); } } }
Example 6
Source File: ParseLogJob.java From 163-bigdate-note with GNU General Public License v3.0 | 5 votes |
public void setup(Context context) throws IOException, InterruptedException { FileSystem fs = FileSystem.get(context.getConfiguration()); Path ipFile = new Path("/user/hadoop/lib/17monipdb.dat"); Path localPath = new Path(this.getClass().getResource("/").getPath()); fs.copyToLocalFile(ipFile, localPath); IPUtil.load("17monipdb.dat"); }
Example 7
Source File: AWSJobConfigurationManager.java From incubator-gobblin with Apache License 2.0 | 5 votes |
@Override public String retrieve(Config config, String targetDir) throws IOException { URI uri = URI.create(this.fsUri); FileSystem fs = FileSystem.get(uri, HadoopUtils.getConfFromState(ConfigUtils.configToState(config))); final Path sourceFile = new Path(path); final String zipFile = appendSlash(targetDir) + StringUtils.substringAfterLast(this.path, File.separator); LOGGER.debug("Downloading to zip: " + zipFile + " from uri: " + sourceFile); fs.copyToLocalFile(sourceFile, new Path(zipFile)); return zipFile; }
Example 8
Source File: MapOpTestVectorUtils.java From mrgeo with Apache License 2.0 | 5 votes |
public void generateBaselineVector(final Configuration conf, final String testName, final String ex) throws IOException, ParserException, JobFailedException, JobCancelledException { runMapAlgebraExpression(conf, testName, ex); final Path src = new Path(outputHdfs, testName); final FileSystem srcfs = src.getFileSystem(conf); if (srcfs.exists(src)) { final Path dst = new Path(inputLocal, testName); final FileSystem fs = dst.getFileSystem(conf); fs.copyToLocalFile(src, dst); } }
Example 9
Source File: HalvadeFileUtils.java From halvade with GNU General Public License v3.0 | 5 votes |
/** * @return returns 0 if successfull, -1 if filesize is incorrect and -2 if an exception occurred */ protected static int privateDownloadFileFromHDFS(TaskInputOutputContext context, FileSystem fs, String from, String to) { try { // check if file is present on local scratch File f = new File(to); if(!f.exists()) { Logger.DEBUG("attempting download of \"" + to + "\""); fs.copyToLocalFile(new Path(from), new Path(to)); context.getCounter(HalvadeCounters.FIN_FROM_HDFS).increment(fs.getFileStatus(new Path(from)).getLen()); } else { // check if filesize is correct if(fs.getFileStatus(new Path(from)).getLen() != f.length()) { // incorrect filesize, remove and download again Logger.DEBUG("incorrect filesize: " + f.length() + " =/= " + fs.getFileStatus(new Path(from)).getLen()); f.delete(); fs.copyToLocalFile(new Path(from), new Path(to)); context.getCounter(HalvadeCounters.FIN_FROM_HDFS).increment(fs.getFileStatus(new Path(from)).getLen()); } else { Logger.DEBUG("file \"" + to + "\" exists"); } } if(fs.getFileStatus(new Path(from)).getLen() != f.length()) return -1; else return 0; } catch (IOException ex) { Logger.DEBUG("failed to download " + from + " from HDFS: " + ex.getLocalizedMessage()); Logger.EXCEPTION(ex); return -2; } }
Example 10
Source File: HdfsResourceLocalizer.java From spliceengine with GNU Affero General Public License v3.0 | 5 votes |
@Override public String copyToLocal(String resourcePath) throws IOException { java.nio.file.Path localPath = Files.createTempDirectory("localized-keytab", PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------"))); Path to = new Path(localPath.toString(), "keytab"); Path from = new Path(resourcePath); FileSystem fs = FileSystem.get(from.toUri(), HConfiguration.unwrapDelegate()); fs.copyToLocalFile(from, to); return to.toString(); }
Example 11
Source File: MapReduceBackupMergeJob.java From hbase with Apache License 2.0 | 5 votes |
/** * Copy file in DFS from p to newPath * @param fs file system * @param p old path * @param newPath new path * @throws IOException exception */ protected void copyFile(FileSystem fs, Path p, Path newPath) throws IOException { File f = File.createTempFile("data", "meta"); Path localPath = new Path(f.getAbsolutePath()); fs.copyToLocalFile(p, localPath); fs.copyFromLocalFile(localPath, newPath); boolean exists = fs.exists(newPath); if (!exists) { throw new IOException("Failed to copy meta file to: "+ newPath); } }
Example 12
Source File: DataSourceCompJobExecutor.java From jumbune with GNU Lesser General Public License v3.0 | 5 votes |
private void copyResult(Configuration conf, String outputDirPath, String workerDirPath) throws IOException { FileSystem fileSystem = FileSystem.get(conf); Path path = new Path( outputDirPath + DataSourceCompConstants.SLASH + DataSourceCompConstants.TRANSFORMATION_VIOLATION); if (fileSystem.exists(path)) { fileSystem.copyToLocalFile(path, new Path(workerDirPath)); } }
Example 13
Source File: FileLocalizer.java From spork with Apache License 2.0 | 4 votes |
/** * Copies the files from remote to local filesystem. * When 'multipleFiles' is set the path could point to multiple files * through globs or a directory. In this case, return array contains multiple * files, otherwise a single file is returned. * * If pig.jars.relative.to.dfs is true then a relative path is assumed to be * relative to the default filesystem's active directory. * Else they are assumed to be relative to the local working directory. * * @param properties * @param filePath * @param multipleFiles * @return */ private static FetchFileRet[] fetchFilesInternal(Properties properties, String filePath, boolean multipleFiles) throws IOException { Path path = new Path(filePath); if (path.getName().isEmpty()) { return new FetchFileRet[0]; } URI uri = path.toUri(); Configuration conf = new Configuration(); ConfigurationUtil.mergeConf(conf, ConfigurationUtil.toConfiguration(properties)); // if there is no schema or if the schema is "local", then it is // expected to be a local path. FileSystem localFs = FileSystem.getLocal(conf); FileSystem srcFs; if ( (!"true".equals(properties.getProperty("pig.jars.relative.to.dfs")) && uri.getScheme() == null )|| // For Windows local files (uri.getScheme() == null && uri.getPath().matches("^/[A-Za-z]:.*")) || (uri.getScheme() != null && uri.getScheme().equals("local")) ) { srcFs = localFs; } else { srcFs = path.getFileSystem(conf); } FileStatus[] files; if (multipleFiles) { files = srcFs.globStatus(path); } else { files = new FileStatus[]{ srcFs.getFileStatus(path) }; } if (files == null || files.length == 0) { throw new ExecException("file '" + filePath + "' does not exist.", 101, PigException.INPUT); } FetchFileRet[] fetchFiles = new FetchFileRet[files.length]; int idx = 0; for(FileStatus file : files) { // should throw an exception if this is not a file? String pathname = file.getPath().toUri().getPath(); String filename = file.getPath().getName(); if (srcFs == localFs) { fetchFiles[idx++] = new FetchFileRet(new File(pathname), false); } else { // fetch from remote: File dest = new File(localTempDir, filename); dest.deleteOnExit(); try { srcFs.copyToLocalFile(file.getPath(), new Path(dest.getAbsolutePath())); } catch (IOException e) { throw new ExecException("Could not copy " + filePath + " to local destination " + dest, 101, PigException.INPUT, e); } fetchFiles[idx++] = new FetchFileRet(dest, true); } } return fetchFiles; }
Example 14
Source File: TestRecoveredEdits.java From hbase with Apache License 2.0 | 4 votes |
private void testReplayWorksWithMemoryCompactionPolicy(MemoryCompactionPolicy policy) throws IOException { Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); // Set it so we flush every 1M or so. Thats a lot. conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024*1024); conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, String.valueOf(policy).toLowerCase()); // The file of recovered edits has a column family of 'meta'. final String columnFamily = "meta"; byte[][] columnFamilyAsByteArray = new byte[][] { Bytes.toBytes(columnFamily) }; TableDescriptor tableDescriptor = TableDescriptorBuilder .newBuilder(TableName.valueOf(testName.getMethodName())).setColumnFamily( ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(columnFamily)).build()) .build(); RegionInfo hri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); final String encodedRegionName = hri.getEncodedName(); Path hbaseRootDir = TEST_UTIL.getDataTestDir(); FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration()); Path tableDir = CommonFSUtils.getTableDir(hbaseRootDir, tableDescriptor.getTableName()); HRegionFileSystem hrfs = new HRegionFileSystem(TEST_UTIL.getConfiguration(), fs, tableDir, hri); if (fs.exists(hrfs.getRegionDir())) { LOG.info("Region directory already exists. Deleting."); fs.delete(hrfs.getRegionDir(), true); } HRegion region = HBaseTestingUtility .createRegionAndWAL(hri, hbaseRootDir, conf, tableDescriptor, blockCache); assertEquals(encodedRegionName, region.getRegionInfo().getEncodedName()); List<String> storeFiles = region.getStoreFileList(columnFamilyAsByteArray); // There should be no store files. assertTrue(storeFiles.isEmpty()); region.close(); Path regionDir = FSUtils.getRegionDirFromRootDir(hbaseRootDir, hri); Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regionDir); // This is a little fragile getting this path to a file of 10M of edits. Path recoveredEditsFile = new Path( System.getProperty("test.build.classes", "target/test-classes"), "0000000000000016310"); // Copy this file under the region's recovered.edits dir so it is replayed on reopen. Path destination = new Path(recoveredEditsDir, recoveredEditsFile.getName()); fs.copyToLocalFile(recoveredEditsFile, destination); assertTrue(fs.exists(destination)); // Now the file 0000000000000016310 is under recovered.edits, reopen the region to replay. region = HRegion.openHRegion(region, null); assertEquals(encodedRegionName, region.getRegionInfo().getEncodedName()); storeFiles = region.getStoreFileList(columnFamilyAsByteArray); // Our 0000000000000016310 is 10MB. Most of the edits are for one region. Lets assume that if // we flush at 1MB, that there are at least 3 flushed files that are there because of the // replay of edits. if(policy == MemoryCompactionPolicy.EAGER || policy == MemoryCompactionPolicy.ADAPTIVE) { assertTrue("Files count=" + storeFiles.size(), storeFiles.size() >= 1); } else { assertTrue("Files count=" + storeFiles.size(), storeFiles.size() > 10); } // Now verify all edits made it into the region. int count = verifyAllEditsMadeItIn(fs, conf, recoveredEditsFile, region); LOG.info("Checked " + count + " edits made it in"); }
Example 15
Source File: HdfsFileLoader.java From cognition with Apache License 2.0 | 4 votes |
public void getFileFromHdfs(String hdfsPath, File dst) throws IllegalArgumentException, IOException { FileSystem fileSystem = this.getHadoopFileSystem(); Path src = new Path(hdfsPath); fileSystem.copyToLocalFile(src, new Path(dst.getAbsolutePath())); }
Example 16
Source File: UserProfileGenerationHDFSSpout.java From Eagle with Apache License 2.0 | 4 votes |
public void copyFiles(){ LOG.info("Inside listFiles()"); //Configuration conf = new Configuration(); JobConf conf = new JobConf(); // _____________ TO TEST THAT CORRECT HADOOP JARs ARE INCLUDED __________________ ClassLoader cl = ClassLoader.getSystemClassLoader(); URL[] urls = ((URLClassLoader)cl).getURLs(); if(LOG.isDebugEnabled()) { for (URL url : urls) { LOG.debug(url.getFile()); } } // _________________________________________ String hdfsConnectionStr = configContext.getString("dataSourceConfig.hdfsConnection"); LOG.info("HDFS connection string: " + hdfsConnectionStr); String hdfsPath = configContext.getString("dataSourceConfig.hdfsPath"); LOG.info("HDFS path: " + hdfsPath); String copyToPath = configContext.getString("dataSourceConfig.copyToPath"); LOG.info("copyToPath: " + copyToPath); String srcPathStr = new String("hdfs://" + hdfsConnectionStr + hdfsPath); Path srcPath = new Path(srcPathStr); LOG.info("listFiles called"); LOG.info("srcPath: " + srcPath); try { FileSystem fs = srcPath.getFileSystem(conf); /*CompressionCodecFactory codecFactory = new CompressionCodecFactory(conf); CompressionCodec codec = codecFactory.getCodec(srcPath); DataInputStream inputStream = new DataInputStream(codec.createInputStream(fs.open(srcPath))); */ Path destPath = new Path(copyToPath); LOG.info("Destination path: " + destPath); String userListFileName = configContext.getString("dataSourceConfig.userList"); //loggerHDFSSpout.info("userListFileName: " + userListFileName); List<String> userList = getUser(userListFileName); for(String user:userList){ Path finalSrcPath = new Path(srcPath.getName() + "/" + user); fs.copyToLocalFile(finalSrcPath, destPath); } LOG.info("Copy to local succeed"); fs.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } }
Example 17
Source File: JobInProgress.java From hadoop-gpu with Apache License 2.0 | 4 votes |
public JobInProgress(JobID jobid, JobTracker jobtracker, JobConf default_conf, int rCount) throws IOException { this.restartCount = rCount; this.jobId = jobid; String url = "http://" + jobtracker.getJobTrackerMachine() + ":" + jobtracker.getInfoPort() + "/jobdetails.jsp?jobid=" + jobid; this.jobtracker = jobtracker; this.status = new JobStatus(jobid, 0.0f, 0.0f, JobStatus.PREP); this.startTime = System.currentTimeMillis(); status.setStartTime(startTime); this.localFs = FileSystem.getLocal(default_conf); JobConf default_job_conf = new JobConf(default_conf); this.localJobFile = default_job_conf.getLocalPath(JobTracker.SUBDIR +"/"+jobid + ".xml"); this.localJarFile = default_job_conf.getLocalPath(JobTracker.SUBDIR +"/"+ jobid + ".jar"); Path jobDir = jobtracker.getSystemDirectoryForJob(jobId); FileSystem fs = jobDir.getFileSystem(default_conf); jobFile = new Path(jobDir, "job.xml"); fs.copyToLocalFile(jobFile, localJobFile); conf = new JobConf(localJobFile); this.priority = conf.getJobPriority(); this.status.setJobPriority(this.priority); this.profile = new JobProfile(conf.getUser(), jobid, jobFile.toString(), url, conf.getJobName(), conf.getQueueName()); String jarFile = conf.getJar(); if (jarFile != null) { fs.copyToLocalFile(new Path(jarFile), localJarFile); conf.setJar(localJarFile.toString()); } this.numMapTasks = conf.getNumMapTasks(); this.numReduceTasks = conf.getNumReduceTasks(); this.taskCompletionEvents = new ArrayList<TaskCompletionEvent> (numMapTasks + numReduceTasks + 10); this.mapFailuresPercent = conf.getMaxMapTaskFailuresPercent(); this.reduceFailuresPercent = conf.getMaxReduceTaskFailuresPercent(); MetricsContext metricsContext = MetricsUtil.getContext("mapred"); this.jobMetrics = MetricsUtil.createRecord(metricsContext, "job"); this.jobMetrics.setTag("user", conf.getUser()); this.jobMetrics.setTag("sessionId", conf.getSessionId()); this.jobMetrics.setTag("jobName", conf.getJobName()); this.jobMetrics.setTag("jobId", jobid.toString()); hasSpeculativeMaps = conf.getMapSpeculativeExecution(); hasSpeculativeReduces = conf.getReduceSpeculativeExecution(); this.maxLevel = jobtracker.getNumTaskCacheLevels(); this.anyCacheLevel = this.maxLevel+1; this.nonLocalMaps = new LinkedList<TaskInProgress>(); this.nonLocalRunningMaps = new LinkedHashSet<TaskInProgress>(); this.runningMapCache = new IdentityHashMap<Node, Set<TaskInProgress>>(); this.nonRunningReduces = new LinkedList<TaskInProgress>(); this.runningReduces = new LinkedHashSet<TaskInProgress>(); this.resourceEstimator = new ResourceEstimator(this); }
Example 18
Source File: HadoopPopularWords.java From ignite with Apache License 2.0 | 3 votes |
/** * Publish job execution results into local file system, so you can view them. * * @param fs Distributed file sytem used in job. * @throws IOException If failed. */ private void publishResults(FileSystem fs) throws IOException { X.println(">>> Cleaning up DFS input directory: " + BOOKS_DFS_DIR); fs.delete(BOOKS_DFS_DIR, true); X.println(">>> Cleaning up LOCAL result directory: " + RESULT_LOCAL_DIR); fs.delete(RESULT_LOCAL_DIR, true); X.println(">>> Moving job results into LOCAL result directory: " + RESULT_LOCAL_DIR); fs.copyToLocalFile(true, RESULT_DFS_DIR, RESULT_LOCAL_DIR); }
Example 19
Source File: CopyTool.java From rya with Apache License 2.0 | 2 votes |
/** * Copies the file from HDFS into the local file system. * @param hdfsInputPath the HDFS input {@link Path}. * @param localOutputPath the local system output {@link Path}. * @param configuration the {@link Configuration} to use. * @throws IOException */ public static void copyHdfsToLocal(final Path hdfsInputPath, final Path localOutputPath, final Configuration configuration) throws IOException { final FileSystem fs = FileSystem.get(configuration); fs.copyToLocalFile(hdfsInputPath, localOutputPath); }
Example 20
Source File: HadoopUtils.java From liteflow with Apache License 2.0 | 2 votes |
/** * 上传至hdfs * * @return * @throws IOException */ public static String download(String fsFilePath, String localFilePath) throws IOException { FileSystem fileSystem = getFileSystem(); fileSystem.copyToLocalFile(new Path(fsFilePath), new Path(localFilePath)); return localFilePath; }