Java Code Examples for org.apache.hadoop.fs.FSDataOutputStream#close()
The following examples show how to use
org.apache.hadoop.fs.FSDataOutputStream#close() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: LoadGenerator.java From RDFS with Apache License 2.0 | 6 votes |
/** Create a file with a length of <code>fileSize</code>. * The file is filled with 'a'. */ private void genFile(Path file, long fileSize) throws IOException { long startTime = System.currentTimeMillis(); FSDataOutputStream out = fs.create(file, true, getConf().getInt("io.file.buffer.size", 4096), (short)getConf().getInt("dfs.replication", 3), fs.getDefaultBlockSize()); executionTime[CREATE] += (System.currentTimeMillis()-startTime); totalNumOfOps[CREATE]++; for (long i=0; i<fileSize; i++) { out.writeByte('a'); } startTime = System.currentTimeMillis(); out.close(); executionTime[WRITE_CLOSE] += (System.currentTimeMillis()-startTime); totalNumOfOps[WRITE_CLOSE]++; }
Example 2
Source File: TupleSchemaRegistry.java From attic-apex-malhar with Apache License 2.0 | 6 votes |
public String generateCommonJar() throws IOException { File file = File.createTempFile("schemaSQL", ".jar"); FileSystem fs = FileSystem.newInstance(file.toURI(), new Configuration()); FSDataOutputStream out = fs.create(new Path(file.getAbsolutePath())); JarOutputStream jout = new JarOutputStream(out); for (Schema schema : schemas.values()) { jout.putNextEntry(new ZipEntry(schema.fqcn.replace(".", "/") + ".class")); jout.write(schema.beanClassBytes); jout.closeEntry(); } jout.close(); out.close(); return file.getAbsolutePath(); }
Example 3
Source File: TestMultiFileInputFormat.java From big-c with Apache License 2.0 | 6 votes |
private Path initFiles(FileSystem fs, int numFiles, int numBytes) throws IOException{ Path dir = new Path(System.getProperty("test.build.data",".") + "/mapred"); Path multiFileDir = new Path(dir, "test.multifile"); fs.delete(multiFileDir, true); fs.mkdirs(multiFileDir); LOG.info("Creating " + numFiles + " file(s) in " + multiFileDir); for(int i=0; i<numFiles ;i++) { Path path = new Path(multiFileDir, "file_" + i); FSDataOutputStream out = fs.create(path); if (numBytes == -1) { numBytes = rand.nextInt(MAX_BYTES); } for(int j=0; j< numBytes; j++) { out.write(rand.nextInt()); } out.close(); if(LOG.isDebugEnabled()) { LOG.debug("Created file " + path + " with length " + numBytes); } lengths.put(path.getName(), new Long(numBytes)); } FileInputFormat.setInputPaths(job, multiFileDir); return multiFileDir; }
Example 4
Source File: TestCatalogJanitor.java From hbase with Apache License 2.0 | 6 votes |
private FileStatus[] addMockStoreFiles(int count, MasterServices services, Path storedir) throws IOException { // get the existing store files FileSystem fs = services.getMasterFileSystem().getFileSystem(); fs.mkdirs(storedir); // create the store files in the parent for (int i = 0; i < count; i++) { Path storeFile = new Path(storedir, "_store" + i); FSDataOutputStream dos = fs.create(storeFile, true); dos.writeBytes("Some data: " + i); dos.close(); } LOG.debug("Adding " + count + " store files to the storedir:" + storedir); // make sure the mock store files are there FileStatus[] storeFiles = fs.listStatus(storedir); assertEquals("Didn't have expected store files", count, storeFiles.length); return storeFiles; }
Example 5
Source File: TestImportTSVWithOperationAttributes.java From hbase with Apache License 2.0 | 5 votes |
/** * Run an ImportTsv job and perform basic validation on the results. Returns * the ImportTsv <code>Tool</code> instance so that other tests can inspect it * for further validation as necessary. This method is static to insure * non-reliance on instance's util/conf facilities. * * @param args * Any arguments to pass BEFORE inputFile path is appended. * @param dataAvailable * @return The Tool instance used to run the test. */ private Tool doMROnTableTest(HBaseTestingUtility util, String family, String data, String[] args, int valueMultiplier, boolean dataAvailable) throws Exception { String table = args[args.length - 1]; Configuration conf = new Configuration(util.getConfiguration()); // populate input file FileSystem fs = FileSystem.get(conf); Path inputPath = fs.makeQualified(new Path(util.getDataTestDirOnTestFS(table), "input.dat")); FSDataOutputStream op = fs.create(inputPath, true); op.write(Bytes.toBytes(data)); op.close(); LOG.debug(String.format("Wrote test data to file: %s", inputPath)); if (conf.getBoolean(FORCE_COMBINER_CONF, true)) { LOG.debug("Forcing combiner."); conf.setInt("mapreduce.map.combine.minspills", 1); } // run the import List<String> argv = new ArrayList<>(Arrays.asList(args)); argv.add(inputPath.toString()); Tool tool = new ImportTsv(); LOG.debug("Running ImportTsv with arguments: " + argv); assertEquals(0, ToolRunner.run(conf, tool, argv.toArray(args))); validateTable(conf, TableName.valueOf(table), family, valueMultiplier, dataAvailable); if (conf.getBoolean(DELETE_AFTER_LOAD_CONF, true)) { LOG.debug("Deleting test subdirectory"); util.cleanupDataTestDirOnTestFS(table); } return tool; }
Example 6
Source File: TestFileAppend.java From big-c with Apache License 2.0 | 5 votes |
/** Test two consecutive appends on a file with a full block. */ @Test public void testAppendTwice() throws Exception { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); final FileSystem fs1 = cluster.getFileSystem(); final FileSystem fs2 = AppendTestUtil.createHdfsWithDifferentUsername(conf); try { final Path p = new Path("/testAppendTwice/foo"); final int len = 1 << 16; final byte[] fileContents = AppendTestUtil.initBuffer(len); { // create a new file with a full block. FSDataOutputStream out = fs2.create(p, true, 4096, (short)1, len); out.write(fileContents, 0, len); out.close(); } //1st append does not add any data so that the last block remains full //and the last block in INodeFileUnderConstruction is a BlockInfo //but not BlockInfoUnderConstruction. fs2.append(p); //2nd append should get AlreadyBeingCreatedException fs1.append(p); Assert.fail(); } catch(RemoteException re) { AppendTestUtil.LOG.info("Got an exception:", re); Assert.assertEquals(AlreadyBeingCreatedException.class.getName(), re.getClassName()); } finally { fs2.close(); fs1.close(); cluster.shutdown(); } }
Example 7
Source File: TestMRJobs.java From big-c with Apache License 2.0 | 5 votes |
private Path createTempFile(String filename, String contents) throws IOException { Path path = new Path(TEST_ROOT_DIR, filename); FSDataOutputStream os = localFs.create(path); os.writeBytes(contents); os.close(); localFs.setPermission(path, new FsPermission("700")); return path; }
Example 8
Source File: FileSystemContractBaseTest.java From hadoop-gpu with Apache License 2.0 | 5 votes |
public void testOutputStreamClosedTwice() throws IOException { //HADOOP-4760 according to Closeable#close() closing already-closed //streams should have no effect. Path src = path("/test/hadoop/file"); FSDataOutputStream out = fs.create(src); out.writeChar('H'); //write some data out.close(); out.close(); }
Example 9
Source File: AbstractContractAppendTest.java From big-c with Apache License 2.0 | 5 votes |
@Test public void testAppendNonexistentFile() throws Throwable { try { FSDataOutputStream out = getFileSystem().append(target); //got here: trouble out.close(); fail("expected a failure"); } catch (Exception e) { //expected handleExpectedException(e); } }
Example 10
Source File: FSStorageAgent.java From attic-apex-core with Apache License 2.0 | 5 votes |
@SuppressWarnings("ThrowFromFinallyBlock") @Override public void save(Object object, int operatorId, long windowId) throws IOException { String operatorIdStr = String.valueOf(operatorId); Path lPath = new Path(path + Path.SEPARATOR + operatorIdStr + Path.SEPARATOR + TMP_FILE); String window = Long.toHexString(windowId); boolean stateSaved = false; FSDataOutputStream stream = null; try { stream = fileContext.create(lPath, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), Options.CreateOpts.CreateParent.createParent()); store(stream, object); stateSaved = true; } catch (Throwable t) { logger.debug("while saving {} {}", operatorId, window, t); stateSaved = false; throw Throwables.propagate(t); } finally { try { if (stream != null) { stream.close(); } } catch (IOException ie) { stateSaved = false; throw new RuntimeException(ie); } finally { if (stateSaved) { logger.debug("Saving {}: {}", operatorId, window); fileContext.rename(lPath, new Path(path + Path.SEPARATOR + operatorIdStr + Path.SEPARATOR + window), Options.Rename.OVERWRITE); } } } }
Example 11
Source File: TestDiskspaceQuotaUpdate.java From hadoop with Apache License 2.0 | 5 votes |
/** * Test if the quota can be correctly updated when file length is updated * through fsync */ @Test (timeout=60000) public void testUpdateQuotaForFSync() throws Exception { final Path foo = new Path("/foo"); final Path bar = new Path(foo, "bar"); DFSTestUtil.createFile(dfs, bar, BLOCKSIZE, REPLICATION, 0L); dfs.setQuota(foo, Long.MAX_VALUE - 1, Long.MAX_VALUE - 1); FSDataOutputStream out = dfs.append(bar); out.write(new byte[BLOCKSIZE / 4]); ((DFSOutputStream) out.getWrappedStream()).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH)); INodeDirectory fooNode = fsdir.getINode4Write(foo.toString()).asDirectory(); QuotaCounts quota = fooNode.getDirectoryWithQuotaFeature() .getSpaceConsumed(); long ns = quota.getNameSpace(); long ds = quota.getStorageSpace(); assertEquals(2, ns); // foo and bar assertEquals(BLOCKSIZE * 2 * REPLICATION, ds); // file is under construction out.write(new byte[BLOCKSIZE / 4]); out.close(); fooNode = fsdir.getINode4Write(foo.toString()).asDirectory(); quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed(); ns = quota.getNameSpace(); ds = quota.getStorageSpace(); assertEquals(2, ns); assertEquals((BLOCKSIZE + BLOCKSIZE / 2) * REPLICATION, ds); // append another block DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE); quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed(); ns = quota.getNameSpace(); ds = quota.getStorageSpace(); assertEquals(2, ns); // foo and bar assertEquals((BLOCKSIZE * 2 + BLOCKSIZE / 2) * REPLICATION, ds); }
Example 12
Source File: SpliceTableAdmin.java From spliceengine with GNU Affero General Public License v3.0 | 5 votes |
/** * This system procedure checks table consistency * @param schemaName * @param tableName * @param level * @param outputFile * @param resultSet * @throws Exception */ public static void CHECK_TABLE(String schemaName, String tableName, int level, String outputFile, boolean fix, final ResultSet[] resultSet) throws Exception { FSDataOutputStream out = null; FileSystem fs = null; Map<String, List<String>> errors = null; try { LanguageConnectionContext lcc = ConnectionUtil.getCurrentLCC(); Activation activation = lcc.getLastActivation(); Configuration conf = (Configuration) SIDriver.driver().getConfiguration().getConfigSource().unwrapDelegate(); String schema = EngineUtils.validateSchema(schemaName); String table = EngineUtils.validateTable(tableName); fs = FileSystem.get(URI.create(outputFile), conf); out = fs.create(new Path(outputFile)); errors = checkTable(schema, table, null, level, fix); resultSet[0] = processResults(errors, out, activation, outputFile); } finally { if (out != null) { out.close(); if (errors == null || errors.size() == 0) { fs.delete(new Path(outputFile), true); } } } }
Example 13
Source File: TestIFile.java From tez with Apache License 2.0 | 5 votes |
private Writer writeTestFile(boolean rle, boolean repeatKeys, List<KVPair> data, CompressionCodec codec) throws IOException { FSDataOutputStream out = localFs.create(outputPath); IFile.Writer writer = new IFile.Writer(defaultConf, out, Text.class, IntWritable.class, codec, null, null, rle); writeTestFile(writer, repeatKeys, data); out.close(); return writer; }
Example 14
Source File: TestBlockTokenWithDFS.java From hadoop with Apache License 2.0 | 4 votes |
/** * testing that APPEND operation can handle token expiration when * re-establishing pipeline is needed */ @Test public void testAppend() throws Exception { MiniDFSCluster cluster = null; int numDataNodes = 2; Configuration conf = getConf(numDataNodes); try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build(); cluster.waitActive(); assertEquals(numDataNodes, cluster.getDataNodes().size()); final NameNode nn = cluster.getNameNode(); final BlockManager bm = nn.getNamesystem().getBlockManager(); final BlockTokenSecretManager sm = bm.getBlockTokenSecretManager(); // set a short token lifetime (1 second) SecurityTestUtil.setBlockTokenLifetime(sm, 1000L); Path fileToAppend = new Path(FILE_TO_APPEND); FileSystem fs = cluster.getFileSystem(); // write a one-byte file FSDataOutputStream stm = writeFile(fs, fileToAppend, (short) numDataNodes, BLOCK_SIZE); stm.write(rawData, 0, 1); stm.close(); // open the file again for append stm = fs.append(fileToAppend); int mid = rawData.length - 1; stm.write(rawData, 1, mid - 1); stm.hflush(); /* * wait till token used in stm expires */ Token<BlockTokenIdentifier> token = DFSTestUtil.getBlockToken(stm); while (!SecurityTestUtil.isBlockTokenExpired(token)) { try { Thread.sleep(10); } catch (InterruptedException ignored) { } } // remove a datanode to force re-establishing pipeline cluster.stopDataNode(0); // append the rest of the file stm.write(rawData, mid, rawData.length - mid); stm.close(); // check if append is successful FSDataInputStream in5 = fs.open(fileToAppend); assertTrue(checkFile1(in5)); } finally { if (cluster != null) { cluster.shutdown(); } } }
Example 15
Source File: OfflineEditsViewerHelper.java From RDFS with Apache License 2.0 | 4 votes |
/** * Run file operations to create edits for all op codes * to be tested. */ private void runOperations() throws IOException { LOG.info("Creating edits by performing fs operations"); // no check, if it's not it throws an exception which is what we want DistributedFileSystem dfs = (DistributedFileSystem)cluster.getFileSystem(); // OP_ADD 0, OP_SET_GENSTAMP 10 Path pathFileCreate = new Path("/file_create"); FSDataOutputStream s = dfs.create(pathFileCreate); // OP_CLOSE 9 s.close(); // OP_RENAME 1 Path pathFileMoved = new Path("/file_moved"); dfs.rename(pathFileCreate, pathFileMoved); // OP_DELETE 2 dfs.delete(pathFileMoved, false); // OP_MKDIR 3 Path pathDirectoryMkdir = new Path("/directory_mkdir"); dfs.mkdirs(pathDirectoryMkdir); // OP_SET_REPLICATION 4 s = dfs.create(pathFileCreate); s.close(); dfs.setReplication(pathFileCreate, (short)1); // OP_SET_PERMISSIONS 7 Short permission = 0777; dfs.setPermission(pathFileCreate, new FsPermission(permission)); // OP_SET_OWNER 8 dfs.setOwner(pathFileCreate, new String("newOwner"), null); // OP_CLOSE 9 see above // OP_SET_GENSTAMP 10 see above // OP_SET_NS_QUOTA 11 obsolete // OP_CLEAR_NS_QUOTA 12 obsolete // OP_TIMES 13 long mtime = 1285195527000L; // Wed, 22 Sep 2010 22:45:27 GMT long atime = mtime; dfs.setTimes(pathFileCreate, mtime, atime); // OP_SET_QUOTA 14 dfs.setQuota(pathDirectoryMkdir, 1000L, FSConstants.QUOTA_DONT_SET); // OP_CONCAT_DELETE 16 Path pathConcatTarget = new Path("/file_concat_target"); Path[] pathConcatFiles = new Path[2]; pathConcatFiles[0] = new Path("/file_concat_0"); pathConcatFiles[1] = new Path("/file_concat_1"); long length = blockSize * 3; // multiple of blocksize for concat short replication = 1; long seed = 1; DFSTestUtil.createFile(dfs, pathConcatTarget, length, replication, seed); DFSTestUtil.createFile(dfs, pathConcatFiles[0], length, replication, seed); DFSTestUtil.createFile(dfs, pathConcatFiles[1], length, replication, seed); dfs.concat(pathConcatTarget, pathConcatFiles, false); // sync to disk, otherwise we parse partial edits cluster.getNameNode().getFSImage().getEditLog().logSync(); dfs.close(); }
Example 16
Source File: TestDataTransferProtocol.java From RDFS with Apache License 2.0 | 4 votes |
void createFile(FileSystem fs, Path path, int fileLen) throws IOException { byte [] arr = new byte[fileLen]; FSDataOutputStream out = fs.create(path); out.write(arr); out.close(); }
Example 17
Source File: TestChecksumFileSystem.java From hadoop with Apache License 2.0 | 4 votes |
@Test public void testVerifyChecksum() throws Exception { Path testPath = new Path(TEST_ROOT_DIR, "testPath"); Path testPath11 = new Path(TEST_ROOT_DIR, "testPath11"); FSDataOutputStream fout = localFs.create(testPath); fout.write("testing".getBytes()); fout.close(); fout = localFs.create(testPath11); fout.write("testing you".getBytes()); fout.close(); // Exercise some boundary cases - a divisor of the chunk size // the chunk size, 2x chunk size, and +/-1 around these. readFile(localFs, testPath, 128); readFile(localFs, testPath, 511); readFile(localFs, testPath, 512); readFile(localFs, testPath, 513); readFile(localFs, testPath, 1023); readFile(localFs, testPath, 1024); readFile(localFs, testPath, 1025); localFs.delete(localFs.getChecksumFile(testPath), true); assertTrue("checksum deleted", !localFs.exists(localFs.getChecksumFile(testPath))); //copying the wrong checksum file FileUtil.copy(localFs, localFs.getChecksumFile(testPath11), localFs, localFs.getChecksumFile(testPath),false,true,localFs.getConf()); assertTrue("checksum exists", localFs.exists(localFs.getChecksumFile(testPath))); boolean errorRead = false; try { readFile(localFs, testPath, 1024); }catch(ChecksumException ie) { errorRead = true; } assertTrue("error reading", errorRead); //now setting verify false, the read should succeed localFs.setVerifyChecksum(false); String str = readFile(localFs, testPath, 1024).toString(); assertTrue("read", "testing".equals(str)); }
Example 18
Source File: Credentials.java From hadoop with Apache License 2.0 | 4 votes |
public void writeTokenStorageFile(Path filename, Configuration conf) throws IOException { FSDataOutputStream os = filename.getFileSystem(conf).create(filename); writeTokenStorageToStream(os); os.close(); }
Example 19
Source File: DefaultSorter.java From tez with Apache License 2.0 | 4 votes |
/** * Handles the degenerate case where serialization fails to fit in * the in-memory buffer, so we must spill the record from collect * directly to a spill file. Consider this "losing". */ private void spillSingleRecord(final Object key, final Object value, int partition) throws IOException { long size = kvbuffer.length + partitions * APPROX_HEADER_LENGTH; FSDataOutputStream out = null; try { // create spill file final TezSpillRecord spillRec = new TezSpillRecord(partitions); final Path filename = mapOutputFile.getSpillFileForWrite(numSpills, size); spillFilePaths.put(numSpills, filename); out = rfs.create(filename); ensureSpillFilePermissions(filename, rfs); // we don't run the combiner for a single record for (int i = 0; i < partitions; ++i) { IFile.Writer writer = null; try { long segmentStart = out.getPos(); // Create a new codec, don't care! if (!sendEmptyPartitionDetails || (i == partition)) { writer = new Writer(conf, out, keyClass, valClass, codec, spilledRecordsCounter, null, false); } if (i == partition) { final long recordStart = out.getPos(); writer.append(key, value); // Note that our map byte count will not be accurate with // compression mapOutputByteCounter.increment(out.getPos() - recordStart); } long rawLength =0; long partLength =0; if (writer != null) { writer.close(); rawLength = writer.getRawLength(); partLength = writer.getCompressedLength(); } adjustSpillCounters(rawLength, partLength); // record offsets TezIndexRecord rec = new TezIndexRecord(segmentStart, rawLength, partLength); spillRec.putIndex(rec, i); writer = null; } catch (IOException e) { if (null != writer) writer.close(); throw e; } } if (totalIndexCacheMemory >= indexCacheMemoryLimit) { // create spill index file Path indexFilename = mapOutputFile.getSpillIndexFileForWrite(numSpills, partitions * MAP_OUTPUT_INDEX_RECORD_LENGTH); spillFileIndexPaths.put(numSpills, indexFilename); spillRec.writeToFile(indexFilename, conf, localFs); } else { indexCacheList.add(spillRec); totalIndexCacheMemory += spillRec.size() * MAP_OUTPUT_INDEX_RECORD_LENGTH; } ++numSpills; if (!isFinalMergeEnabled()) { numShuffleChunks.setValue(numSpills); } else if (numSpills > 1) { //Increment only when there is atleast one previous spill numAdditionalSpills.increment(1); } } finally { if (out != null) out.close(); } }
Example 20
Source File: SwiftBaseTest.java From stocator with Apache License 2.0 | 2 votes |
/** * Create and then close a file * * @param path path to create * @throws IOException on a failure */ protected static void createEmptyFile(Path path) throws IOException { FSDataOutputStream out = sFileSystem.create(path); out.close(); }