Java Code Examples for org.apache.hadoop.fs.FSDataOutputStream#sync()
The following examples show how to use
org.apache.hadoop.fs.FSDataOutputStream#sync() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestHftpFileSystem.java From RDFS with Apache License 2.0 | 6 votes |
/** * Scenario: Read an under construction file using hftp. * * Expected: Hftp should be able to read the latest byte after the file * has been hdfsSynced (but not yet closed). * * @throws IOException */ public void testConcurrentRead() throws IOException { // Write a test file. FSDataOutputStream out = hdfs.create(TEST_FILE, true); out.writeBytes("123"); out.sync(); // sync but not close // Try read using hftp. FSDataInputStream in = hftpFs.open(TEST_FILE); assertEquals('1', in.read()); assertEquals('2', in.read()); assertEquals('3', in.read()); in.close(); // Try seek and read. in = hftpFs.open(TEST_FILE); in.seek(2); assertEquals('3', in.read()); in.close(); out.close(); }
Example 2
Source File: TestAvatarSetQuota.java From RDFS with Apache License 2.0 | 6 votes |
@Test public void testAvatarSetQuota() throws Exception { String test = "/testAvatarSetQuota"; DFSTestUtil util = new DFSTestUtil(test, 10, 10, 1024); util.createFiles(fs, test); FSDataOutputStream out = fs.create(new Path(test + "/abc")); byte[] buffer = new byte[10 * 1024]; Random r = new Random(); r.nextBytes(buffer); out.write(buffer); out.sync(); ((DistributedFileSystem) fs).setQuota(new Path(test), 5, -1); out.close(); cluster.getStandbyAvatar(0).avatar.quiesceStandby(-1); }
Example 3
Source File: TestLeaseRecovery2.java From RDFS with Apache License 2.0 | 6 votes |
private Path createFile(DistributedFileSystem dfs, int size, boolean triggerSoftLease) throws IOException, InterruptedException { // create a random file name String filestr = "/foo" + AppendTestUtil.nextInt(); System.out.println("filestr=" + filestr); Path filepath = new Path(filestr); FSDataOutputStream stm = dfs.create(filepath, true, bufferSize, REPLICATION_NUM, BLOCK_SIZE); assertTrue(dfs.dfs.exists(filestr)); // write random number of bytes into it. System.out.println("size=" + size); stm.write(buffer, 0, size); // sync file AppendTestUtil.LOG.info("sync"); stm.sync(); if (triggerSoftLease) { AppendTestUtil.LOG.info("leasechecker.interruptAndJoin()"); dfs.dfs.leasechecker.interruptAndJoin(); } return filepath; }
Example 4
Source File: TestLeaseRecovery3.java From RDFS with Apache License 2.0 | 6 votes |
private Path createFile(DistributedFileSystem dfs, int size ) throws IOException, InterruptedException { // create a random file name String filestr = "/foo" + AppendTestUtil.nextInt(); System.out.println("filestr=" + filestr); Path filepath = new Path(filestr); FSDataOutputStream stm = dfs.create(filepath, true, bufferSize, REPLICATION_NUM, BLOCK_SIZE); assertTrue(dfs.dfs.exists(filestr)); // write random number of bytes into it. System.out.println("size=" + size); stm.write(buffer, 0, size); // sync file AppendTestUtil.LOG.info("sync"); stm.sync(); // write another piece of data to file. This piece of data // is not yet synced stm.write(buffer, 0, size); return filepath; }
Example 5
Source File: TestFileCreationClient.java From hadoop-gpu with Apache License 2.0 | 6 votes |
public void run() { FSDataOutputStream out = null; int i = 0; try { out = fs.create(filepath); for(; running; i++) { System.out.println(getName() + " writes " + i); out.write(i); out.sync(); sleep(100); } } catch(Exception e) { System.out.println(getName() + " dies: e=" + e); } finally { System.out.println(getName() + ": i=" + i); IOUtils.closeStream(out); } }
Example 6
Source File: TestStuckDataNode.java From RDFS with Apache License 2.0 | 5 votes |
/** This creates a slow writer and check to see * if pipeline heartbeats work fine */ public void testStuckDataNode() throws Exception { final int DATANODE_NUM = 3; Configuration conf = new Configuration(); final int timeout = 8000; conf.setInt("dfs.socket.timeout",timeout); final Path p = new Path("/pipelineHeartbeat/foo"); System.out.println("p=" + p); MiniDFSCluster cluster = new MiniDFSCluster(conf, DATANODE_NUM, true, null); DistributedFileSystem fs = (DistributedFileSystem)cluster.getFileSystem(); DataNodeMetrics metrics = cluster.getDataNodes().get(0).myMetrics; MetricsTimeVaryingLong spyBytesWritten = spy(metrics.bytesWritten); DelayAnswer delayAnswer = new DelayAnswer(); doAnswer(delayAnswer).when(spyBytesWritten).inc(anyInt()); metrics.bytesWritten = spyBytesWritten; try { // create a new file. FSDataOutputStream stm = fs.create(p); stm.write(1); stm.sync(); stm.write(2); stm.close(); // verify that entire file is good FSDataInputStream in = fs.open(p); assertEquals(1, in.read()); assertEquals(2, in.read()); in.close(); } finally { fs.close(); cluster.shutdown(); } }
Example 7
Source File: TestAvatarLease.java From RDFS with Apache License 2.0 | 5 votes |
@Test public void testLeaseAfterFailover() throws Exception { String fileName = "/testLeaseAfterFailover"; FSDataOutputStream out = fs.create(new Path(fileName)); byte[] buffer = new byte[1024]; random.nextBytes(buffer); out.write(buffer); out.sync(); FSNamesystem primary = cluster.getPrimaryAvatar(0).avatar.namesystem; // Prevents lease recovery to work. cluster.shutDownDataNodes(); // Expire the lease. primary.leaseManager.setLeasePeriod(0, 0); primary.leaseManager.checkLeases(); cluster.killPrimary(); cluster.restartDataNodes(false); AvatarNode standbyAvatar = cluster.getStandbyAvatar(0).avatar; standbyAvatar.setAvatar(Avatar.ACTIVE); String lease = standbyAvatar.namesystem.leaseManager .getLeaseByPath(fileName).getHolder(); assertEquals(HdfsConstants.NN_RECOVERY_LEASEHOLDER, lease); }
Example 8
Source File: TestStandbySafeMode.java From RDFS with Apache License 2.0 | 5 votes |
@Test public void testLeaseExpiry() throws Exception { setUp(true); h.setIgnoreDatanodes(false); LeaseManager leaseManager = cluster.getStandbyAvatar(0).avatar.namesystem.leaseManager; // Set low lease periods. leaseManager.setLeasePeriod(LEASE_PERIOD, LEASE_PERIOD); String src = "/testLeaseExpiry"; // Create some data. FSDataOutputStream out = fs.create(new Path(src)); byte[] buffer = new byte[BLOCK_SIZE * 2]; random.nextBytes(buffer); out.write(buffer); out.sync(); // Wait for the hard lease time to expire. Thread.sleep(LEASE_PERIOD * 2); cluster.failOver(); LOG.info("Failover done"); // Renew lease. ((DistributedFileSystem)fs).getClient().leasechecker.renew(); LOG.info("Lease renewal done"); // Wait to see whether lease expires. long start = System.currentTimeMillis(); while (System.currentTimeMillis() - start < MAX_WAIT_TIME && leaseManager.getLeaseByPath(src) != null) { Thread.sleep(1000); } LOG.info("Wait for lease done"); // Now try to write to the file. out.write(buffer); out.sync(); }
Example 9
Source File: TestOpenFilesInfo.java From RDFS with Apache License 2.0 | 5 votes |
private static void createOpenFiles(int nFiles, String prefix) throws Exception { for (int i = 0; i < nFiles; i++) { FSDataOutputStream out = fs.create(new Path("/" + prefix + random.nextInt())); byte[] buffer = new byte[random.nextInt(MAX_FILE_SIZE)]; random.nextBytes(buffer); out.write(buffer); out.sync(); } }
Example 10
Source File: TestFileStatusExtended.java From RDFS with Apache License 2.0 | 5 votes |
@Test public void testFileUnderConstruction() throws Exception { String fileName = "/testFileUnderConstruction"; FSDataOutputStream out = fs.create(new Path(fileName)); byte [] buffer = new byte[BLOCK_SIZE * 5 + 256]; random.nextBytes(buffer); out.write(buffer); out.sync(); NameNode nn = cluster.getNameNode(); List<FileStatusExtended> stats = nn.getRandomFiles(1); assertEquals(1, stats.size()); assertEquals(((DistributedFileSystem) fs).getClient().clientName, stats .get(0).getHolder()); }
Example 11
Source File: TestDFSConcurrentFileOperations.java From RDFS with Apache License 2.0 | 5 votes |
public void testLeaseRecoveryOnTrashedFile() throws Exception { Configuration conf = new Configuration(); conf.setLong("dfs.block.size", blockSize); init(conf); String src = "/file-1"; String dst = "/file-2"; Path srcPath = new Path(src); Path dstPath = new Path(dst); FSDataOutputStream fos = fs.create(srcPath); AppendTestUtil.write(fos, 0, writeSize); fos.sync(); // renaming a file out from under a client will cause close to fail // and result in the lease remaining while the blocks are finalized on // the DNs fs.rename(srcPath, dstPath); try { fos.close(); fail("expected IOException"); } catch (IOException e) { //expected } FileSystem fs2 = AppendTestUtil.createHdfsWithDifferentUsername(conf); AppendTestUtil.recoverFile(cluster, fs2, dstPath); AppendTestUtil.check(fs2, dstPath, writeSize); }
Example 12
Source File: TestParallelRBW.java From RDFS with Apache License 2.0 | 5 votes |
private void createFile(FileSystem fs, FSDataOutputStream out, String fileName, int fileLen) throws IOException { Random random = new Random(fileName.hashCode()); byte buffer[] = new byte[fileLen]; random.nextBytes(buffer); out.write(buffer); out.sync(); ((DFSOutputStream) out.getWrappedStream()).abortForTests(); }
Example 13
Source File: TestDatanodeUpgrade.java From RDFS with Apache License 2.0 | 5 votes |
private void createFile(FileSystem fs, FSDataOutputStream out, String fileName, int fileLen) throws IOException { Random random = new Random(fileName.hashCode()); byte buffer[] = new byte[fileLen]; random.nextBytes(buffer); out.write(buffer); out.sync(); ((DFSOutputStream) out.getWrappedStream()).abortForTests(); }
Example 14
Source File: TestFileAppend.java From RDFS with Apache License 2.0 | 4 votes |
/** This creates a slow writer and check to see * if pipeline heartbeats work fine */ public void testPipelineHeartbeat() throws Exception { final int DATANODE_NUM = 2; final int fileLen = 6; Configuration conf = new Configuration(); final int timeout = 2000; conf.setInt("dfs.socket.timeout",timeout); final Path p = new Path("/pipelineHeartbeat/foo"); System.out.println("p=" + p); MiniDFSCluster cluster = new MiniDFSCluster(conf, DATANODE_NUM, true, null); DistributedFileSystem fs = (DistributedFileSystem)cluster.getFileSystem(); initBuffer(fileLen); try { // create a new file. FSDataOutputStream stm = createFile(fs, p, DATANODE_NUM); stm.write(fileContents, 0, 1); Thread.sleep(timeout); stm.sync(); System.out.println("Wrote 1 byte and hflush " + p); // write another byte Thread.sleep(timeout); stm.write(fileContents, 1, 1); stm.sync(); stm.write(fileContents, 2, 1); Thread.sleep(timeout); stm.sync(); stm.write(fileContents, 3, 1); Thread.sleep(timeout); stm.write(fileContents, 4, 1); stm.sync(); stm.write(fileContents, 5, 1); Thread.sleep(timeout); stm.close(); // verify that entire file is good checkFullFile(fs, p); } finally { fs.close(); cluster.shutdown(); } }
Example 15
Source File: TestDFSIsUnderConstruction.java From RDFS with Apache License 2.0 | 4 votes |
public void testSecondLastBlockNotReceived() throws Exception { String fileName = "/testSecondLastBlockNotReceived"; Path growingFile = new Path(fileName); FSDataInputStream fis = null; FSDataOutputStream fos = fs.create(growingFile, false, 1024, (short)1, 1024); try { int fileLength = 2096; AppendTestUtil.write(fos, 0, fileLength); fos.sync(); fis = fs.open(growingFile); for (int i = 0; i < fileLength; i++) { fis.read(); } fis.close(); FSNamesystem fsns = cluster.getNameNode().namesystem; INode[] inodes = fsns.dir.getExistingPathINodes(fileName); BlockInfo[] bis = ((INodeFile) (inodes[inodes.length - 1])).getBlocks(); bis[bis.length - 2].setNumBytes(1); try { fis = fs.open(growingFile); TestCase.fail(); } catch (IOException e) { } bis[bis.length - 2].setNumBytes(1024); bis[bis.length - 1].setNumBytes(1); fis = fs.open(growingFile); for (int i = 0; i < fileLength; i++) { fis.read(); } } finally { if (fos != null) { fos.close(); } if (fis != null) { fis.close(); } } }
Example 16
Source File: TestLeaseRecovery3.java From RDFS with Apache License 2.0 | 4 votes |
public void testBlockSynchronization() throws Exception { final long softLease = 1000; final long hardLease = 60 * 60 *1000; final long packetSize = BLOCK_SIZE/2; conf.setLong("dfs.block.size", BLOCK_SIZE); conf.setInt("dfs.heartbeat.interval", 1); conf.setInt("dfs.write.packet.size", (int) packetSize); MiniDFSCluster cluster = null; byte[] actual = new byte[FILE_SIZE]; FSDataOutputStream stm =null; try { cluster = new MiniDFSCluster(conf, 5, true, null); cluster.waitActive(); //create a file, write 2 blocks, then do a sync and then write //another block. The last block is not synced. DistributedFileSystem dfs = (DistributedFileSystem)cluster.getFileSystem(); int fileSize = 2 * (int)BLOCK_SIZE; String filestr = "/foo.dhruba"; System.out.println("Creating file " + filestr); Path filepath = new Path(filestr); stm = dfs.create(filepath, true, bufferSize, REPLICATION_NUM, BLOCK_SIZE); assertTrue(dfs.dfs.exists(filestr)); // write 2 blocks worth of data System.out.println("Writing data " + fileSize); stm.write(buffer, 0, fileSize); stm.sync(); AppendTestUtil.LOG.info("sync done"); // write another piece of data to file. This piece of data // is not yet synced System.out.println("Writing data again " + (packetSize + 1)); stm.write(buffer, 0, (int)(packetSize + 1)); // invoke recoverLease from a different client. This should truncate the // file to 2 blocks. recoverLease(filepath, null); // verify that file is 2 blocks long. verifyFile(null, filepath, actual, fileSize); } finally { try { if (stm != null) { stm.close(); } if (cluster != null) {cluster.getFileSystem().close(); cluster.shutdown();} } catch (Exception e) { // ignore } } }
Example 17
Source File: TestRbwReportSafeMode.java From RDFS with Apache License 2.0 | 4 votes |
@Test public void testRBW() throws Exception { String fileName = "/testRBW"; FSDataOutputStream out = fs.create(new Path(fileName)); // Create RBW. byte[] buffer = new byte[1024 * 10 + 100]; Random r = new Random(); r.nextBytes(buffer); out.write(buffer); out.sync(); cluster.restartNameNode(0, new String[] {}, false); ((DFSOutputStream) out.getWrappedStream()).abortForTests(); // Send multiple RBW reports. waitForBlocks(); cluster.restartDataNodes(); Thread.sleep(10000); System.out.println("Restarts done"); FSNamesystem namesystem = cluster.getNameNode().namesystem; long totalBlocks = namesystem.getBlocksTotal(); long safeBlocks = namesystem.getSafeBlocks(); long startTime = System.currentTimeMillis(); while (totalBlocks != safeBlocks && (System.currentTimeMillis() - startTime < 15000)) { Thread.sleep(1000); System.out.println("Waiting for blocks, Total : " + totalBlocks + " Safe : " + safeBlocks); totalBlocks = namesystem.getBlocksTotal(); safeBlocks = namesystem.getSafeBlocks(); } assertEquals(11, totalBlocks); assertEquals(totalBlocks, safeBlocks); for (DataNode dn : cluster.getDataNodes()) { assertEquals(1, dn.data.getBlocksBeingWrittenReport(cluster.getNameNode() .getNamespaceID()).length); } }
Example 18
Source File: GenWriterThread.java From RDFS with Apache License 2.0 | 4 votes |
public void run() { try { fs.mkdirs(outputPath); long endTime = System.currentTimeMillis() + rtc.max_time; long lastRollTime = System.currentTimeMillis(); long lastSyncTime = System.currentTimeMillis(); long currentId = 0; FSDataOutputStream out = null; while (System.currentTimeMillis() < endTime) { Path fileName = new Path(outputPath, "part" + currentId); try { out = fs.create(fileName, (short)3); dc.openFile(); long size = 0; while (true) { rb.nextBytes(buffer); dc.getFileChecksum().update(buffer, 0, rtc.buffer_size); tb.getTokens(rtc.buffer_size); out.write(buffer, 0, rtc.buffer_size); size += rtc.buffer_size; if (rtc.sync_interval > 0 && System.currentTimeMillis() - lastSyncTime > rtc.sync_interval) { // Sync the file out.sync(); LOG.info("file " + fileName + " is synced"); lastSyncTime = System.currentTimeMillis() + rb.nextInt((int)rtc.sync_interval); } if (System.currentTimeMillis() > endTime || rtc.roll_interval > 0 && System.currentTimeMillis() - lastRollTime > rtc.roll_interval) { // Roll the file out.close(); out = null; currentId++; files_processed++; processed_size += size; LOG.info("file " + fileName + " is closed with " + size + " bytes"); lastRollTime = System.currentTimeMillis() + rb.nextInt((int)rtc.roll_interval); break; } } } catch (Exception e) { LOG.error("Error in writing file: " + fileName, e); this.errors.add(e); } finally { IOUtils.closeStream(out); dc.closeFile(); } } LOG.info("Checksum of files under dir " + outputPath + " is " + dc.getDirectoryChecksum()); LOG.info("Thread " + name + "_" + id + " is done."); } catch (Exception ioe) { LOG.error("Error:", ioe); this.errors.add(ioe); } }
Example 19
Source File: TestAvatarDataNodeRBW.java From RDFS with Apache License 2.0 | 4 votes |
private void createRBWFile(String fileName) throws IOException { FSDataOutputStream out = dafs.create(new Path(fileName)); random.nextBytes(buffer); out.write(buffer); out.sync(); }
Example 20
Source File: TestFileConcurrentReader.java From RDFS with Apache License 2.0 | 4 votes |
private void writeFileAndSync(FSDataOutputStream stm, int size) throws IOException { byte[] buffer = DFSTestUtil.generateSequentialBytes(0, size); stm.write(buffer, 0, size); stm.sync(); }