Java Code Examples for org.apache.hadoop.fs.FSDataOutputStream.write()

The following are Jave code examples for showing how to use write() of the org.apache.hadoop.fs.FSDataOutputStream class. You can vote up the examples you like. Your votes will be used in our system to get more good examples.
+ Save this method
Example 1
Project: hadoop   File: SwiftTestUtils.java   View Source Code Vote up 6 votes
/**
 * Write a file.
 * Optional flags control
 * whether file overwrite operations should be enabled
 * @param fs filesystem
 * @param path path to write to
 * @param len length of data
 * @param overwrite should the create option allow overwrites?
 * @throws IOException IO problems
 */
public static void writeDataset(FileSystem fs,
                                 Path path,
                                 byte[] src,
                                 int len,
                                 int blocksize,
                                 boolean overwrite) throws IOException {
  assertTrue(
    "Not enough data in source array to write " + len + " bytes",
    src.length >= len);
  FSDataOutputStream out = fs.create(path,
                                     overwrite,
                                     fs.getConf()
                                       .getInt(IO_FILE_BUFFER_SIZE,
                                               4096),
                                     (short) 1,
                                     blocksize);
  out.write(src, 0, len);
  out.close();
  assertFileHasLength(fs, path, len);
}
 
Example 2
Project: hadoop   File: ContractTestUtils.java   View Source Code Vote up 6 votes
/**
 * Write a file.
 * Optional flags control
 * whether file overwrite operations should be enabled
 * @param fs filesystem
 * @param path path to write to
 * @param len length of data
 * @param overwrite should the create option allow overwrites?
 * @throws IOException IO problems
 */
public static void writeDataset(FileSystem fs,
                                 Path path,
                                 byte[] src,
                                 int len,
                                 int buffersize,
                                 boolean overwrite) throws IOException {
  assertTrue(
    "Not enough data in source array to write " + len + " bytes",
    src.length >= len);
  FSDataOutputStream out = fs.create(path,
                                     overwrite,
                                     fs.getConf()
                                       .getInt(IO_FILE_BUFFER_SIZE,
                                               4096),
                                     (short) 1,
                                     buffersize);
  out.write(src, 0, len);
  out.close();
  assertFileHasLength(fs, path, len);
}
 
Example 3
Project: hadoop   File: DFSTestUtil.java   View Source Code Vote up 6 votes
public static void createFile(FileSystem fs, Path fileName, long fileLen, 
    short replFactor, long seed) throws IOException {
  if (!fs.mkdirs(fileName.getParent())) {
    throw new IOException("Mkdirs failed to create " + 
                          fileName.getParent().toString());
  }
  FSDataOutputStream out = null;
  try {
    out = fs.create(fileName, replFactor);
    byte[] toWrite = new byte[1024];
    Random rb = new Random(seed);
    long bytesToWrite = fileLen;
    while (bytesToWrite>0) {
      rb.nextBytes(toWrite);
      int bytesToWriteNext = (1024<bytesToWrite)?1024:(int)bytesToWrite;

      out.write(toWrite, 0, bytesToWriteNext);
      bytesToWrite -= bytesToWriteNext;
    }
    out.close();
    out = null;
  } finally {
    IOUtils.closeStream(out);
  }
}
 
Example 4
Project: hadoop   File: TestFiDataTransferProtocol.java   View Source Code Vote up 6 votes
/**
 * 1. create files with dfs
 * 2. write 1 byte
 * 3. close file
 * 4. open the same file
 * 5. read the 1 byte and compare results
 */
static void write1byte(String methodName) throws IOException {
  final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf
      ).numDataNodes(REPLICATION + 1).build();
  final FileSystem dfs = cluster.getFileSystem();
  try {
    final Path p = new Path("/" + methodName + "/foo");
    final FSDataOutputStream out = createFile(dfs, p);
    out.write(1);
    out.close();
    
    final FSDataInputStream in = dfs.open(p);
    final int b = in.read();
    in.close();
    Assert.assertEquals(1, b);
  }
  finally {
    dfs.close();
    cluster.shutdown();
  }
}
 
Example 5
Project: hadoop   File: TestLeaseRecovery2.java   View Source Code Vote up 6 votes
private Path createFile(final String filestr, final int size,
    final boolean triggerLeaseRenewerInterrupt)
throws IOException, InterruptedException {
  AppendTestUtil.LOG.info("filestr=" + filestr);
  Path filepath = new Path(filestr);
  FSDataOutputStream stm = dfs.create(filepath, true, BUF_SIZE,
    REPLICATION_NUM, BLOCK_SIZE);
  assertTrue(dfs.dfs.exists(filestr));

  AppendTestUtil.LOG.info("size=" + size);
  stm.write(buffer, 0, size);

  // hflush file
  AppendTestUtil.LOG.info("hflush");
  stm.hflush();

  if (triggerLeaseRenewerInterrupt) {
    AppendTestUtil.LOG.info("leasechecker.interruptAndJoin()");
    dfs.dfs.getLeaseRenewer().interruptAndJoin();
  }
  return filepath;
}
 
Example 6
Project: hadoop   File: TestSwiftFileSystemExtendedContract.java   View Source Code Vote up 6 votes
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testWriteReadFile() throws Exception {
  final Path f = new Path("/test/test");
  final FSDataOutputStream fsDataOutputStream = fs.create(f);
  final String message = "Test string";
  fsDataOutputStream.write(message.getBytes());
  fsDataOutputStream.close();
  assertExists("created file", f);
  FSDataInputStream open = null;
  try {
    open = fs.open(f);
    final byte[] bytes = new byte[512];
    final int read = open.read(bytes);
    final byte[] buffer = new byte[read];
    System.arraycopy(bytes, 0, buffer, 0, read);
    assertEquals(message, new String(buffer));
  } finally {
    fs.delete(f, false);
    IOUtils.closeStream(open);
  }
}
 
Example 7
Project: hadoop   File: FileSystemRMStateStore.java   View Source Code Vote up 6 votes
protected void writeFile(Path outputPath, byte[] data, boolean
        makeUnradableByAdmin) throws Exception {
  Path tempPath =
      new Path(outputPath.getParent(), outputPath.getName() + ".tmp");
  FSDataOutputStream fsOut = null;
  // This file will be overwritten when app/attempt finishes for saving the
  // final status.
  try {
    fsOut = fs.create(tempPath, true);
    if (makeUnradableByAdmin) {
      setUnreadableBySuperuserXattrib(tempPath);
    }
    fsOut.write(data);
    fsOut.close();
    fsOut = null;
    fs.rename(tempPath, outputPath);
  } finally {
    IOUtils.cleanup(LOG, fsOut);
  }
}
 
Example 8
Project: hadoop   File: TestRbwSpaceReservation.java   View Source Code Vote up 5 votes
@SuppressWarnings("unchecked")
@Test(timeout = 30000)
public void testRBWFileCreationError() throws Exception {

  final short replication = 1;
  startCluster(BLOCK_SIZE, replication, -1);

  final FsVolumeImpl fsVolumeImpl = (FsVolumeImpl) cluster.getDataNodes()
      .get(0).getFSDataset().getVolumes().get(0);
  final String methodName = GenericTestUtils.getMethodName();
  final Path file = new Path("/" + methodName + ".01.dat");

  // Mock BlockPoolSlice so that RBW file creation gives IOExcception
  BlockPoolSlice blockPoolSlice = Mockito.mock(BlockPoolSlice.class);
  Mockito.when(blockPoolSlice.createRbwFile((Block) Mockito.any()))
      .thenThrow(new IOException("Synthetic IO Exception Throgh MOCK"));

  Field field = FsVolumeImpl.class.getDeclaredField("bpSlices");
  field.setAccessible(true);
  Map<String, BlockPoolSlice> bpSlices = (Map<String, BlockPoolSlice>) field
      .get(fsVolumeImpl);
  bpSlices.put(fsVolumeImpl.getBlockPoolList()[0], blockPoolSlice);

  try {
    // Write 1 byte to the file
    FSDataOutputStream os = fs.create(file, replication);
    os.write(new byte[1]);
    os.hsync();
    os.close();
    fail("Expecting IOException file creation failure");
  } catch (IOException e) {
    // Exception can be ignored (expected)
  }

  // Ensure RBW space reserved is released
  assertTrue("Expected ZERO but got " + fsVolumeImpl.getReservedForRbw(),
      fsVolumeImpl.getReservedForRbw() == 0);
}
 
Example 9
Project: hadoop   File: TestSmallBlock.java   View Source Code Vote up 5 votes
private void writeFile(FileSystem fileSys, Path name) throws IOException {
  // create and write a file that contains three blocks of data
  FSDataOutputStream stm = fileSys.create(name, true, fileSys.getConf()
      .getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096),
      (short) 1, blockSize);
  byte[] buffer = new byte[fileSize];
  Random rand = new Random(seed);
  rand.nextBytes(buffer);
  stm.write(buffer);
  stm.close();
}
 
Example 10
Project: hadoop   File: TestFileAppend4.java   View Source Code Vote up 5 votes
/**
 * Test the updation of NeededReplications for the Appended Block
 */
@Test(timeout = 60000)
public void testUpdateNeededReplicationsForAppendedFile() throws Exception {
  Configuration conf = new Configuration();
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
      .build();
  DistributedFileSystem fileSystem = null;
  try {
    // create a file.
    fileSystem = cluster.getFileSystem();
    Path f = new Path("/testAppend");
    FSDataOutputStream create = fileSystem.create(f, (short) 2);
    create.write("/testAppend".getBytes());
    create.close();

    // Append to the file.
    FSDataOutputStream append = fileSystem.append(f);
    append.write("/testAppend".getBytes());
    append.close();

    // Start a new datanode
    cluster.startDataNodes(conf, 1, true, null, null);

    // Check for replications
    DFSTestUtil.waitReplication(fileSystem, f, (short) 2);
  } finally {
    if (null != fileSystem) {
      fileSystem.close();
    }
    cluster.shutdown();
  }
}
 
Example 11
Project: hadoop   File: JobControlTestUtils.java   View Source Code Vote up 5 votes
/**
 * Generates data that can be used for Job Control tests.
 * 
 * @param fs FileSystem to create data in.
 * @param dirPath Path to create the data in.
 * @throws IOException If an error occurs creating the data.
 */
static void generateData(FileSystem fs, Path dirPath) throws IOException {
  FSDataOutputStream out = fs.create(new Path(dirPath, "data.txt"));
  for (int i = 0; i < 10000; i++) {
    String line = generateRandomLine();
    out.write(line.getBytes("UTF-8"));
  }
  out.close();
}
 
Example 12
Project: hadoop   File: TestSwiftFileSystemPartitionedUploads.java   View Source Code Vote up 5 votes
/**
 * Test sticks up a very large partitioned file and verifies that
 * it comes back unchanged.
 * @throws Throwable
 */
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testManyPartitionedFile() throws Throwable {
  final Path path = new Path("/test/testManyPartitionedFile");

  int len = PART_SIZE_BYTES * 15;
  final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
  FSDataOutputStream out = fs.create(path,
                                     false,
                                     getBufferSize(),
                                     (short) 1,
                                     BLOCK_SIZE);

  out.write(src, 0, src.length);
  int expected =
    getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
  out.close();
  assertPartitionsWritten("write completed", out, expected);
  assertEquals("too few bytes written", len,
               SwiftNativeFileSystem.getBytesWritten(out));
  assertEquals("too few bytes uploaded", len,
               SwiftNativeFileSystem.getBytesUploaded(out));
  //now we verify that the data comes back. If it
  //doesn't, it means that the ordering of the partitions
  //isn't right
  byte[] dest = readDataset(fs, path, len);
  //compare data
  SwiftTestUtils.compareByteArrays(src, dest, len);
  //finally, check the data
  FileStatus[] stats = fs.listStatus(path);
  assertEquals("wrong entry count in "
               + SwiftTestUtils.dumpStats(path.toString(), stats),
               expected, stats.length);
}
 
Example 13
Project: ditb   File: TestImportTSVWithOperationAttributes.java   View Source Code Vote up 5 votes
/**
 * Run an ImportTsv job and perform basic validation on the results. Returns
 * the ImportTsv <code>Tool</code> instance so that other tests can inspect it
 * for further validation as necessary. This method is static to insure
 * non-reliance on instance's util/conf facilities.
 * 
 * @param args
 *          Any arguments to pass BEFORE inputFile path is appended.
 * @param dataAvailable
 * @return The Tool instance used to run the test.
 */
private Tool doMROnTableTest(HBaseTestingUtility util, String family, String data, String[] args,
    int valueMultiplier, boolean dataAvailable) throws Exception {
  String table = args[args.length - 1];
  Configuration conf = new Configuration(util.getConfiguration());

  // populate input file
  FileSystem fs = FileSystem.get(conf);
  Path inputPath = fs.makeQualified(new Path(util.getDataTestDirOnTestFS(table), "input.dat"));
  FSDataOutputStream op = fs.create(inputPath, true);
  op.write(Bytes.toBytes(data));
  op.close();
  LOG.debug(String.format("Wrote test data to file: %s", inputPath));

  if (conf.getBoolean(FORCE_COMBINER_CONF, true)) {
    LOG.debug("Forcing combiner.");
    conf.setInt("mapreduce.map.combine.minspills", 1);
  }

  // run the import
  List<String> argv = new ArrayList<String>(Arrays.asList(args));
  argv.add(inputPath.toString());
  Tool tool = new ImportTsv();
  LOG.debug("Running ImportTsv with arguments: " + argv);
  assertEquals(0, ToolRunner.run(conf, tool, argv.toArray(args)));

  validateTable(conf, TableName.valueOf(table), family, valueMultiplier, dataAvailable);

  if (conf.getBoolean(DELETE_AFTER_LOAD_CONF, true)) {
    LOG.debug("Deleting test subdirectory");
    util.cleanupDataTestDirOnTestFS(table);
  }
  return tool;
}
 
Example 14
Project: hadoop   File: TestListFiles.java   View Source Code Vote up 5 votes
private static void writeFile(FileSystem fileSys, Path name, int fileSize)
throws IOException {
  // Create and write a file that contains three blocks of data
  FSDataOutputStream stm = fileSys.create(name);
  byte[] buffer = new byte[fileSize];
  Random rand = new Random(seed);
  rand.nextBytes(buffer);
  stm.write(buffer);
  stm.close();
}
 
Example 15
Project: hadoop   File: Application.java   View Source Code Vote up 5 votes
private void writePasswordToLocalFile(String localPasswordFile,
    byte[] password, JobConf conf) throws IOException {
  FileSystem localFs = FileSystem.getLocal(conf);
  Path localPath = new Path(localPasswordFile);
  FSDataOutputStream out = FileSystem.create(localFs, localPath,
      new FsPermission("400"));
  out.write(password);
  out.close();
}
 
Example 16
Project: hadoop   File: TestShortCircuitLocalRead.java   View Source Code Vote up 4 votes
@Test(timeout=10000)
public void testSkipWithVerifyChecksum() throws IOException {
  int size = blockSize;
  Configuration conf = new Configuration();
  conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, true);
  conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY, false);
  conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY,
      "/tmp/testSkipWithVerifyChecksum._PORT");
  DomainSocket.disableBindPathValidation();
  if (simulatedStorage) {
    SimulatedFSDataset.setFactory(conf);
  }
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
      .format(true).build();
  FileSystem fs = cluster.getFileSystem();
  try {
    // check that / exists
    Path path = new Path("/");
    assertTrue("/ should be a directory", fs.getFileStatus(path)
        .isDirectory() == true);
    
    byte[] fileData = AppendTestUtil.randomBytes(seed, size*3);
    // create a new file in home directory. Do not close it.
    Path file1 = new Path("filelocal.dat");
    FSDataOutputStream stm = createFile(fs, file1, 1);

    // write to file
    stm.write(fileData);
    stm.close();
    
    // now test the skip function
    FSDataInputStream instm = fs.open(file1);
    byte[] actual = new byte[fileData.length];
    // read something from the block first, otherwise BlockReaderLocal.skip()
    // will not be invoked
    int nread = instm.read(actual, 0, 3);
    long skipped = 2*size+3;
    instm.seek(skipped);
    nread = instm.read(actual, (int)(skipped + nread), 3);
    instm.close();
      
  } finally {
    fs.close();
    cluster.shutdown();
  }
}
 
Example 17
Project: hadoop   File: TestDataTransferProtocol.java   View Source Code Vote up 4 votes
void createFile(FileSystem fs, Path path, int fileLen) throws IOException {
  byte [] arr = new byte[fileLen];
  FSDataOutputStream out = fs.create(path);
  out.write(arr);
  out.close();
}
 
Example 18
Project: hadoop   File: TestFileAppend4.java   View Source Code Vote up 4 votes
/**
 * Test that an append with no locations fails with an exception
 * showing insufficient locations.
 */
@Test(timeout = 60000)
public void testAppendInsufficientLocations() throws Exception {
  Configuration conf = new Configuration();

  // lower heartbeat interval for fast recognition of DN
  conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,
      1000);
  conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
  conf.setInt(DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, 3000);

  cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4)
      .build();
  DistributedFileSystem fileSystem = null;
  try {
    // create a file with replication 3
    fileSystem = cluster.getFileSystem();
    Path f = new Path("/testAppend");
    FSDataOutputStream create = fileSystem.create(f, (short) 2);
    create.write("/testAppend".getBytes());
    create.close();

    // Check for replications
    DFSTestUtil.waitReplication(fileSystem, f, (short) 2);

    // Shut down all DNs that have the last block location for the file
    LocatedBlocks lbs = fileSystem.dfs.getNamenode().
        getBlockLocations("/testAppend", 0, Long.MAX_VALUE);
    List<DataNode> dnsOfCluster = cluster.getDataNodes();
    DatanodeInfo[] dnsWithLocations = lbs.getLastLocatedBlock().
        getLocations();
    for( DataNode dn : dnsOfCluster) {
      for(DatanodeInfo loc: dnsWithLocations) {
        if(dn.getDatanodeId().equals(loc)){
          dn.shutdown();
          DFSTestUtil.waitForDatanodeDeath(dn);
        }
      }
    }

    // Wait till 0 replication is recognized
    DFSTestUtil.waitReplication(fileSystem, f, (short) 0);

    // Append to the file, at this state there are 3 live DNs but none of them
    // have the block.
    try{
      fileSystem.append(f);
      fail("Append should fail because insufficient locations");
    } catch (IOException e){
      LOG.info("Expected exception: ", e);
    }
    FSDirectory dir = cluster.getNamesystem().getFSDirectory();
    final INodeFile inode = INodeFile.
        valueOf(dir.getINode("/testAppend"), "/testAppend");
    assertTrue("File should remain closed", !inode.isUnderConstruction());
  } finally {
    if (null != fileSystem) {
      fileSystem.close();
    }
    cluster.shutdown();
  }
}
 
Example 19
Project: hadoop   File: TestFileAppend.java   View Source Code Vote up 4 votes
/** Tests appending after soft-limit expires. */
@Test
public void testAppend2AfterSoftLimit() throws Exception {
  Configuration conf = new HdfsConfiguration();
  conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
  //Set small soft-limit for lease
  final long softLimit = 1L;
  final long hardLimit = 9999999L;

  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
      .build();
  cluster.setLeasePeriod(softLimit, hardLimit);
  cluster.waitActive();

  DistributedFileSystem fs = cluster.getFileSystem();
  DistributedFileSystem fs2 = new DistributedFileSystem();
  fs2.initialize(fs.getUri(), conf);

  final Path testPath = new Path("/testAppendAfterSoftLimit");
  final byte[] fileContents = AppendTestUtil.initBuffer(32);

  // create a new file without closing
  FSDataOutputStream out = fs.create(testPath);
  out.write(fileContents);

  //Wait for > soft-limit
  Thread.sleep(250);

  try {
    FSDataOutputStream appendStream2 = fs2.append(testPath,
        EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null);
    appendStream2.write(fileContents);
    appendStream2.close();
    assertEquals(fileContents.length, fs.getFileStatus(testPath).getLen());
    // make sure we now have 1 block since the first writer was revoked
    LocatedBlocks blks = fs.getClient().getLocatedBlocks(testPath.toString(),
        0L);
    assertEquals(1, blks.getLocatedBlocks().size());
    for (LocatedBlock blk : blks.getLocatedBlocks()) {
      assertEquals(fileContents.length, blk.getBlockSize());
    }
  } finally {
    fs.close();
    fs2.close();
    cluster.shutdown();
  }
}
 
Example 20
Project: hadoop   File: TestQuotasWithHA.java   View Source Code Vote up 4 votes
/**
 * Test that quotas are properly tracked by the standby through
 * create, append, delete.
 */
@Test(timeout=60000)
public void testQuotasTrackedOnStandby() throws Exception {
  fs.mkdirs(TEST_DIR);
  DistributedFileSystem dfs = (DistributedFileSystem)fs;
  dfs.setQuota(TEST_DIR, NS_QUOTA, DS_QUOTA);
  long expectedSize = 3 * BLOCK_SIZE + BLOCK_SIZE/2;
  DFSTestUtil.createFile(fs, TEST_FILE, expectedSize, (short)1, 1L);

  HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
  ContentSummary cs = nn1.getRpcServer().getContentSummary(TEST_DIR_STR);
  assertEquals(NS_QUOTA, cs.getQuota());
  assertEquals(DS_QUOTA, cs.getSpaceQuota());
  assertEquals(expectedSize, cs.getSpaceConsumed());
  assertEquals(1, cs.getDirectoryCount());
  assertEquals(1, cs.getFileCount());

  // Append to the file and make sure quota is updated correctly.
  FSDataOutputStream stm = fs.append(TEST_FILE);
  try {
    byte[] data = new byte[(int) (BLOCK_SIZE * 3 / 2)];
    stm.write(data);
    expectedSize += data.length;
  } finally {
    IOUtils.closeStream(stm);
  }
  
  HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
  cs = nn1.getRpcServer().getContentSummary(TEST_DIR_STR);
  assertEquals(NS_QUOTA, cs.getQuota());
  assertEquals(DS_QUOTA, cs.getSpaceQuota());
  assertEquals(expectedSize, cs.getSpaceConsumed());
  assertEquals(1, cs.getDirectoryCount());
  assertEquals(1, cs.getFileCount());

  
  fs.delete(TEST_FILE, true);
  expectedSize = 0;
  HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
  cs = nn1.getRpcServer().getContentSummary(TEST_DIR_STR);
  assertEquals(NS_QUOTA, cs.getQuota());
  assertEquals(DS_QUOTA, cs.getSpaceQuota());
  assertEquals(expectedSize, cs.getSpaceConsumed());
  assertEquals(1, cs.getDirectoryCount());
  assertEquals(0, cs.getFileCount());
}