Java Code Examples for org.apache.hadoop.fs.FileSystem#append()

The following examples show how to use org.apache.hadoop.fs.FileSystem#append() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
HadoopRecoverableFsDataOutputStream(
		FileSystem fs,
		HadoopFsRecoverable recoverable) throws IOException {

	ensureTruncateInitialized();

	this.fs = checkNotNull(fs);
	this.targetFile = checkNotNull(recoverable.targetFile());
	this.tempFile = checkNotNull(recoverable.tempFile());

	safelyTruncateFile(fs, tempFile, recoverable);

	out = fs.append(tempFile);

	// sanity check
	long pos = out.getPos();
	if (pos != recoverable.offset()) {
		IOUtils.closeQuietly(out);
		throw new IOException("Truncate failed: " + tempFile +
				" (requested=" + recoverable.offset() + " ,size=" + pos + ')');
	}
}
 
Example 2
HadoopRecoverableFsDataOutputStream(
		FileSystem fs,
		HadoopFsRecoverable recoverable) throws IOException {

	ensureTruncateInitialized();

	this.fs = checkNotNull(fs);
	this.targetFile = checkNotNull(recoverable.targetFile());
	this.tempFile = checkNotNull(recoverable.tempFile());

	safelyTruncateFile(fs, tempFile, recoverable);

	out = fs.append(tempFile);

	// sanity check
	long pos = out.getPos();
	if (pos != recoverable.offset()) {
		IOUtils.closeQuietly(out);
		throw new IOException("Truncate failed: " + tempFile +
				" (requested=" + recoverable.offset() + " ,size=" + pos + ')');
	}
}
 
Example 3
/** {@inheritDoc} */
@Override public OutputStream append(IgfsPath path, int bufSize, boolean create,
    @Nullable Map<String, String> props) {
    try {
        Path hadoopPath = convert(path);

        FileSystem fs = fileSystemForUser();

        if (create && !fs.exists(hadoopPath))
            return fs.create(hadoopPath, false, bufSize);
        else
            return fs.append(convert(path), bufSize);
    }
    catch (IOException e) {
        throw handleSecondaryFsError(e, "Failed to append file [path=" + path + ", bufSize=" + bufSize + "]");
    }
}
 
Example 4
Source Project: big-c   File: RetriableFileCopyCommand.java    License: Apache License 2.0 6 votes vote down vote up
private long copyToFile(Path targetPath, FileSystem targetFS,
    FileStatus sourceFileStatus, long sourceOffset, Mapper.Context context,
    EnumSet<FileAttribute> fileAttributes, final FileChecksum sourceChecksum)
    throws IOException {
  FsPermission permission = FsPermission.getFileDefault().applyUMask(
      FsPermission.getUMask(targetFS.getConf()));
  final OutputStream outStream;
  if (action == FileAction.OVERWRITE) {
    final short repl = getReplicationFactor(fileAttributes, sourceFileStatus,
        targetFS, targetPath);
    final long blockSize = getBlockSize(fileAttributes, sourceFileStatus,
        targetFS, targetPath);
    FSDataOutputStream out = targetFS.create(targetPath, permission,
        EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
        BUFFER_SIZE, repl, blockSize, context,
        getChecksumOpt(fileAttributes, sourceChecksum));
    outStream = new BufferedOutputStream(out);
  } else {
    outStream = new BufferedOutputStream(targetFS.append(targetPath,
        BUFFER_SIZE));
  }
  return copyBytes(sourceFileStatus, sourceOffset, outStream, BUFFER_SIZE,
      context);
}
 
Example 5
Source Project: Transwarp-Sample-Code   File: HDFSSequenceFile.java    License: MIT License 6 votes vote down vote up
protected void open(Path dstPath, CompressionCodec codeC,
    CompressionType compType, Configuration conf, FileSystem hdfs)
        throws IOException {
  if(useRawLocalFileSystem) {
    if(hdfs instanceof LocalFileSystem) {
      hdfs = ((LocalFileSystem)hdfs).getRaw();
    } else {
      logger.warn("useRawLocalFileSystem is set to true but file system " +
          "is not of type LocalFileSystem: " + hdfs.getClass().getName());
    }
  }
  if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile
          (dstPath)) {
    outStream = hdfs.append(dstPath);
  } else {
    outStream = hdfs.create(dstPath);
  }
  writer = SequenceFile.createWriter(conf, outStream,
      serializer.getKeyClass(), serializer.getValueClass(), compType, codeC);

  registerCurrentStream(outStream, hdfs, dstPath);
}
 
Example 6
@Override
public Void run() throws Exception {
    String effectiveDirPath = "/user/" + (serviceAsNamespace ? "" : (hdfsUser + "/")) + filePath;
    FileSystem fileSystem = fsGetter.get();
    Path path = new Path(effectiveDirPath);
    FSDataOutputStream out = fileSystem.append(path);

    if (out == null) {
        fileSystem.close();
        throw new CygnusPersistenceError("The /user/" + (serviceAsNamespace ? "" : (hdfsUser + "/"))
                + filePath + " file could not be created in HDFS");
    } // if

    BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out));
    writer.append(data + "\n");
    writer.close();
    fileSystem.close();
    return null;
}
 
Example 7
@Test
public void append_shouldFail_whenFileDoesNotExist() throws Exception {
  URI path = gcsFsIHelper.getUniqueObjectUri("append_shouldFail_whenFileDoesNotExist");
  Path hadoopPath = new Path(path);

  FileSystem ghfs = GoogleHadoopFileSystemIntegrationHelper.createGhfs(path, getTestConfig());

  // Test appending three 9-character strings to existing object using 20 bytes buffer size
  FSDataOutputStream fsos = ghfs.append(hadoopPath, 20, /* progress= */ () -> {});
  fsos.write("_append-1".getBytes(UTF_8));

  assertThrows(GoogleJsonResponseException.class, fsos::hsync);

  assertThrows(NullPointerException.class, fsos::close);

  // Validate that file wasn't created
  assertThat(ghfs.exists(hadoopPath)).isFalse();
}
 
Example 8
Source Project: big-c   File: TestEncryptedTransfer.java    License: Apache License 2.0 5 votes vote down vote up
private static void writeTestDataToFile(FileSystem fs) throws IOException {
  OutputStream out = null;
  if (!fs.exists(TEST_PATH)) {
    out = fs.create(TEST_PATH);
  } else {
    out = fs.append(TEST_PATH);
  }
  out.write(PLAIN_TEXT.getBytes());
  out.close();
}
 
Example 9
Source Project: RDFS   File: TestFileAppend4.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Test that a file is not considered complete when it only has in-progress
 * blocks. This ensures that when a block is appended to, it is converted
 * back into the right kind of "in progress" state.
 */
public void testNotPrematurelyComplete() throws Exception {
  LOG.info("START");
  cluster = new MiniDFSCluster(conf, 3, true, null);
  FileSystem fs1 = cluster.getFileSystem();
  try {
    int halfBlock = (int)BLOCK_SIZE/2;
    short rep = 3; // replication
    assertTrue(BLOCK_SIZE%4 == 0);

    file1 = new Path("/delayedReceiveBlock");

    // write 1/2 block & close
    stm = fs1.create(file1, true, (int)BLOCK_SIZE*2, rep, BLOCK_SIZE);
    AppendTestUtil.write(stm, 0, halfBlock);
    stm.close();

    NameNode nn = cluster.getNameNode();
    LOG.info("======== Appending");
    stm = fs1.append(file1);
    LOG.info("======== Writing");
    AppendTestUtil.write(stm, 0, halfBlock/2);
    LOG.info("======== Checking progress");
    assertFalse(NameNodeAdapter.checkFileProgress(nn.namesystem, "/delayedReceiveBlock", true));
    LOG.info("======== Closing");
    stm.close();

  } catch (Throwable e) {
    e.printStackTrace();
    throw new IOException(e);
  } finally {
    LOG.info("======== Cleaning up");
    fs1.close();
    cluster.shutdown();
  }
}
 
Example 10
public static void main(String[] args) throws IOException {
    Configuration conf = new Configuration();
    
//    conf.addResource(new Path("/etc/hadoop/conf/hdfs-site.xml"));
//    conf.addResource(new Path("/etc/hadoop/conf/core-site.xml"));
//    conf.addResource(new Path("/etc/hadoop/conf/yarn-site.xml"));
//    conf.addResource(new Path("/etc/hadoop/conf/mapred-site.xml"));
    String hostname="";
    try {
      hostname = InetAddress.getLocalHost().getHostName();
    } catch (UnknownHostException e) {
      e.printStackTrace();
    }
    
    String uri = "hdfs://"+hostname+":8020";
    System.out.println(uri);
    
    
    System.out.println("STARTING");
    String filepath = uri+"/user/yarn/"+UUID.randomUUID().toString();
    System.out.println(filepath);
    Path path = new Path(filepath);
    
    System.out.println("FIRST WRITE");
    FileSystem fs = FileSystem.get(URI.create(uri), conf);
    FSDataOutputStream os = fs.create(path);
    os.write("hellothere\n".getBytes());
    os.close();
    
    System.out.println("APPEND");
    FileSystem fs2 = FileSystem.get(URI.create(uri), conf);
    System.out.println("GETTING OUTPUT STREAM");
    FSDataOutputStream os2 = fs2.append(path);
    System.out.println("DOING THE SECOND WRITE");
    os2.write("Hope this works\n".getBytes());
    System.out.println("CLOSING SECOND WRITE");
    os2.close();
    System.out.println("DONE");
    
  }
 
Example 11
/**
 * Test which randomly alternates between appending with
 * CRC32 and with CRC32C, crossing several block boundaries.
 * Then, checks that all of the data can be read back correct.
 */
@Test(timeout=RANDOM_TEST_RUNTIME*2)
public void testAlgoSwitchRandomized() throws IOException {
  FileSystem fsWithCrc32 = createFsWithChecksum("CRC32", 512);
  FileSystem fsWithCrc32C = createFsWithChecksum("CRC32C", 512);

  Path p = new Path("/testAlgoSwitchRandomized");
  long seed = Time.now();
  System.out.println("seed: " + seed);
  Random r = new Random(seed);
  
  // Create empty to start
  IOUtils.closeStream(fsWithCrc32.create(p));
  
  long st = Time.now();
  int len = 0;
  while (Time.now() - st < RANDOM_TEST_RUNTIME) {
    int thisLen = r.nextInt(500);
    FileSystem fs = (r.nextBoolean() ? fsWithCrc32 : fsWithCrc32C);
    FSDataOutputStream stm = fs.append(p);
    try {
      AppendTestUtil.write(stm, len, thisLen);
    } finally {
      stm.close();
    }
    len += thisLen;
  }
  
  AppendTestUtil.check(fsWithCrc32, p, len);
  AppendTestUtil.check(fsWithCrc32C, p, len);
}
 
Example 12
Source Project: Transwarp-Sample-Code   File: HDFSDataStream.java    License: MIT License 5 votes vote down vote up
protected void doOpen(Configuration conf,
  Path dstPath, FileSystem hdfs) throws
  IOException {
  if(useRawLocalFileSystem) {
    if(hdfs instanceof LocalFileSystem) {
      hdfs = ((LocalFileSystem)hdfs).getRaw();
    } else {
      logger.warn("useRawLocalFileSystem is set to true but file system " +
          "is not of type LocalFileSystem: " + hdfs.getClass().getName());
    }
  }

  boolean appending = false;
  if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile
          (dstPath)) {
    outStream = hdfs.append(dstPath);
    appending = true;
  } else {
    outStream = hdfs.create(dstPath);
  }

  serializer = EventSerializerFactory.getInstance(
      serializerType, serializerContext, outStream);
  if (appending && !serializer.supportsReopen()) {
    outStream.close();
    serializer = null;
    throw new IOException("serializer (" + serializerType +
        ") does not support append");
  }

  // must call superclass to check for replication issues
  registerCurrentStream(outStream, hdfs, dstPath);

  if (appending) {
    serializer.afterReopen();
  } else {
    serializer.afterCreate();
  }
}
 
Example 13
Source Project: hadoop   File: TestEncryptedTransfer.java    License: Apache License 2.0 5 votes vote down vote up
private static void writeTestDataToFile(FileSystem fs) throws IOException {
  OutputStream out = null;
  if (!fs.exists(TEST_PATH)) {
    out = fs.create(TEST_PATH);
  } else {
    out = fs.append(TEST_PATH);
  }
  out.write(PLAIN_TEXT.getBytes());
  out.close();
}
 
Example 14
/**
 * Test which randomly alternates between appending with
 * CRC32 and with CRC32C, crossing several block boundaries.
 * Then, checks that all of the data can be read back correct.
 */
@Test(timeout=RANDOM_TEST_RUNTIME*2)
public void testAlgoSwitchRandomized() throws IOException {
  FileSystem fsWithCrc32 = createFsWithChecksum("CRC32", 512);
  FileSystem fsWithCrc32C = createFsWithChecksum("CRC32C", 512);

  Path p = new Path("/testAlgoSwitchRandomized");
  long seed = Time.now();
  System.out.println("seed: " + seed);
  Random r = new Random(seed);
  
  // Create empty to start
  IOUtils.closeStream(fsWithCrc32.create(p));
  
  long st = Time.now();
  int len = 0;
  while (Time.now() - st < RANDOM_TEST_RUNTIME) {
    int thisLen = r.nextInt(500);
    FileSystem fs = (r.nextBoolean() ? fsWithCrc32 : fsWithCrc32C);
    FSDataOutputStream stm = fs.append(p);
    try {
      AppendTestUtil.write(stm, len, thisLen);
    } finally {
      stm.close();
    }
    len += thisLen;
  }
  
  AppendTestUtil.check(fsWithCrc32, p, len);
  AppendTestUtil.check(fsWithCrc32C, p, len);
}
 
Example 15
Source Project: pulsar   File: HdfsAbstractSink.java    License: Apache License 2.0 5 votes vote down vote up
protected FSDataOutputStream getHdfsStream() throws IllegalArgumentException, IOException {
    if (hdfsStream == null) {
        Path path = getPath();
        FileSystem fs = getFileSystemAsUser(getConfiguration(), getUserGroupInformation());
        hdfsStream = fs.exists(path) ? fs.append(path) : fs.create(path);
    }
    return hdfsStream;
}
 
Example 16
Source Project: big-c   File: TestDecommission.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout=120000)
public void testDecommissionWithOpenfile() throws IOException, InterruptedException {
  LOG.info("Starting test testDecommissionWithOpenfile");
  
  //At most 4 nodes will be decommissioned
  startCluster(1, 7, conf);
      
  FileSystem fileSys = cluster.getFileSystem(0);
  FSNamesystem ns = cluster.getNamesystem(0);
  
  String openFile = "/testDecommissionWithOpenfile.dat";
         
  writeFile(fileSys, new Path(openFile), (short)3);   
  // make sure the file was open for write
  FSDataOutputStream fdos =  fileSys.append(new Path(openFile)); 
  
  LocatedBlocks lbs = NameNodeAdapter.getBlockLocations(cluster.getNameNode(0), openFile, 0, fileSize);
            
  DatanodeInfo[] dnInfos4LastBlock = lbs.getLastLocatedBlock().getLocations();
  DatanodeInfo[] dnInfos4FirstBlock = lbs.get(0).getLocations();
  
  ArrayList<String> nodes = new ArrayList<String>();
  ArrayList<DatanodeInfo> dnInfos = new ArrayList<DatanodeInfo>();

  DatanodeManager dm = ns.getBlockManager().getDatanodeManager();
  for (DatanodeInfo datanodeInfo : dnInfos4FirstBlock) {
    DatanodeInfo found = datanodeInfo;
    for (DatanodeInfo dif: dnInfos4LastBlock) {
      if (datanodeInfo.equals(dif)) {
       found = null;
      }
    }
    if (found != null) {
      nodes.add(found.getXferAddr());
      dnInfos.add(dm.getDatanode(found));
    }
  }
  //decommission one of the 3 nodes which have last block
  nodes.add(dnInfos4LastBlock[0].getXferAddr());
  dnInfos.add(dm.getDatanode(dnInfos4LastBlock[0]));
  
  writeConfigFile(excludeFile, nodes);
  refreshNodes(ns, conf);  
  for (DatanodeInfo dn : dnInfos) {
    waitNodeState(dn, AdminStates.DECOMMISSIONED);
  }           

  fdos.close();
}
 
Example 17
Source Project: RDFS   File: TestFileAppend4.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Test for a race in appendFile where the file might get removed in between
 * the two synchronized sections.
 */
public void testAppendFileRace() throws Throwable {
  LOG.info("START");
  cluster = new MiniDFSCluster(conf, 1, true, null);
  final FileSystem fs1 = cluster.getFileSystem();;

  try {
    createFile(fs1, "/testAppendFileRace", 1, BBW_SIZE);
    stm.close();

    NameNode nn = cluster.getNameNode();
    FSEditLog editLogSpy = FSImageAdapter.injectEditLogSpy(nn.getNamesystem());
    DelayAnswer  delayer = new DelayAnswer();
    doAnswer(delayer).when(editLogSpy).logSync();

    final AtomicReference<Throwable> err = new AtomicReference<Throwable>();
    Thread appender = new Thread() {
        public void run() {
          try {
            stm = fs1.append(file1);
          } catch (Throwable t) {
            err.set(t);
          }
        }
      };
    LOG.info("Triggering append in other thread");
    appender.start();

    LOG.info("Waiting for logsync");
    delayer.waitForCall();

    LOG.info("Resetting spy");
    reset(editLogSpy);

    LOG.info("Deleting file");
    fs1.delete(file1, true);

    LOG.info("Allowing append to proceed");
    delayer.proceed();

    LOG.info("Waiting for append to finish");

    appender.join();

    if (err.get() != null) {
      if (err.get().getMessage().contains(
            "File does not exist.")) {
        LOG.info("Got expected exception", err.get());
      } else {
        throw err.get();
      }
    }
    LOG.info("Closing stream");
    stm.close();
  } finally {
    fs1.close();
    cluster.shutdown();
  }
}
 
Example 18
Source Project: hadoop   File: TestEncryptionZones.java    License: Apache License 2.0 4 votes vote down vote up
private void appendOneByte(FileSystem fs, Path p) throws IOException {
  final FSDataOutputStream out = fs.append(p);
  out.write((byte) 0x123);
  out.close();
}
 
Example 19
@Override
public void open(String filePath, CompressionCodec codec,
    CompressionType cType) throws IOException {
  Configuration conf = new Configuration();
  Path dstPath = new Path(filePath);
  FileSystem hdfs = dstPath.getFileSystem(conf);
  if(useRawLocalFileSystem) {
    if(hdfs instanceof LocalFileSystem) {
      hdfs = ((LocalFileSystem)hdfs).getRaw();
    } else {
      logger.warn("useRawLocalFileSystem is set to true but file system " +
          "is not of type LocalFileSystem: " + hdfs.getClass().getName());
    }
  }

  boolean appending = false;
  if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile
  (dstPath)) {
    fsOut = hdfs.append(dstPath);
    appending = true;
  } else {
    fsOut = hdfs.create(dstPath);
  }
  cmpOut = codec.createOutputStream(fsOut);
  serializer = EventSerializerFactory.getInstance(serializerType,
      serializerContext, cmpOut);
  if (appending && !serializer.supportsReopen()) {
    cmpOut.close();
    serializer = null;
    throw new IOException("serializer (" + serializerType
        + ") does not support append");
  }

  registerCurrentStream(fsOut, hdfs, dstPath);

  if (appending) {
    serializer.afterReopen();
  } else {
    serializer.afterCreate();
  }
  isFinished = false;
}
 
Example 20
Source Project: hadoop   File: TestHAAppend.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Test to verify the processing of PendingDataNodeMessageQueue in case of
 * append. One block will marked as corrupt if the OP_ADD, OP_UPDATE_BLOCKS
 * comes in one edit log segment and OP_CLOSE edit comes in next log segment
 * which is loaded during failover. Regression test for HDFS-3605.
 */
@Test
public void testMultipleAppendsDuringCatchupTailing() throws Exception {
  Configuration conf = new Configuration();
  
  // Set a length edits tailing period, and explicit rolling, so we can
  // control the ingest of edits by the standby for this test.
  conf.set(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, "5000");
  conf.setInt(DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY, -1);

  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
      .nnTopology(MiniDFSNNTopology.simpleHATopology())
      .numDataNodes(3).build();
  FileSystem fs = null;
  try {
    cluster.transitionToActive(0);
    fs = HATestUtil.configureFailoverFs(cluster, conf);

    Path fileToAppend = new Path("/FileToAppend");
    Path fileToTruncate = new Path("/FileToTruncate");
    
    final byte[] data = new byte[1 << 16];
    DFSUtil.getRandom().nextBytes(data);
    final int[] appendPos = AppendTestUtil.randomFilePartition(
        data.length, COUNT);
    final int[] truncatePos = AppendTestUtil.randomFilePartition(
        data.length, 1);

    // Create file, write some data, and hflush so that the first
    // block is in the edit log prior to roll.
    FSDataOutputStream out = createAndHflush(
        fs, fileToAppend, data, appendPos[0]);

    FSDataOutputStream out4Truncate = createAndHflush(
        fs, fileToTruncate, data, data.length);
    
    // Let the StandbyNode catch the creation of the file. 
    cluster.getNameNode(0).getRpcServer().rollEditLog();
    cluster.getNameNode(1).getNamesystem().getEditLogTailer().doTailEdits();
    out.close();
    out4Truncate.close();

    // Append and re-close a few time, so that many block entries are queued.
    for (int i = 0; i < COUNT; i++) {
      int end = i < COUNT - 1? appendPos[i + 1]: data.length;
      out = fs.append(fileToAppend);
      out.write(data, appendPos[i], end - appendPos[i]);
      out.close();
    }
    boolean isTruncateReady = fs.truncate(fileToTruncate, truncatePos[0]);

    // Ensure that blocks have been reported to the SBN ahead of the edits
    // arriving.
    cluster.triggerBlockReports();

    // Failover the current standby to active.
    cluster.shutdownNameNode(0);
    cluster.transitionToActive(1);
    
    // Check the FSCK doesn't detect any bad blocks on the SBN.
    int rc = ToolRunner.run(new DFSck(cluster.getConfiguration(1)),
        new String[] { "/", "-files", "-blocks" });
    assertEquals(0, rc);
    
    assertEquals("CorruptBlocks should be empty.", 0, cluster.getNameNode(1)
        .getNamesystem().getCorruptReplicaBlocks());

    AppendTestUtil.checkFullFile(fs, fileToAppend, data.length, data,
        fileToAppend.toString());

    if (!isTruncateReady) {
      TestFileTruncate.checkBlockRecovery(fileToTruncate,
          cluster.getFileSystem(1));
    }
    AppendTestUtil.checkFullFile(fs, fileToTruncate, truncatePos[0], data,
        fileToTruncate.toString());
  } finally {
    if (null != cluster) {
      cluster.shutdown();
    }
    if (null != fs) {
      fs.close();
    }
  }
}