Java Code Examples for org.apache.hadoop.fs.FSDataOutputStream#close()

/** Create a file with a length of <code>fileSize</code>.
 * The file is filled with 'a'.
 */
private void genFile(Path file, long fileSize) throws IOException {
  long startTime = System.currentTimeMillis();
  FSDataOutputStream out = fs.create(file, true, 
      getConf().getInt("io.file.buffer.size", 4096),
      (short)getConf().getInt("dfs.replication", 3),
      fs.getDefaultBlockSize());
  executionTime[CREATE] += (System.currentTimeMillis()-startTime);
  totalNumOfOps[CREATE]++;

  for (long i=0; i<fileSize; i++) {
    out.writeByte('a');
  }
  startTime = System.currentTimeMillis();
  out.close();
  executionTime[WRITE_CLOSE] += (System.currentTimeMillis()-startTime);
  totalNumOfOps[WRITE_CLOSE]++;
}
 

public String generateCommonJar() throws IOException
{
  File file = File.createTempFile("schemaSQL", ".jar");

  FileSystem fs = FileSystem.newInstance(file.toURI(), new Configuration());
  FSDataOutputStream out = fs.create(new Path(file.getAbsolutePath()));
  JarOutputStream jout = new JarOutputStream(out);

  for (Schema schema : schemas.values()) {
    jout.putNextEntry(new ZipEntry(schema.fqcn.replace(".", "/") + ".class"));
    jout.write(schema.beanClassBytes);
    jout.closeEntry();
  }

  jout.close();
  out.close();

  return file.getAbsolutePath();
}
 

private Path initFiles(FileSystem fs, int numFiles, int numBytes) throws IOException{
  Path dir = new Path(System.getProperty("test.build.data",".") + "/mapred");
  Path multiFileDir = new Path(dir, "test.multifile");
  fs.delete(multiFileDir, true);
  fs.mkdirs(multiFileDir);
  LOG.info("Creating " + numFiles + " file(s) in " + multiFileDir);
  for(int i=0; i<numFiles ;i++) {
    Path path = new Path(multiFileDir, "file_" + i);
     FSDataOutputStream out = fs.create(path);
     if (numBytes == -1) {
       numBytes = rand.nextInt(MAX_BYTES);
     }
     for(int j=0; j< numBytes; j++) {
       out.write(rand.nextInt());
     }
     out.close();
     if(LOG.isDebugEnabled()) {
       LOG.debug("Created file " + path + " with length " + numBytes);
     }
     lengths.put(path.getName(), new Long(numBytes));
  }
  FileInputFormat.setInputPaths(job, multiFileDir);
  return multiFileDir;
}
 

private FileStatus[] addMockStoreFiles(int count, MasterServices services, Path storedir)
  throws IOException {
  // get the existing store files
  FileSystem fs = services.getMasterFileSystem().getFileSystem();
  fs.mkdirs(storedir);
  // create the store files in the parent
  for (int i = 0; i < count; i++) {
    Path storeFile = new Path(storedir, "_store" + i);
    FSDataOutputStream dos = fs.create(storeFile, true);
    dos.writeBytes("Some data: " + i);
    dos.close();
  }
  LOG.debug("Adding " + count + " store files to the storedir:" + storedir);
  // make sure the mock store files are there
  FileStatus[] storeFiles = fs.listStatus(storedir);
  assertEquals("Didn't have expected store files", count, storeFiles.length);
  return storeFiles;
}
 

/**
 * Run an ImportTsv job and perform basic validation on the results. Returns
 * the ImportTsv <code>Tool</code> instance so that other tests can inspect it
 * for further validation as necessary. This method is static to insure
 * non-reliance on instance's util/conf facilities.
 *
 * @param args
 *          Any arguments to pass BEFORE inputFile path is appended.
 * @param dataAvailable
 * @return The Tool instance used to run the test.
 */
private Tool doMROnTableTest(HBaseTestingUtility util, String family, String data, String[] args,
    int valueMultiplier, boolean dataAvailable) throws Exception {
  String table = args[args.length - 1];
  Configuration conf = new Configuration(util.getConfiguration());

  // populate input file
  FileSystem fs = FileSystem.get(conf);
  Path inputPath = fs.makeQualified(new Path(util.getDataTestDirOnTestFS(table), "input.dat"));
  FSDataOutputStream op = fs.create(inputPath, true);
  op.write(Bytes.toBytes(data));
  op.close();
  LOG.debug(String.format("Wrote test data to file: %s", inputPath));

  if (conf.getBoolean(FORCE_COMBINER_CONF, true)) {
    LOG.debug("Forcing combiner.");
    conf.setInt("mapreduce.map.combine.minspills", 1);
  }

  // run the import
  List<String> argv = new ArrayList<>(Arrays.asList(args));
  argv.add(inputPath.toString());
  Tool tool = new ImportTsv();
  LOG.debug("Running ImportTsv with arguments: " + argv);
  assertEquals(0, ToolRunner.run(conf, tool, argv.toArray(args)));

  validateTable(conf, TableName.valueOf(table), family, valueMultiplier, dataAvailable);

  if (conf.getBoolean(DELETE_AFTER_LOAD_CONF, true)) {
    LOG.debug("Deleting test subdirectory");
    util.cleanupDataTestDirOnTestFS(table);
  }
  return tool;
}
 

/** Test two consecutive appends on a file with a full block. */
@Test
public void testAppendTwice() throws Exception {
  Configuration conf = new HdfsConfiguration();
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
  final FileSystem fs1 = cluster.getFileSystem();
  final FileSystem fs2 = AppendTestUtil.createHdfsWithDifferentUsername(conf);
  try {

    final Path p = new Path("/testAppendTwice/foo");
    final int len = 1 << 16;
    final byte[] fileContents = AppendTestUtil.initBuffer(len);

    {
      // create a new file with a full block.
      FSDataOutputStream out = fs2.create(p, true, 4096, (short)1, len);
      out.write(fileContents, 0, len);
      out.close();
    }

    //1st append does not add any data so that the last block remains full
    //and the last block in INodeFileUnderConstruction is a BlockInfo
    //but not BlockInfoUnderConstruction. 
    fs2.append(p);
    
    //2nd append should get AlreadyBeingCreatedException
    fs1.append(p);
    Assert.fail();
  } catch(RemoteException re) {
    AppendTestUtil.LOG.info("Got an exception:", re);
    Assert.assertEquals(AlreadyBeingCreatedException.class.getName(),
        re.getClassName());
  } finally {
    fs2.close();
    fs1.close();
    cluster.shutdown();
  }
}
 

private Path createTempFile(String filename, String contents)
    throws IOException {
  Path path = new Path(TEST_ROOT_DIR, filename);
  FSDataOutputStream os = localFs.create(path);
  os.writeBytes(contents);
  os.close();
  localFs.setPermission(path, new FsPermission("700"));
  return path;
}
 

public void testOutputStreamClosedTwice() throws IOException {
  //HADOOP-4760 according to Closeable#close() closing already-closed 
  //streams should have no effect. 
  Path src = path("/test/hadoop/file");
  FSDataOutputStream out = fs.create(src);
  out.writeChar('H'); //write some data
  out.close();
  out.close();
}
 

@Test
public void testAppendNonexistentFile() throws Throwable {
  try {
    FSDataOutputStream out = getFileSystem().append(target);
    //got here: trouble
    out.close();
    fail("expected a failure");
  } catch (Exception e) {
    //expected
    handleExpectedException(e);
  }
}
 

@SuppressWarnings("ThrowFromFinallyBlock")
@Override
public void save(Object object, int operatorId, long windowId) throws IOException
{
  String operatorIdStr = String.valueOf(operatorId);
  Path lPath = new Path(path + Path.SEPARATOR + operatorIdStr + Path.SEPARATOR + TMP_FILE);
  String window = Long.toHexString(windowId);
  boolean stateSaved = false;
  FSDataOutputStream stream = null;
  try {
    stream = fileContext.create(lPath, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
      Options.CreateOpts.CreateParent.createParent());
    store(stream, object);
    stateSaved = true;
  } catch (Throwable t) {
    logger.debug("while saving {} {}", operatorId, window, t);
    stateSaved = false;
    throw Throwables.propagate(t);
  } finally {
    try {
      if (stream != null) {
        stream.close();
      }
    } catch (IOException ie) {
      stateSaved = false;
      throw new RuntimeException(ie);
    } finally {
      if (stateSaved) {
        logger.debug("Saving {}: {}", operatorId, window);
        fileContext.rename(lPath, new Path(path + Path.SEPARATOR + operatorIdStr + Path.SEPARATOR + window),
            Options.Rename.OVERWRITE);
      }
    }
  }
}
 

/**
 * Test if the quota can be correctly updated when file length is updated
 * through fsync
 */
@Test (timeout=60000)
public void testUpdateQuotaForFSync() throws Exception {
  final Path foo = new Path("/foo");
  final Path bar = new Path(foo, "bar");
  DFSTestUtil.createFile(dfs, bar, BLOCKSIZE, REPLICATION, 0L);
  dfs.setQuota(foo, Long.MAX_VALUE - 1, Long.MAX_VALUE - 1);

  FSDataOutputStream out = dfs.append(bar);
  out.write(new byte[BLOCKSIZE / 4]);
  ((DFSOutputStream) out.getWrappedStream()).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));

  INodeDirectory fooNode = fsdir.getINode4Write(foo.toString()).asDirectory();
  QuotaCounts quota = fooNode.getDirectoryWithQuotaFeature()
      .getSpaceConsumed();
  long ns = quota.getNameSpace();
  long ds = quota.getStorageSpace();
  assertEquals(2, ns); // foo and bar
  assertEquals(BLOCKSIZE * 2 * REPLICATION, ds); // file is under construction

  out.write(new byte[BLOCKSIZE / 4]);
  out.close();

  fooNode = fsdir.getINode4Write(foo.toString()).asDirectory();
  quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
  ns = quota.getNameSpace();
  ds = quota.getStorageSpace();
  assertEquals(2, ns);
  assertEquals((BLOCKSIZE + BLOCKSIZE / 2) * REPLICATION, ds);

  // append another block
  DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE);

  quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
  ns = quota.getNameSpace();
  ds = quota.getStorageSpace();
  assertEquals(2, ns); // foo and bar
  assertEquals((BLOCKSIZE * 2 + BLOCKSIZE / 2) * REPLICATION, ds);
}
 

/**
 * This system procedure checks table consistency
 * @param schemaName
 * @param tableName
 * @param level
 * @param outputFile
 * @param resultSet
 * @throws Exception
 */
public static void CHECK_TABLE(String schemaName, String tableName, int level,
                               String outputFile, boolean fix, final ResultSet[] resultSet) throws Exception {

    FSDataOutputStream out = null;
    FileSystem fs = null;
    Map<String, List<String>> errors = null;
    try {
        LanguageConnectionContext lcc = ConnectionUtil.getCurrentLCC();
        Activation activation = lcc.getLastActivation();

        Configuration conf = (Configuration) SIDriver.driver().getConfiguration().getConfigSource().unwrapDelegate();
        String schema = EngineUtils.validateSchema(schemaName);
        String table = EngineUtils.validateTable(tableName);
        fs = FileSystem.get(URI.create(outputFile), conf);
        out = fs.create(new Path(outputFile));


        errors = checkTable(schema, table, null, level, fix);
        resultSet[0] = processResults(errors, out, activation, outputFile);
    } finally {
        if (out != null) {
            out.close();
            if (errors == null || errors.size() == 0) {
                fs.delete(new Path(outputFile), true);
            }
        }

    }
}
 

private Writer writeTestFile(boolean rle, boolean repeatKeys,
    List<KVPair> data, CompressionCodec codec) throws IOException {
  FSDataOutputStream out = localFs.create(outputPath);
  IFile.Writer writer = new IFile.Writer(defaultConf, out,
      Text.class, IntWritable.class, codec, null, null, rle);
  writeTestFile(writer, repeatKeys, data);
  out.close();
  return  writer;
}
 

/**
 * testing that APPEND operation can handle token expiration when
 * re-establishing pipeline is needed
 */
@Test
public void testAppend() throws Exception {
  MiniDFSCluster cluster = null;
  int numDataNodes = 2;
  Configuration conf = getConf(numDataNodes);

  try {
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
    cluster.waitActive();
    assertEquals(numDataNodes, cluster.getDataNodes().size());

    final NameNode nn = cluster.getNameNode();
    final BlockManager bm = nn.getNamesystem().getBlockManager();
    final BlockTokenSecretManager sm = bm.getBlockTokenSecretManager();

    // set a short token lifetime (1 second)
    SecurityTestUtil.setBlockTokenLifetime(sm, 1000L);
    Path fileToAppend = new Path(FILE_TO_APPEND);
    FileSystem fs = cluster.getFileSystem();

    // write a one-byte file
    FSDataOutputStream stm = writeFile(fs, fileToAppend,
        (short) numDataNodes, BLOCK_SIZE);
    stm.write(rawData, 0, 1);
    stm.close();
    // open the file again for append
    stm = fs.append(fileToAppend);
    int mid = rawData.length - 1;
    stm.write(rawData, 1, mid - 1);
    stm.hflush();

    /*
     * wait till token used in stm expires
     */
    Token<BlockTokenIdentifier> token = DFSTestUtil.getBlockToken(stm);
    while (!SecurityTestUtil.isBlockTokenExpired(token)) {
      try {
        Thread.sleep(10);
      } catch (InterruptedException ignored) {
      }
    }

    // remove a datanode to force re-establishing pipeline
    cluster.stopDataNode(0);
    // append the rest of the file
    stm.write(rawData, mid, rawData.length - mid);
    stm.close();
    // check if append is successful
    FSDataInputStream in5 = fs.open(fileToAppend);
    assertTrue(checkFile1(in5));
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

/**
 * Run file operations to create edits for all op codes
 * to be tested.
 */
private void runOperations() throws IOException {

  LOG.info("Creating edits by performing fs operations");
  // no check, if it's not it throws an exception which is what we want
  DistributedFileSystem dfs =
    (DistributedFileSystem)cluster.getFileSystem();
  // OP_ADD 0, OP_SET_GENSTAMP 10
  Path pathFileCreate = new Path("/file_create");
  FSDataOutputStream s = dfs.create(pathFileCreate);
  // OP_CLOSE 9
  s.close();
  // OP_RENAME 1
  Path pathFileMoved = new Path("/file_moved");
  dfs.rename(pathFileCreate, pathFileMoved);
  // OP_DELETE 2
  dfs.delete(pathFileMoved, false);
  // OP_MKDIR 3
  Path pathDirectoryMkdir = new Path("/directory_mkdir");
  dfs.mkdirs(pathDirectoryMkdir);
  // OP_SET_REPLICATION 4
  s = dfs.create(pathFileCreate);
  s.close();
  dfs.setReplication(pathFileCreate, (short)1);
  // OP_SET_PERMISSIONS 7
  Short permission = 0777;
  dfs.setPermission(pathFileCreate, new FsPermission(permission));
  // OP_SET_OWNER 8
  dfs.setOwner(pathFileCreate, new String("newOwner"), null);
  // OP_CLOSE 9 see above
  // OP_SET_GENSTAMP 10 see above
  // OP_SET_NS_QUOTA 11 obsolete
  // OP_CLEAR_NS_QUOTA 12 obsolete
  // OP_TIMES 13
  long mtime = 1285195527000L; // Wed, 22 Sep 2010 22:45:27 GMT
  long atime = mtime;
  dfs.setTimes(pathFileCreate, mtime, atime);
  // OP_SET_QUOTA 14
  dfs.setQuota(pathDirectoryMkdir, 1000L, FSConstants.QUOTA_DONT_SET);
  // OP_CONCAT_DELETE 16
  Path   pathConcatTarget = new Path("/file_concat_target");
  Path[] pathConcatFiles  = new Path[2];
  pathConcatFiles[0]      = new Path("/file_concat_0");
  pathConcatFiles[1]      = new Path("/file_concat_1");

  long  length      = blockSize * 3; // multiple of blocksize for concat
  short replication = 1;
  long  seed        = 1;

  DFSTestUtil.createFile(dfs, pathConcatTarget, length, replication, seed);
  DFSTestUtil.createFile(dfs, pathConcatFiles[0], length, replication, seed);
  DFSTestUtil.createFile(dfs, pathConcatFiles[1], length, replication, seed);
  dfs.concat(pathConcatTarget, pathConcatFiles, false);

  // sync to disk, otherwise we parse partial edits
  cluster.getNameNode().getFSImage().getEditLog().logSync();
  dfs.close();
}
 

void createFile(FileSystem fs, Path path, int fileLen) throws IOException {
  byte [] arr = new byte[fileLen];
  FSDataOutputStream out = fs.create(path);
  out.write(arr);
  out.close();
}
 

@Test
public void testVerifyChecksum() throws Exception {    
  Path testPath = new Path(TEST_ROOT_DIR, "testPath");
  Path testPath11 = new Path(TEST_ROOT_DIR, "testPath11");
  FSDataOutputStream fout = localFs.create(testPath);
  fout.write("testing".getBytes());
  fout.close();
  
  fout = localFs.create(testPath11);
  fout.write("testing you".getBytes());
  fout.close();

  // Exercise some boundary cases - a divisor of the chunk size
  // the chunk size, 2x chunk size, and +/-1 around these.
  readFile(localFs, testPath, 128);
  readFile(localFs, testPath, 511);
  readFile(localFs, testPath, 512);
  readFile(localFs, testPath, 513);
  readFile(localFs, testPath, 1023);
  readFile(localFs, testPath, 1024);
  readFile(localFs, testPath, 1025);

  localFs.delete(localFs.getChecksumFile(testPath), true);
  assertTrue("checksum deleted", !localFs.exists(localFs.getChecksumFile(testPath)));
  
  //copying the wrong checksum file
  FileUtil.copy(localFs, localFs.getChecksumFile(testPath11), localFs, 
      localFs.getChecksumFile(testPath),false,true,localFs.getConf());
  assertTrue("checksum exists", localFs.exists(localFs.getChecksumFile(testPath)));
  
  boolean errorRead = false;
  try {
    readFile(localFs, testPath, 1024);
  }catch(ChecksumException ie) {
    errorRead = true;
  }
  assertTrue("error reading", errorRead);
  
  //now setting verify false, the read should succeed
  localFs.setVerifyChecksum(false);
  String str = readFile(localFs, testPath, 1024).toString();
  assertTrue("read", "testing".equals(str));
}
 

public void writeTokenStorageFile(Path filename, 
                                  Configuration conf) throws IOException {
  FSDataOutputStream os = filename.getFileSystem(conf).create(filename);
  writeTokenStorageToStream(os);
  os.close();
}
 

/**
 * Handles the degenerate case where serialization fails to fit in
 * the in-memory buffer, so we must spill the record from collect
 * directly to a spill file. Consider this "losing".
 */
private void spillSingleRecord(final Object key, final Object value,
                               int partition) throws IOException {
  long size = kvbuffer.length + partitions * APPROX_HEADER_LENGTH;
  FSDataOutputStream out = null;
  try {
    // create spill file
    final TezSpillRecord spillRec = new TezSpillRecord(partitions);
    final Path filename =
        mapOutputFile.getSpillFileForWrite(numSpills, size);
    spillFilePaths.put(numSpills, filename);
    out = rfs.create(filename);
    ensureSpillFilePermissions(filename, rfs);

    // we don't run the combiner for a single record
    for (int i = 0; i < partitions; ++i) {
      IFile.Writer writer = null;
      try {
        long segmentStart = out.getPos();
        // Create a new codec, don't care!
        if (!sendEmptyPartitionDetails || (i == partition)) {
          writer = new Writer(conf, out, keyClass, valClass, codec,
              spilledRecordsCounter, null, false);
        }
        if (i == partition) {
          final long recordStart = out.getPos();
          writer.append(key, value);
          // Note that our map byte count will not be accurate with
          // compression
          mapOutputByteCounter.increment(out.getPos() - recordStart);
        }
        long rawLength =0;
        long partLength =0;
        if (writer != null) {
          writer.close();
          rawLength = writer.getRawLength();
          partLength = writer.getCompressedLength();
        }
        adjustSpillCounters(rawLength, partLength);

        // record offsets
        TezIndexRecord rec = new TezIndexRecord(segmentStart, rawLength, partLength);
        spillRec.putIndex(rec, i);

        writer = null;
      } catch (IOException e) {
        if (null != writer) writer.close();
        throw e;
      }
    }
    if (totalIndexCacheMemory >= indexCacheMemoryLimit) {
      // create spill index file
      Path indexFilename =
          mapOutputFile.getSpillIndexFileForWrite(numSpills, partitions
              * MAP_OUTPUT_INDEX_RECORD_LENGTH);
      spillFileIndexPaths.put(numSpills, indexFilename);
      spillRec.writeToFile(indexFilename, conf, localFs);
    } else {
      indexCacheList.add(spillRec);
      totalIndexCacheMemory +=
        spillRec.size() * MAP_OUTPUT_INDEX_RECORD_LENGTH;
    }
    ++numSpills;
    if (!isFinalMergeEnabled()) {
      numShuffleChunks.setValue(numSpills);
    } else if (numSpills > 1) {
      //Increment only when there is atleast one previous spill
      numAdditionalSpills.increment(1);
    }
  } finally {
    if (out != null) out.close();
  }
}
 

/**
 * Create and then close a file
 *
 * @param path path to create
 * @throws IOException on a failure
 */
protected static void createEmptyFile(Path path) throws IOException {
  FSDataOutputStream out = sFileSystem.create(path);
  out.close();
}