Java Code Examples for org.apache.hadoop.fs.FSDataOutputStream#close()

The following examples show how to use org.apache.hadoop.fs.FSDataOutputStream#close() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: LoadGenerator.java    From RDFS with Apache License 2.0 6 votes vote down vote up
/** Create a file with a length of <code>fileSize</code>.
 * The file is filled with 'a'.
 */
private void genFile(Path file, long fileSize) throws IOException {
  long startTime = System.currentTimeMillis();
  FSDataOutputStream out = fs.create(file, true, 
      getConf().getInt("io.file.buffer.size", 4096),
      (short)getConf().getInt("dfs.replication", 3),
      fs.getDefaultBlockSize());
  executionTime[CREATE] += (System.currentTimeMillis()-startTime);
  totalNumOfOps[CREATE]++;

  for (long i=0; i<fileSize; i++) {
    out.writeByte('a');
  }
  startTime = System.currentTimeMillis();
  out.close();
  executionTime[WRITE_CLOSE] += (System.currentTimeMillis()-startTime);
  totalNumOfOps[WRITE_CLOSE]++;
}
 
Example 2
Source File: TupleSchemaRegistry.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
public String generateCommonJar() throws IOException
{
  File file = File.createTempFile("schemaSQL", ".jar");

  FileSystem fs = FileSystem.newInstance(file.toURI(), new Configuration());
  FSDataOutputStream out = fs.create(new Path(file.getAbsolutePath()));
  JarOutputStream jout = new JarOutputStream(out);

  for (Schema schema : schemas.values()) {
    jout.putNextEntry(new ZipEntry(schema.fqcn.replace(".", "/") + ".class"));
    jout.write(schema.beanClassBytes);
    jout.closeEntry();
  }

  jout.close();
  out.close();

  return file.getAbsolutePath();
}
 
Example 3
Source File: TestMultiFileInputFormat.java    From big-c with Apache License 2.0 6 votes vote down vote up
private Path initFiles(FileSystem fs, int numFiles, int numBytes) throws IOException{
  Path dir = new Path(System.getProperty("test.build.data",".") + "/mapred");
  Path multiFileDir = new Path(dir, "test.multifile");
  fs.delete(multiFileDir, true);
  fs.mkdirs(multiFileDir);
  LOG.info("Creating " + numFiles + " file(s) in " + multiFileDir);
  for(int i=0; i<numFiles ;i++) {
    Path path = new Path(multiFileDir, "file_" + i);
     FSDataOutputStream out = fs.create(path);
     if (numBytes == -1) {
       numBytes = rand.nextInt(MAX_BYTES);
     }
     for(int j=0; j< numBytes; j++) {
       out.write(rand.nextInt());
     }
     out.close();
     if(LOG.isDebugEnabled()) {
       LOG.debug("Created file " + path + " with length " + numBytes);
     }
     lengths.put(path.getName(), new Long(numBytes));
  }
  FileInputFormat.setInputPaths(job, multiFileDir);
  return multiFileDir;
}
 
Example 4
Source File: TestCatalogJanitor.java    From hbase with Apache License 2.0 6 votes vote down vote up
private FileStatus[] addMockStoreFiles(int count, MasterServices services, Path storedir)
  throws IOException {
  // get the existing store files
  FileSystem fs = services.getMasterFileSystem().getFileSystem();
  fs.mkdirs(storedir);
  // create the store files in the parent
  for (int i = 0; i < count; i++) {
    Path storeFile = new Path(storedir, "_store" + i);
    FSDataOutputStream dos = fs.create(storeFile, true);
    dos.writeBytes("Some data: " + i);
    dos.close();
  }
  LOG.debug("Adding " + count + " store files to the storedir:" + storedir);
  // make sure the mock store files are there
  FileStatus[] storeFiles = fs.listStatus(storedir);
  assertEquals("Didn't have expected store files", count, storeFiles.length);
  return storeFiles;
}
 
Example 5
Source File: TestImportTSVWithOperationAttributes.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Run an ImportTsv job and perform basic validation on the results. Returns
 * the ImportTsv <code>Tool</code> instance so that other tests can inspect it
 * for further validation as necessary. This method is static to insure
 * non-reliance on instance's util/conf facilities.
 *
 * @param args
 *          Any arguments to pass BEFORE inputFile path is appended.
 * @param dataAvailable
 * @return The Tool instance used to run the test.
 */
private Tool doMROnTableTest(HBaseTestingUtility util, String family, String data, String[] args,
    int valueMultiplier, boolean dataAvailable) throws Exception {
  String table = args[args.length - 1];
  Configuration conf = new Configuration(util.getConfiguration());

  // populate input file
  FileSystem fs = FileSystem.get(conf);
  Path inputPath = fs.makeQualified(new Path(util.getDataTestDirOnTestFS(table), "input.dat"));
  FSDataOutputStream op = fs.create(inputPath, true);
  op.write(Bytes.toBytes(data));
  op.close();
  LOG.debug(String.format("Wrote test data to file: %s", inputPath));

  if (conf.getBoolean(FORCE_COMBINER_CONF, true)) {
    LOG.debug("Forcing combiner.");
    conf.setInt("mapreduce.map.combine.minspills", 1);
  }

  // run the import
  List<String> argv = new ArrayList<>(Arrays.asList(args));
  argv.add(inputPath.toString());
  Tool tool = new ImportTsv();
  LOG.debug("Running ImportTsv with arguments: " + argv);
  assertEquals(0, ToolRunner.run(conf, tool, argv.toArray(args)));

  validateTable(conf, TableName.valueOf(table), family, valueMultiplier, dataAvailable);

  if (conf.getBoolean(DELETE_AFTER_LOAD_CONF, true)) {
    LOG.debug("Deleting test subdirectory");
    util.cleanupDataTestDirOnTestFS(table);
  }
  return tool;
}
 
Example 6
Source File: TestFileAppend.java    From big-c with Apache License 2.0 5 votes vote down vote up
/** Test two consecutive appends on a file with a full block. */
@Test
public void testAppendTwice() throws Exception {
  Configuration conf = new HdfsConfiguration();
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
  final FileSystem fs1 = cluster.getFileSystem();
  final FileSystem fs2 = AppendTestUtil.createHdfsWithDifferentUsername(conf);
  try {

    final Path p = new Path("/testAppendTwice/foo");
    final int len = 1 << 16;
    final byte[] fileContents = AppendTestUtil.initBuffer(len);

    {
      // create a new file with a full block.
      FSDataOutputStream out = fs2.create(p, true, 4096, (short)1, len);
      out.write(fileContents, 0, len);
      out.close();
    }

    //1st append does not add any data so that the last block remains full
    //and the last block in INodeFileUnderConstruction is a BlockInfo
    //but not BlockInfoUnderConstruction. 
    fs2.append(p);
    
    //2nd append should get AlreadyBeingCreatedException
    fs1.append(p);
    Assert.fail();
  } catch(RemoteException re) {
    AppendTestUtil.LOG.info("Got an exception:", re);
    Assert.assertEquals(AlreadyBeingCreatedException.class.getName(),
        re.getClassName());
  } finally {
    fs2.close();
    fs1.close();
    cluster.shutdown();
  }
}
 
Example 7
Source File: TestMRJobs.java    From big-c with Apache License 2.0 5 votes vote down vote up
private Path createTempFile(String filename, String contents)
    throws IOException {
  Path path = new Path(TEST_ROOT_DIR, filename);
  FSDataOutputStream os = localFs.create(path);
  os.writeBytes(contents);
  os.close();
  localFs.setPermission(path, new FsPermission("700"));
  return path;
}
 
Example 8
Source File: FileSystemContractBaseTest.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
public void testOutputStreamClosedTwice() throws IOException {
  //HADOOP-4760 according to Closeable#close() closing already-closed 
  //streams should have no effect. 
  Path src = path("/test/hadoop/file");
  FSDataOutputStream out = fs.create(src);
  out.writeChar('H'); //write some data
  out.close();
  out.close();
}
 
Example 9
Source File: AbstractContractAppendTest.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testAppendNonexistentFile() throws Throwable {
  try {
    FSDataOutputStream out = getFileSystem().append(target);
    //got here: trouble
    out.close();
    fail("expected a failure");
  } catch (Exception e) {
    //expected
    handleExpectedException(e);
  }
}
 
Example 10
Source File: FSStorageAgent.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("ThrowFromFinallyBlock")
@Override
public void save(Object object, int operatorId, long windowId) throws IOException
{
  String operatorIdStr = String.valueOf(operatorId);
  Path lPath = new Path(path + Path.SEPARATOR + operatorIdStr + Path.SEPARATOR + TMP_FILE);
  String window = Long.toHexString(windowId);
  boolean stateSaved = false;
  FSDataOutputStream stream = null;
  try {
    stream = fileContext.create(lPath, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
      Options.CreateOpts.CreateParent.createParent());
    store(stream, object);
    stateSaved = true;
  } catch (Throwable t) {
    logger.debug("while saving {} {}", operatorId, window, t);
    stateSaved = false;
    throw Throwables.propagate(t);
  } finally {
    try {
      if (stream != null) {
        stream.close();
      }
    } catch (IOException ie) {
      stateSaved = false;
      throw new RuntimeException(ie);
    } finally {
      if (stateSaved) {
        logger.debug("Saving {}: {}", operatorId, window);
        fileContext.rename(lPath, new Path(path + Path.SEPARATOR + operatorIdStr + Path.SEPARATOR + window),
            Options.Rename.OVERWRITE);
      }
    }
  }
}
 
Example 11
Source File: TestDiskspaceQuotaUpdate.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Test if the quota can be correctly updated when file length is updated
 * through fsync
 */
@Test (timeout=60000)
public void testUpdateQuotaForFSync() throws Exception {
  final Path foo = new Path("/foo");
  final Path bar = new Path(foo, "bar");
  DFSTestUtil.createFile(dfs, bar, BLOCKSIZE, REPLICATION, 0L);
  dfs.setQuota(foo, Long.MAX_VALUE - 1, Long.MAX_VALUE - 1);

  FSDataOutputStream out = dfs.append(bar);
  out.write(new byte[BLOCKSIZE / 4]);
  ((DFSOutputStream) out.getWrappedStream()).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));

  INodeDirectory fooNode = fsdir.getINode4Write(foo.toString()).asDirectory();
  QuotaCounts quota = fooNode.getDirectoryWithQuotaFeature()
      .getSpaceConsumed();
  long ns = quota.getNameSpace();
  long ds = quota.getStorageSpace();
  assertEquals(2, ns); // foo and bar
  assertEquals(BLOCKSIZE * 2 * REPLICATION, ds); // file is under construction

  out.write(new byte[BLOCKSIZE / 4]);
  out.close();

  fooNode = fsdir.getINode4Write(foo.toString()).asDirectory();
  quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
  ns = quota.getNameSpace();
  ds = quota.getStorageSpace();
  assertEquals(2, ns);
  assertEquals((BLOCKSIZE + BLOCKSIZE / 2) * REPLICATION, ds);

  // append another block
  DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE);

  quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
  ns = quota.getNameSpace();
  ds = quota.getStorageSpace();
  assertEquals(2, ns); // foo and bar
  assertEquals((BLOCKSIZE * 2 + BLOCKSIZE / 2) * REPLICATION, ds);
}
 
Example 12
Source File: SpliceTableAdmin.java    From spliceengine with GNU Affero General Public License v3.0 5 votes vote down vote up
/**
 * This system procedure checks table consistency
 * @param schemaName
 * @param tableName
 * @param level
 * @param outputFile
 * @param resultSet
 * @throws Exception
 */
public static void CHECK_TABLE(String schemaName, String tableName, int level,
                               String outputFile, boolean fix, final ResultSet[] resultSet) throws Exception {

    FSDataOutputStream out = null;
    FileSystem fs = null;
    Map<String, List<String>> errors = null;
    try {
        LanguageConnectionContext lcc = ConnectionUtil.getCurrentLCC();
        Activation activation = lcc.getLastActivation();

        Configuration conf = (Configuration) SIDriver.driver().getConfiguration().getConfigSource().unwrapDelegate();
        String schema = EngineUtils.validateSchema(schemaName);
        String table = EngineUtils.validateTable(tableName);
        fs = FileSystem.get(URI.create(outputFile), conf);
        out = fs.create(new Path(outputFile));


        errors = checkTable(schema, table, null, level, fix);
        resultSet[0] = processResults(errors, out, activation, outputFile);
    } finally {
        if (out != null) {
            out.close();
            if (errors == null || errors.size() == 0) {
                fs.delete(new Path(outputFile), true);
            }
        }

    }
}
 
Example 13
Source File: TestIFile.java    From tez with Apache License 2.0 5 votes vote down vote up
private Writer writeTestFile(boolean rle, boolean repeatKeys,
    List<KVPair> data, CompressionCodec codec) throws IOException {
  FSDataOutputStream out = localFs.create(outputPath);
  IFile.Writer writer = new IFile.Writer(defaultConf, out,
      Text.class, IntWritable.class, codec, null, null, rle);
  writeTestFile(writer, repeatKeys, data);
  out.close();
  return  writer;
}
 
Example 14
Source File: TestBlockTokenWithDFS.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * testing that APPEND operation can handle token expiration when
 * re-establishing pipeline is needed
 */
@Test
public void testAppend() throws Exception {
  MiniDFSCluster cluster = null;
  int numDataNodes = 2;
  Configuration conf = getConf(numDataNodes);

  try {
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
    cluster.waitActive();
    assertEquals(numDataNodes, cluster.getDataNodes().size());

    final NameNode nn = cluster.getNameNode();
    final BlockManager bm = nn.getNamesystem().getBlockManager();
    final BlockTokenSecretManager sm = bm.getBlockTokenSecretManager();

    // set a short token lifetime (1 second)
    SecurityTestUtil.setBlockTokenLifetime(sm, 1000L);
    Path fileToAppend = new Path(FILE_TO_APPEND);
    FileSystem fs = cluster.getFileSystem();

    // write a one-byte file
    FSDataOutputStream stm = writeFile(fs, fileToAppend,
        (short) numDataNodes, BLOCK_SIZE);
    stm.write(rawData, 0, 1);
    stm.close();
    // open the file again for append
    stm = fs.append(fileToAppend);
    int mid = rawData.length - 1;
    stm.write(rawData, 1, mid - 1);
    stm.hflush();

    /*
     * wait till token used in stm expires
     */
    Token<BlockTokenIdentifier> token = DFSTestUtil.getBlockToken(stm);
    while (!SecurityTestUtil.isBlockTokenExpired(token)) {
      try {
        Thread.sleep(10);
      } catch (InterruptedException ignored) {
      }
    }

    // remove a datanode to force re-establishing pipeline
    cluster.stopDataNode(0);
    // append the rest of the file
    stm.write(rawData, mid, rawData.length - mid);
    stm.close();
    // check if append is successful
    FSDataInputStream in5 = fs.open(fileToAppend);
    assertTrue(checkFile1(in5));
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 
Example 15
Source File: OfflineEditsViewerHelper.java    From RDFS with Apache License 2.0 4 votes vote down vote up
/**
 * Run file operations to create edits for all op codes
 * to be tested.
 */
private void runOperations() throws IOException {

  LOG.info("Creating edits by performing fs operations");
  // no check, if it's not it throws an exception which is what we want
  DistributedFileSystem dfs =
    (DistributedFileSystem)cluster.getFileSystem();
  // OP_ADD 0, OP_SET_GENSTAMP 10
  Path pathFileCreate = new Path("/file_create");
  FSDataOutputStream s = dfs.create(pathFileCreate);
  // OP_CLOSE 9
  s.close();
  // OP_RENAME 1
  Path pathFileMoved = new Path("/file_moved");
  dfs.rename(pathFileCreate, pathFileMoved);
  // OP_DELETE 2
  dfs.delete(pathFileMoved, false);
  // OP_MKDIR 3
  Path pathDirectoryMkdir = new Path("/directory_mkdir");
  dfs.mkdirs(pathDirectoryMkdir);
  // OP_SET_REPLICATION 4
  s = dfs.create(pathFileCreate);
  s.close();
  dfs.setReplication(pathFileCreate, (short)1);
  // OP_SET_PERMISSIONS 7
  Short permission = 0777;
  dfs.setPermission(pathFileCreate, new FsPermission(permission));
  // OP_SET_OWNER 8
  dfs.setOwner(pathFileCreate, new String("newOwner"), null);
  // OP_CLOSE 9 see above
  // OP_SET_GENSTAMP 10 see above
  // OP_SET_NS_QUOTA 11 obsolete
  // OP_CLEAR_NS_QUOTA 12 obsolete
  // OP_TIMES 13
  long mtime = 1285195527000L; // Wed, 22 Sep 2010 22:45:27 GMT
  long atime = mtime;
  dfs.setTimes(pathFileCreate, mtime, atime);
  // OP_SET_QUOTA 14
  dfs.setQuota(pathDirectoryMkdir, 1000L, FSConstants.QUOTA_DONT_SET);
  // OP_CONCAT_DELETE 16
  Path   pathConcatTarget = new Path("/file_concat_target");
  Path[] pathConcatFiles  = new Path[2];
  pathConcatFiles[0]      = new Path("/file_concat_0");
  pathConcatFiles[1]      = new Path("/file_concat_1");

  long  length      = blockSize * 3; // multiple of blocksize for concat
  short replication = 1;
  long  seed        = 1;

  DFSTestUtil.createFile(dfs, pathConcatTarget, length, replication, seed);
  DFSTestUtil.createFile(dfs, pathConcatFiles[0], length, replication, seed);
  DFSTestUtil.createFile(dfs, pathConcatFiles[1], length, replication, seed);
  dfs.concat(pathConcatTarget, pathConcatFiles, false);

  // sync to disk, otherwise we parse partial edits
  cluster.getNameNode().getFSImage().getEditLog().logSync();
  dfs.close();
}
 
Example 16
Source File: TestDataTransferProtocol.java    From RDFS with Apache License 2.0 4 votes vote down vote up
void createFile(FileSystem fs, Path path, int fileLen) throws IOException {
  byte [] arr = new byte[fileLen];
  FSDataOutputStream out = fs.create(path);
  out.write(arr);
  out.close();
}
 
Example 17
Source File: TestChecksumFileSystem.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testVerifyChecksum() throws Exception {    
  Path testPath = new Path(TEST_ROOT_DIR, "testPath");
  Path testPath11 = new Path(TEST_ROOT_DIR, "testPath11");
  FSDataOutputStream fout = localFs.create(testPath);
  fout.write("testing".getBytes());
  fout.close();
  
  fout = localFs.create(testPath11);
  fout.write("testing you".getBytes());
  fout.close();

  // Exercise some boundary cases - a divisor of the chunk size
  // the chunk size, 2x chunk size, and +/-1 around these.
  readFile(localFs, testPath, 128);
  readFile(localFs, testPath, 511);
  readFile(localFs, testPath, 512);
  readFile(localFs, testPath, 513);
  readFile(localFs, testPath, 1023);
  readFile(localFs, testPath, 1024);
  readFile(localFs, testPath, 1025);

  localFs.delete(localFs.getChecksumFile(testPath), true);
  assertTrue("checksum deleted", !localFs.exists(localFs.getChecksumFile(testPath)));
  
  //copying the wrong checksum file
  FileUtil.copy(localFs, localFs.getChecksumFile(testPath11), localFs, 
      localFs.getChecksumFile(testPath),false,true,localFs.getConf());
  assertTrue("checksum exists", localFs.exists(localFs.getChecksumFile(testPath)));
  
  boolean errorRead = false;
  try {
    readFile(localFs, testPath, 1024);
  }catch(ChecksumException ie) {
    errorRead = true;
  }
  assertTrue("error reading", errorRead);
  
  //now setting verify false, the read should succeed
  localFs.setVerifyChecksum(false);
  String str = readFile(localFs, testPath, 1024).toString();
  assertTrue("read", "testing".equals(str));
}
 
Example 18
Source File: Credentials.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public void writeTokenStorageFile(Path filename, 
                                  Configuration conf) throws IOException {
  FSDataOutputStream os = filename.getFileSystem(conf).create(filename);
  writeTokenStorageToStream(os);
  os.close();
}
 
Example 19
Source File: DefaultSorter.java    From tez with Apache License 2.0 4 votes vote down vote up
/**
 * Handles the degenerate case where serialization fails to fit in
 * the in-memory buffer, so we must spill the record from collect
 * directly to a spill file. Consider this "losing".
 */
private void spillSingleRecord(final Object key, final Object value,
                               int partition) throws IOException {
  long size = kvbuffer.length + partitions * APPROX_HEADER_LENGTH;
  FSDataOutputStream out = null;
  try {
    // create spill file
    final TezSpillRecord spillRec = new TezSpillRecord(partitions);
    final Path filename =
        mapOutputFile.getSpillFileForWrite(numSpills, size);
    spillFilePaths.put(numSpills, filename);
    out = rfs.create(filename);
    ensureSpillFilePermissions(filename, rfs);

    // we don't run the combiner for a single record
    for (int i = 0; i < partitions; ++i) {
      IFile.Writer writer = null;
      try {
        long segmentStart = out.getPos();
        // Create a new codec, don't care!
        if (!sendEmptyPartitionDetails || (i == partition)) {
          writer = new Writer(conf, out, keyClass, valClass, codec,
              spilledRecordsCounter, null, false);
        }
        if (i == partition) {
          final long recordStart = out.getPos();
          writer.append(key, value);
          // Note that our map byte count will not be accurate with
          // compression
          mapOutputByteCounter.increment(out.getPos() - recordStart);
        }
        long rawLength =0;
        long partLength =0;
        if (writer != null) {
          writer.close();
          rawLength = writer.getRawLength();
          partLength = writer.getCompressedLength();
        }
        adjustSpillCounters(rawLength, partLength);

        // record offsets
        TezIndexRecord rec = new TezIndexRecord(segmentStart, rawLength, partLength);
        spillRec.putIndex(rec, i);

        writer = null;
      } catch (IOException e) {
        if (null != writer) writer.close();
        throw e;
      }
    }
    if (totalIndexCacheMemory >= indexCacheMemoryLimit) {
      // create spill index file
      Path indexFilename =
          mapOutputFile.getSpillIndexFileForWrite(numSpills, partitions
              * MAP_OUTPUT_INDEX_RECORD_LENGTH);
      spillFileIndexPaths.put(numSpills, indexFilename);
      spillRec.writeToFile(indexFilename, conf, localFs);
    } else {
      indexCacheList.add(spillRec);
      totalIndexCacheMemory +=
        spillRec.size() * MAP_OUTPUT_INDEX_RECORD_LENGTH;
    }
    ++numSpills;
    if (!isFinalMergeEnabled()) {
      numShuffleChunks.setValue(numSpills);
    } else if (numSpills > 1) {
      //Increment only when there is atleast one previous spill
      numAdditionalSpills.increment(1);
    }
  } finally {
    if (out != null) out.close();
  }
}
 
Example 20
Source File: SwiftBaseTest.java    From stocator with Apache License 2.0 2 votes vote down vote up
/**
 * Create and then close a file
 *
 * @param path path to create
 * @throws IOException on a failure
 */
protected static void createEmptyFile(Path path) throws IOException {
  FSDataOutputStream out = sFileSystem.create(path);
  out.close();
}