Java Code Examples for org.apache.hadoop.fs.FSDataOutputStream#writeChars()

The following examples show how to use org.apache.hadoop.fs.FSDataOutputStream#writeChars() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestHDFSIntegration.java    From incubator-sentry with Apache License 2.0 6 votes vote down vote up
private void loadData(Statement stmt) throws IOException, SQLException {
  FSDataOutputStream f1 = miniDFS.getFileSystem().create(new Path("/tmp/f1.txt"));
  f1.writeChars("m1d1_t1\n");
  f1.writeChars("m1d1_t2\n");
  f1.writeChars("m1d1_t3\n");
  f1.flush();
  f1.close();
  stmt.execute("load data inpath \'/tmp/f1.txt\' overwrite into table p1 partition (month=1, day=1)");
  FSDataOutputStream f2 = miniDFS.getFileSystem().create(new Path("/tmp/f2.txt"));
  f2.writeChars("m2d2_t4\n");
  f2.writeChars("m2d2_t5\n");
  f2.writeChars("m2d2_t6\n");
  f2.flush();
  f2.close();
  stmt.execute("load data inpath \'/tmp/f2.txt\' overwrite into table p1 partition (month=2, day=2)");
  ResultSet rs = stmt.executeQuery("select * from p1");
  List<String> vals = new ArrayList<String>(); 
  while (rs.next()) {
    vals.add(rs.getString(1));
  }
  Assert.assertEquals(6, vals.size());
  rs.close();
}
 
Example 2
Source File: TestHDFSIntegration.java    From incubator-sentry with Apache License 2.0 6 votes vote down vote up
private void loadDataTwoCols(Statement stmt) throws IOException, SQLException {
  FSDataOutputStream f1 = miniDFS.getFileSystem().create(new Path("/tmp/f2.txt"));
  f1.writeChars("m1d1_t1, m1d1_t2\n");
  f1.writeChars("m1d1_t2, m1d1_t2\n");
  f1.writeChars("m1d1_t3, m1d1_t2\n");
  f1.flush();
  f1.close();
  stmt.execute("load data inpath \'/tmp/f2.txt\' overwrite into table p1 partition (month=1, day=1)");
  ResultSet rs = stmt.executeQuery("select * from p1");
  List<String> vals = new ArrayList<String>();
  while (rs.next()) {
    vals.add(rs.getString(1));
  }
  Assert.assertEquals(3, vals.size());
  rs.close();
}
 
Example 3
Source File: TestFsck.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void writeFile(final DistributedFileSystem dfs,
    Path dir, String fileName) throws IOException {
  Path filePath = new Path(dir.toString() + Path.SEPARATOR + fileName);
  final FSDataOutputStream out = dfs.create(filePath);
  out.writeChars("teststring");
  out.close();
}
 
Example 4
Source File: GenReaderThread.java    From RDFS with Apache License 2.0 5 votes vote down vote up
@Override
public void output(FSDataOutputStream out) throws IOException {
  out.writeChars("Number of open files:\t\t\t" + total_open_files + "\n");
  out.writeChars("Number of corrupt dirs:\t\t\t" + corrupt_dir.size() + "\n");
  if (corrupt_dir.size() > 0) {
    out.writeChars("-----------------------------\n");
    out.writeChars("Corrupt Dirs:\n");
    out.writeChars("-----------------------------\n");
    for (String file : corrupt_dir) {
      out.writeChars(file + "\n");
    }
  }
}
 
Example 5
Source File: TestTezJobs.java    From tez with Apache License 2.0 5 votes vote down vote up
public static void generateOrderedWordCountInput(Path inputDir, FileSystem fs) throws IOException {
  Path dataPath1 = new Path(inputDir, "inPath1");
  Path dataPath2 = new Path(inputDir, "inPath2");

  FSDataOutputStream f1 = null;
  FSDataOutputStream f2 = null;
  try {
    f1 = fs.create(dataPath1);
    f2 = fs.create(dataPath2);

    final String prefix = "a";
    for (int i = 1; i <= 10; ++i) {
      final String word = prefix + "_" + i;
      for (int j = 10; j >= i; --j) {
        LOG.info("Writing " + word + " to input files");
        f1.write(word.getBytes());
        f1.writeChars("\t");
        f2.write(word.getBytes());
        f2.writeChars("\t");
      }
    }
    f1.hsync();
    f2.hsync();
  } finally {
    if (f1 != null) {
      f1.close();
    }
    if (f2 != null) {
      f2.close();
    }
  }
}
 
Example 6
Source File: SandboxTester.java    From sequenceiq-samples with Apache License 2.0 5 votes vote down vote up
private static final void testHdfs(Configuration configuration, String fileName, String content) throws IOException {
	Path hdfsFile = new Path(fileName);
	FileSystem fs = FileSystem.get(hdfsFile.toUri(), configuration);
	FSDataOutputStream out = fs.create(hdfsFile);
	try {
		out.writeChars(content);
	} finally {
		Closeables.close(out, false);
	}
}
 
Example 7
Source File: TestHDFSIntegration.java    From incubator-sentry with Apache License 2.0 5 votes vote down vote up
private void writeToPath(String path, int numRows, String user, String group) throws IOException {
    Path p = new Path(path);
    miniDFS.getFileSystem().mkdirs(p);
    miniDFS.getFileSystem().setOwner(p, user, group);
//    miniDFS.getFileSystem().setPermission(p, FsPermission.valueOf("-rwxrwx---"));
    FSDataOutputStream f1 = miniDFS.getFileSystem().create(new Path(path + "/stuff.txt"));
    for (int i = 0; i < numRows; i++) {
      f1.writeChars("random" + i + "\n");
    }
    f1.flush();
    f1.close();
    miniDFS.getFileSystem().setOwner(new Path(path + "/stuff.txt"), "asuresh", "supergroup");
    miniDFS.getFileSystem().setPermission(new Path(path + "/stuff.txt"), FsPermission.valueOf("-rwxrwx---"));
  }
 
Example 8
Source File: TestMover.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testMoverFailedRetry() throws Exception {
  // HDFS-8147
  final Configuration conf = new HdfsConfiguration();
  conf.set(DFSConfigKeys.DFS_MOVER_RETRY_MAX_ATTEMPTS_KEY, "2");
  final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
      .numDataNodes(3)
      .storageTypes(
          new StorageType[][] {{StorageType.DISK, StorageType.ARCHIVE},
              {StorageType.DISK, StorageType.ARCHIVE},
              {StorageType.DISK, StorageType.ARCHIVE}}).build();
  try {
    cluster.waitActive();
    final DistributedFileSystem dfs = cluster.getFileSystem();
    final String file = "/testMoverFailedRetry";
    // write to DISK
    final FSDataOutputStream out = dfs.create(new Path(file), (short) 2);
    out.writeChars("testMoverFailedRetry");
    out.close();

    // Delete block file so, block move will fail with FileNotFoundException
    LocatedBlock lb = dfs.getClient().getLocatedBlocks(file, 0).get(0);
    cluster.corruptBlockOnDataNodesByDeletingBlockFile(lb.getBlock());
    // move to ARCHIVE
    dfs.setStoragePolicy(new Path(file), "COLD");
    int rc = ToolRunner.run(conf, new Mover.Cli(),
        new String[] {"-p", file.toString()});
    Assert.assertEquals("Movement should fail after some retry",
        ExitStatus.IO_EXCEPTION.getExitCode(), rc);
  } finally {
    cluster.shutdown();
  }
}
 
Example 9
Source File: TestMover.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testScheduleSameBlock() throws IOException {
  final Configuration conf = new HdfsConfiguration();
  final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
      .numDataNodes(4).build();
  try {
    cluster.waitActive();
    final DistributedFileSystem dfs = cluster.getFileSystem();
    final String file = "/testScheduleSameBlock/file";
    
    {
      final FSDataOutputStream out = dfs.create(new Path(file));
      out.writeChars("testScheduleSameBlock");
      out.close();
    }

    final Mover mover = newMover(conf);
    mover.init();
    final Mover.Processor processor = mover.new Processor();

    final LocatedBlock lb = dfs.getClient().getLocatedBlocks(file, 0).get(0);
    final List<MLocation> locations = MLocation.toLocations(lb);
    final MLocation ml = locations.get(0);
    final DBlock db = mover.newDBlock(lb.getBlock().getLocalBlock(), locations);

    final List<StorageType> storageTypes = new ArrayList<StorageType>(
        Arrays.asList(StorageType.DEFAULT, StorageType.DEFAULT));
    Assert.assertTrue(processor.scheduleMoveReplica(db, ml, storageTypes));
    Assert.assertFalse(processor.scheduleMoveReplica(db, ml, storageTypes));
  } finally {
    cluster.shutdown();
  }
}
 
Example 10
Source File: TestCombineFileInputFormat.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Test when input files are from non-default file systems
 */
@Test
public void testForNonDefaultFileSystem() throws Throwable {
  Configuration conf = new Configuration();

  // use a fake file system scheme as default
  conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, DUMMY_FS_URI);

  // default fs path
  assertEquals(DUMMY_FS_URI, FileSystem.getDefaultUri(conf).toString());
  // add a local file
  Path localPath = new Path("testFile1");
  FileSystem lfs = FileSystem.getLocal(conf);
  FSDataOutputStream dos = lfs.create(localPath);
  dos.writeChars("Local file for CFIF");
  dos.close();

  Job job = Job.getInstance(conf);
  FileInputFormat.setInputPaths(job, lfs.makeQualified(localPath));
  DummyInputFormat inFormat = new DummyInputFormat();
  List<InputSplit> splits = inFormat.getSplits(job);
  assertTrue(splits.size() > 0);
  for (InputSplit s : splits) {
    CombineFileSplit cfs = (CombineFileSplit)s;
    for (Path p : cfs.getPaths()) {
      assertEquals(p.toUri().getScheme(), "file");
    }
  }
}
 
Example 11
Source File: TestFsck.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void writeFile(final DistributedFileSystem dfs,
    Path dir, String fileName) throws IOException {
  Path filePath = new Path(dir.toString() + Path.SEPARATOR + fileName);
  final FSDataOutputStream out = dfs.create(filePath);
  out.writeChars("teststring");
  out.close();
}
 
Example 12
Source File: TestMover.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testMoverFailedRetry() throws Exception {
  // HDFS-8147
  final Configuration conf = new HdfsConfiguration();
  conf.set(DFSConfigKeys.DFS_MOVER_RETRY_MAX_ATTEMPTS_KEY, "2");
  final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
      .numDataNodes(3)
      .storageTypes(
          new StorageType[][] {{StorageType.DISK, StorageType.ARCHIVE},
              {StorageType.DISK, StorageType.ARCHIVE},
              {StorageType.DISK, StorageType.ARCHIVE}}).build();
  try {
    cluster.waitActive();
    final DistributedFileSystem dfs = cluster.getFileSystem();
    final String file = "/testMoverFailedRetry";
    // write to DISK
    final FSDataOutputStream out = dfs.create(new Path(file), (short) 2);
    out.writeChars("testMoverFailedRetry");
    out.close();

    // Delete block file so, block move will fail with FileNotFoundException
    LocatedBlock lb = dfs.getClient().getLocatedBlocks(file, 0).get(0);
    cluster.corruptBlockOnDataNodesByDeletingBlockFile(lb.getBlock());
    // move to ARCHIVE
    dfs.setStoragePolicy(new Path(file), "COLD");
    int rc = ToolRunner.run(conf, new Mover.Cli(),
        new String[] {"-p", file.toString()});
    Assert.assertEquals("Movement should fail after some retry",
        ExitStatus.IO_EXCEPTION.getExitCode(), rc);
  } finally {
    cluster.shutdown();
  }
}
 
Example 13
Source File: TestMover.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testScheduleSameBlock() throws IOException {
  final Configuration conf = new HdfsConfiguration();
  final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
      .numDataNodes(4).build();
  try {
    cluster.waitActive();
    final DistributedFileSystem dfs = cluster.getFileSystem();
    final String file = "/testScheduleSameBlock/file";
    
    {
      final FSDataOutputStream out = dfs.create(new Path(file));
      out.writeChars("testScheduleSameBlock");
      out.close();
    }

    final Mover mover = newMover(conf);
    mover.init();
    final Mover.Processor processor = mover.new Processor();

    final LocatedBlock lb = dfs.getClient().getLocatedBlocks(file, 0).get(0);
    final List<MLocation> locations = MLocation.toLocations(lb);
    final MLocation ml = locations.get(0);
    final DBlock db = mover.newDBlock(lb.getBlock().getLocalBlock(), locations);

    final List<StorageType> storageTypes = new ArrayList<StorageType>(
        Arrays.asList(StorageType.DEFAULT, StorageType.DEFAULT));
    Assert.assertTrue(processor.scheduleMoveReplica(db, ml, storageTypes));
    Assert.assertFalse(processor.scheduleMoveReplica(db, ml, storageTypes));
  } finally {
    cluster.shutdown();
  }
}
 
Example 14
Source File: TestCombineFileInputFormat.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Test when input files are from non-default file systems
 */
@Test
public void testForNonDefaultFileSystem() throws Throwable {
  Configuration conf = new Configuration();

  // use a fake file system scheme as default
  conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, DUMMY_FS_URI);

  // default fs path
  assertEquals(DUMMY_FS_URI, FileSystem.getDefaultUri(conf).toString());
  // add a local file
  Path localPath = new Path("testFile1");
  FileSystem lfs = FileSystem.getLocal(conf);
  FSDataOutputStream dos = lfs.create(localPath);
  dos.writeChars("Local file for CFIF");
  dos.close();

  Job job = Job.getInstance(conf);
  FileInputFormat.setInputPaths(job, lfs.makeQualified(localPath));
  DummyInputFormat inFormat = new DummyInputFormat();
  List<InputSplit> splits = inFormat.getSplits(job);
  assertTrue(splits.size() > 0);
  for (InputSplit s : splits) {
    CombineFileSplit cfs = (CombineFileSplit)s;
    for (Path p : cfs.getPaths()) {
      assertEquals(p.toUri().getScheme(), "file");
    }
  }
}
 
Example 15
Source File: PersistedHDFSManager.java    From Knowage-Server with GNU Affero General Public License v3.0 5 votes vote down vote up
private void appendObjectWithCast(FSDataOutputStream fsOS, Object value, Class clz) {
	try {
		if (value == null) {
			fsOS.writeChars("\"NULL\"");
			return;
		}
		Class objClz = value.getClass();
		if (clz.equals(String.class)) {
			fsOS.writeChars("\"" + (String) value + "\"");
		} else if (clz.equals(Integer.class)) {
			fsOS.writeChars(value.toString());
		} else if (clz.equals(Double.class)) {
			fsOS.writeChars(value.toString());
		} else if (clz.equals(Long.class)) {
			fsOS.writeChars(value.toString());
		} else if (clz.equals(Date.class)) {
			Date dt = (Date) value;
			fsOS.writeChars("\"" + dt.toString() + "\"");
		} else if (clz.equals(Timestamp.class)) {
			Timestamp ts = (Timestamp) value;
			fsOS.writeChars("\"" + ts.toString() + "\"");
		} else if (clz.equals(Boolean.class)) {
			fsOS.writeChars(value.toString());
		} else {
			fsOS.writeChars((String) value);
		}
	} catch (IOException e) {
		logger.error("Impossible to write on hdfs, error during casting object for writing ");
		throw new SpagoBIRuntimeException("Impossible to write on hdfs, error during casting object for writing " + e);
	}
}
 
Example 16
Source File: HCatalogTestUtils.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 5 votes vote down vote up
private void createInputFile(Path path, int rowCount)
  throws IOException {
  if (fs.exists(path)) {
    fs.delete(path, true);
  }
  FSDataOutputStream os = fs.create(path);
  for (int i = 0; i < rowCount; i++) {
    String s = i + "\n";
    os.writeChars(s);
  }
  os.close();
}
 
Example 17
Source File: TestRecoveryHdfs.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Test
public void testTruncatedLog() throws Exception {
  try {
    TestInjection.skipIndexWriterCommitOnClose = true;
    final Semaphore logReplay = new Semaphore(0);
    final Semaphore logReplayFinish = new Semaphore(0);

    UpdateLog.testing_logReplayHook = () -> {
      try {
        assertTrue(logReplay.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
      } catch (Exception e) {
        throw new RuntimeException(e);
      }
    };

    UpdateLog.testing_logReplayFinishHook = () -> logReplayFinish.release();

    String logDir = h.getCore().getUpdateHandler().getUpdateLog().getLogDir();

    clearIndex();
    assertU(commit());

    assertU(adoc("id","F1"));
    assertU(adoc("id","F2"));
    assertU(adoc("id","F3"));
    
    h.close();
    

    
    String[] files = HdfsUpdateLog.getLogList(fs, new Path(logDir));
    Arrays.sort(files);

    FSDataOutputStream dos = fs.append(new Path(logDir, files[files.length-1]));
  
    dos.writeLong(0xffffffffffffffffL);
    dos.writeChars("This should be appended to a good log file, representing a bad partially written record.");
    dos.close();

    logReplay.release(1000);
    logReplayFinish.drainPermits();
    ignoreException("OutOfBoundsException");  // this is what the corrupted log currently produces... subject to change.
    createCore();
    assertTrue(logReplayFinish.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
    resetExceptionIgnores();
    assertJQ(req("q","*:*") ,"/response/numFound==3");

    //
    // Now test that the bad log file doesn't mess up retrieving latest versions
    //

    updateJ(jsonAdd(sdoc("id","F4", "_version_","104")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
    updateJ(jsonAdd(sdoc("id","F5", "_version_","105")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
    updateJ(jsonAdd(sdoc("id","F6", "_version_","106")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));

    // This currently skips the bad log file and also returns the version of the clearIndex (del *:*)
    // assertJQ(req("qt","/get", "getVersions","6"), "/versions==[106,105,104]");
    assertJQ(req("qt","/get", "getVersions","3"), "/versions==[106,105,104]");

  } finally {
    UpdateLog.testing_logReplayHook = null;
    UpdateLog.testing_logReplayFinishHook = null;
  }
}
 
Example 18
Source File: Reduce.java    From RDFS with Apache License 2.0 4 votes vote down vote up
@Override
public void reduce(Text key, Iterator<Text> values,
		OutputCollector<Text, Text> output, Reporter reporter)
		throws IOException {
	int size = 0;

	double averageIORate = 0;
	List<Float> list = new ArrayList<Float>();

	fs = FileSystem.get(conf);
	FSDataOutputStream out;

	if (fs.exists(new Path(OUTPUT, "result-writing")))
		out = fs.create(new Path(OUTPUT, "result-reading"), true);
	else
		out = fs.create(new Path(OUTPUT, "result-writing"), true);

	String mapper = conf.get("mapred.job.name");
	long bufferSize = mapper.equals("dfstest-writing") ? Long
			.parseLong(conf.get("dfs.buffer.size.write")) : Long
			.parseLong(conf.get("dfs.buffer.size.read"));
	long blockSize = Long.parseLong(conf.get("dfs.block.size"));
	long nTasks = Long.parseLong(conf.get("dfs.nTasks"));
	long replications = Long.parseLong(conf.get("dfs.replication"));
	long nmaps = Long.parseLong(conf.get("dfs.nmaps"));


	out.writeChars("-----------------------------\n");
	out.writeChars("Number of tasks:\t" + nmaps + "\n");
	out.writeChars("Replications:\t\t" + replications + "\n");
	out.writeChars("Files per task:\t\t" + nTasks + "\n");
	out.writeChars("BlockSize:\t\t" + blockSize + "\n");
	out.writeChars("BufferSize: " + bufferSize + "\tIORate\n");
	float min = Float.MAX_VALUE;
	float max = Float.MIN_VALUE;

	// TODO Auto-generated method stub
	for (; values.hasNext();) {
		size++;
		// tokens.nextToken(); there is only one value per line
		double ioRate = Double.parseDouble(values.next().toString());
		if (ioRate > max)
			max = (float) ioRate;
		if (ioRate < min)
			min = (float) ioRate;
		list.add((float) ioRate);
		// this is for testing
		// output.collect(new Text(String.valueOf(bufferSize)), new
		// Text(
		// String.valueOf(ioRate)));
		// out.writeChars(bufferSize + " bytes\t\t" + ioRate +
		// " Mb/s\n");
		averageIORate += ioRate;
	}
	out.writeChars("Min\t\t\t" + min + "\n");
	out.writeChars("Max\t\t\t" + max + "\n");
	averageIORate /= size;
	float temp = (float) 0.0;
	for (int i = 0; i < list.size(); i++) {
		temp += Math.pow(list.get(i) - averageIORate, 2);
	}
	out.writeChars("Average\t\t\t: " + averageIORate + "\n");
	float dev = (float) Math.sqrt(temp / size);
	out.writeChars("Std. dev\t\t: " + dev + "\n");
	out.close();
}
 
Example 19
Source File: GenReduce.java    From RDFS with Apache License 2.0 4 votes vote down vote up
@Override
public void reduce(Text key, Iterator<Text> values,
    OutputCollector<Text, Text> output, Reporter reporter)
    throws IOException {
  FSDataOutputStream out = null;
  try {
    int size = 0;
    FileSystem fs = FileSystem.get(conf);
    double averageIORate = 0;
    List<Float> list = new ArrayList<Float>();
    
    String output_dir = conf.get(OUTPUT_DIR_KEY);
    Path result_file = new Path(output_dir, "results");
    if (fs.exists(result_file)) {
      fs.delete(result_file);
    }
    out = fs.create(result_file, true);
  
    long nmaps = conf.getLong(NUMBER_OF_MAPS_KEY, NMAPS);
    long nthreads = conf.getLong(NUMBER_OF_THREADS_KEY, NTHREADS);

    out.writeChars("-----------------------------\n");
    out.writeChars("Number of mapper :\t\t\t" + nmaps + "\n");
    out.writeChars("Number of threads :\t\t\t" + nthreads + "\n");
  
    float min = Float.MAX_VALUE;
    float max = Float.MIN_VALUE;
    Class<?> clazz = conf.getClass(THREAD_CLASS_KEY, null);
    if (clazz == null) {
      throw new IOException("Class " + conf.get(THREAD_CLASS_KEY) + " not found");
    }
    GenThread t = (GenThread)ReflectionUtils.newInstance(clazz, conf);
    t.reset();
    long total_files = 0;
    long total_processed_size = 0;
    long total_num_errors = 0;
    String total_error = "";
    TypeReference<Map<String, String>> type = 
        new TypeReference<Map<String, String>>() { };
    while (values.hasNext()) {
      Map<String, String> stat = mapper.readValue(values.next().toString(), type);
      size++;
      total_files += Long.parseLong(stat.get("files"));
      total_processed_size += Long.parseLong(stat.get("size"));
      total_num_errors += Long.parseLong(stat.get("nerrors"));
      total_error += stat.get("errors");
      double ioRate = Double.parseDouble(stat.get("rate"));
      if (ioRate > max)
        max = (float) ioRate;
      if (ioRate < min)
        min = (float) ioRate;
      list.add((float) ioRate);
      averageIORate += ioRate;
      t.analyze(stat);
    }
  
    out.writeChars("Number of files processed:\t\t" + total_files + "\n");
    out.writeChars("Number of size processed:\t\t" + total_processed_size + "\n");
    out.writeChars("Min IO Rate(MB/sec): \t\t\t" + min + "\n");
    out.writeChars("Max IO Rate(MB/sec): \t\t\t" + max + "\n");
    averageIORate /= size;
    float temp = (float) 0.0;
    for (int i = 0; i < list.size(); i++) {
      temp += Math.pow(list.get(i) - averageIORate, 2);
    }
    out.writeChars("Average(MB/sec): \t\t\t" + averageIORate + "\n");
    float dev = (float) Math.sqrt(temp / size);
    out.writeChars("Std. dev: \t\t\t\t" + dev + "\n");
    out.writeChars("Total number of errors:\t\t\t" + total_num_errors + "\n");
    out.writeChars(total_error);
    t.output(out);
  } catch (IOException e) {
    LOG.error("Error:", e);
    throw e;
  } finally {
    out.close();
    
  }
}
 
Example 20
Source File: DirReduce.java    From RDFS with Apache License 2.0 4 votes vote down vote up
@Override
public void reduce(Text key, Iterator<Text> values,
    OutputCollector<Text, Text> output, Reporter reporter)
    throws IOException {
  int size = 0;

  double averageIORate = 0;
  List<Float> list = new ArrayList<Float>();

  fs = FileSystem.get(conf);
  FSDataOutputStream out;

  if (fs.exists(new Path(OUTPUT, "result-writing")))
    out = fs.create(new Path(OUTPUT, "result-reading"), true);
  else
    out = fs.create(new Path(OUTPUT, "result-writing"), true);

  long nTasks = Long.parseLong(conf.get("dfs.nTasks"));
  long nmaps = Long.parseLong(conf.get("dfs.nmaps"));


  out.writeChars("-----------------------------\n");
  out.writeChars("Number of tasks:\t" + nmaps + "\n");
  out.writeChars("Files per task:\t\t" + nTasks + "\n");
  float min = Float.MAX_VALUE;
  float max = Float.MIN_VALUE;

  // TODO Auto-generated method stub
  for (; values.hasNext();) {
    size++;
    // tokens.nextToken(); there is only one value per line
    double ioRate = Double.parseDouble(values.next().toString());
    if (ioRate > max)
      max = (float) ioRate;
    if (ioRate < min)
      min = (float) ioRate;
    list.add((float) ioRate);
    // this is for testing
    // output.collect(new Text(String.valueOf(bufferSize)), new
    // Text(
    // String.valueOf(ioRate)));
    // out.writeChars(bufferSize + " bytes\t\t" + ioRate +
    // " Mb/s\n");
    averageIORate += ioRate;
  }
  out.writeChars("Min\t\t\t" + min + "\n");
  out.writeChars("Max\t\t\t" + max + "\n");
  averageIORate /= size;
  float temp = (float) 0.0;
  for (int i = 0; i < list.size(); i++) {
    temp += Math.pow(list.get(i) - averageIORate, 2);
  }
  out.writeChars("Average\t\t\t: " + averageIORate + "\n");
  float dev = (float) Math.sqrt(temp / size);
  out.writeChars("Std. dev\t\t: " + dev + "\n");
  out.close();
}