Java Code Examples for org.apache.hadoop.mapreduce.FileSystemCounter

The following examples show how to use org.apache.hadoop.mapreduce.FileSystemCounter. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: TestCounters.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("deprecation")
private void checkLegacyNames(Counters counters) {
  assertEquals("New name", 1, counters.findCounter(
      TaskCounter.class.getName(), "MAP_INPUT_RECORDS").getValue());
  assertEquals("Legacy name", 1, counters.findCounter(
      "org.apache.hadoop.mapred.Task$Counter",
      "MAP_INPUT_RECORDS").getValue());
  assertEquals("Legacy enum", 1,
      counters.findCounter(Task.Counter.MAP_INPUT_RECORDS).getValue());

  assertEquals("New name", 1, counters.findCounter(
      JobCounter.class.getName(), "DATA_LOCAL_MAPS").getValue());
  assertEquals("Legacy name", 1, counters.findCounter(
      "org.apache.hadoop.mapred.JobInProgress$Counter",
      "DATA_LOCAL_MAPS").getValue());
  assertEquals("Legacy enum", 1,
      counters.findCounter(JobInProgress.Counter.DATA_LOCAL_MAPS).getValue());

  assertEquals("New name", 1, counters.findCounter(
      FileSystemCounter.class.getName(), "FILE_BYTES_READ").getValue());
  assertEquals("New name and method", 1, counters.findCounter("file",
      FileSystemCounter.BYTES_READ).getValue());
  assertEquals("Legacy name", 1, counters.findCounter(
      "FileSystemCounters",
      "FILE_BYTES_READ").getValue());
}
 
Example 2
Source Project: hadoop   Source File: TestCounters.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFileSystemGroupIteratorConcurrency() {
  Counters counters = new Counters();
  // create 2 filesystem counter groups
  counters.findCounter("fs1", FileSystemCounter.BYTES_READ).increment(1);
  counters.findCounter("fs2", FileSystemCounter.BYTES_READ).increment(1);
  
  // Iterate over the counters in this group while updating counters in
  // the group
  Group group = counters.getGroup(FileSystemCounter.class.getName());
  Iterator<Counter> iterator = group.iterator();
  counters.findCounter("fs3", FileSystemCounter.BYTES_READ).increment(1);
  assertTrue(iterator.hasNext());
  iterator.next();
  counters.findCounter("fs3", FileSystemCounter.BYTES_READ).increment(1);
  assertTrue(iterator.hasNext());
  iterator.next();
}
 
Example 3
Source Project: hadoop   Source File: TestCounters.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testLegacyGetGroupNames() {
  Counters counters = new Counters();
  // create 2 filesystem counter groups
  counters.findCounter("fs1", FileSystemCounter.BYTES_READ).increment(1);
  counters.findCounter("fs2", FileSystemCounter.BYTES_READ).increment(1);
  counters.incrCounter("group1", "counter1", 1);
  
  HashSet<String> groups = new HashSet<String>(counters.getGroupNames());
  HashSet<String> expectedGroups = new HashSet<String>();
  expectedGroups.add("group1");
  expectedGroups.add("FileSystemCounters"); //Legacy Name
  expectedGroups.add("org.apache.hadoop.mapreduce.FileSystemCounter");

  assertEquals(expectedGroups, groups);
}
 
Example 4
Source Project: big-c   Source File: TestCounters.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("deprecation")
private void checkLegacyNames(Counters counters) {
  assertEquals("New name", 1, counters.findCounter(
      TaskCounter.class.getName(), "MAP_INPUT_RECORDS").getValue());
  assertEquals("Legacy name", 1, counters.findCounter(
      "org.apache.hadoop.mapred.Task$Counter",
      "MAP_INPUT_RECORDS").getValue());
  assertEquals("Legacy enum", 1,
      counters.findCounter(Task.Counter.MAP_INPUT_RECORDS).getValue());

  assertEquals("New name", 1, counters.findCounter(
      JobCounter.class.getName(), "DATA_LOCAL_MAPS").getValue());
  assertEquals("Legacy name", 1, counters.findCounter(
      "org.apache.hadoop.mapred.JobInProgress$Counter",
      "DATA_LOCAL_MAPS").getValue());
  assertEquals("Legacy enum", 1,
      counters.findCounter(JobInProgress.Counter.DATA_LOCAL_MAPS).getValue());

  assertEquals("New name", 1, counters.findCounter(
      FileSystemCounter.class.getName(), "FILE_BYTES_READ").getValue());
  assertEquals("New name and method", 1, counters.findCounter("file",
      FileSystemCounter.BYTES_READ).getValue());
  assertEquals("Legacy name", 1, counters.findCounter(
      "FileSystemCounters",
      "FILE_BYTES_READ").getValue());
}
 
Example 5
Source Project: big-c   Source File: TestCounters.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFileSystemGroupIteratorConcurrency() {
  Counters counters = new Counters();
  // create 2 filesystem counter groups
  counters.findCounter("fs1", FileSystemCounter.BYTES_READ).increment(1);
  counters.findCounter("fs2", FileSystemCounter.BYTES_READ).increment(1);
  
  // Iterate over the counters in this group while updating counters in
  // the group
  Group group = counters.getGroup(FileSystemCounter.class.getName());
  Iterator<Counter> iterator = group.iterator();
  counters.findCounter("fs3", FileSystemCounter.BYTES_READ).increment(1);
  assertTrue(iterator.hasNext());
  iterator.next();
  counters.findCounter("fs3", FileSystemCounter.BYTES_READ).increment(1);
  assertTrue(iterator.hasNext());
  iterator.next();
}
 
Example 6
Source Project: big-c   Source File: TestCounters.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testLegacyGetGroupNames() {
  Counters counters = new Counters();
  // create 2 filesystem counter groups
  counters.findCounter("fs1", FileSystemCounter.BYTES_READ).increment(1);
  counters.findCounter("fs2", FileSystemCounter.BYTES_READ).increment(1);
  counters.incrCounter("group1", "counter1", 1);
  
  HashSet<String> groups = new HashSet<String>(counters.getGroupNames());
  HashSet<String> expectedGroups = new HashSet<String>();
  expectedGroups.add("group1");
  expectedGroups.add("FileSystemCounters"); //Legacy Name
  expectedGroups.add("org.apache.hadoop.mapreduce.FileSystemCounter");

  assertEquals(expectedGroups, groups);
}
 
Example 7
Source Project: circus-train   Source File: DistCpCopier.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Metrics copy() throws CircusTrainException {
  LOG.info("Copying table data.");
  LOG.debug("Invoking DistCp: {} -> {}", sourceDataBaseLocation, replicaDataLocation);

  DistCpOptions distCpOptions = parseCopierOptions(copierOptions);
  LOG.debug("Invoking DistCp with options: {}", distCpOptions);

  CircusTrainCopyListing.setAsCopyListingClass(conf);
  CircusTrainCopyListing.setRootPath(conf, sourceDataBaseLocation);

  try {
    distCpOptions.setBlocking(false);
    Job job = executor.exec(conf, distCpOptions);
    String counter = String
        .format("%s_BYTES_WRITTEN", replicaDataLocation.toUri().getScheme().toUpperCase(Locale.ROOT));
    registerRunningJobMetrics(job, counter);
    if (!job.waitForCompletion(true)) {
      throw new IOException(
          "DistCp failure: Job " + job.getJobID() + " has failed: " + job.getStatus().getFailureInfo());
    }

    return new JobMetrics(job, FileSystemCounter.class.getName(), counter);
  } catch (Exception e) {
    cleanUpReplicaDataLocation();
    throw new CircusTrainException("Unable to copy file(s)", e);
  }
}
 
Example 8
Source Project: hadoop   Source File: Task.java    License: Apache License 2.0 5 votes vote down vote up
void updateCounters() {
  if (readBytesCounter == null) {
    readBytesCounter = counters.findCounter(scheme,
        FileSystemCounter.BYTES_READ);
  }
  if (writeBytesCounter == null) {
    writeBytesCounter = counters.findCounter(scheme,
        FileSystemCounter.BYTES_WRITTEN);
  }
  if (readOpsCounter == null) {
    readOpsCounter = counters.findCounter(scheme,
        FileSystemCounter.READ_OPS);
  }
  if (largeReadOpsCounter == null) {
    largeReadOpsCounter = counters.findCounter(scheme,
        FileSystemCounter.LARGE_READ_OPS);
  }
  if (writeOpsCounter == null) {
    writeOpsCounter = counters.findCounter(scheme,
        FileSystemCounter.WRITE_OPS);
  }
  long readBytes = 0;
  long writeBytes = 0;
  long readOps = 0;
  long largeReadOps = 0;
  long writeOps = 0;
  for (FileSystem.Statistics stat: stats) {
    readBytes = readBytes + stat.getBytesRead();
    writeBytes = writeBytes + stat.getBytesWritten();
    readOps = readOps + stat.getReadOps();
    largeReadOps = largeReadOps + stat.getLargeReadOps();
    writeOps = writeOps + stat.getWriteOps();
  }
  readBytesCounter.setValue(readBytes);
  writeBytesCounter.setValue(writeBytes);
  readOpsCounter.setValue(readOps);
  largeReadOpsCounter.setValue(largeReadOps);
  writeOpsCounter.setValue(writeOps);
}
 
Example 9
Source Project: hadoop   Source File: AbstractCounters.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Find the file system counter for the given scheme and enum.
 * @param scheme of the file system
 * @param key the enum of the counter
 * @return the file system counter
 */
@InterfaceAudience.Private
public synchronized C findCounter(String scheme, FileSystemCounter key) {
  return ((FileSystemCounterGroup<C>) getGroup(
      FileSystemCounter.class.getName()).getUnderlyingGroup()).
      findCounter(scheme, key);
}
 
Example 10
Source Project: hadoop   Source File: FileSystemCounterGroup.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public C findCounter(String counterName, boolean create) {
  try {
    String[] pair = parseCounterName(counterName);
    return findCounter(pair[0], FileSystemCounter.valueOf(pair[1]));
  }
  catch (Exception e) {
    if (create) throw new IllegalArgumentException(e);
    LOG.warn(counterName + " is not a recognized counter.");
    return null;
  }
}
 
Example 11
Source Project: hadoop   Source File: FileSystemCounterGroup.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public synchronized C findCounter(String scheme, FileSystemCounter key) {
  final String canonicalScheme = checkScheme(scheme);
  Object[] counters = map.get(canonicalScheme);
  int ord = key.ordinal();
  if (counters == null) {
    counters = new Object[FileSystemCounter.values().length];
    map.put(canonicalScheme, counters);
    counters[ord] = newCounter(canonicalScheme, key);
  }
  else if (counters[ord] == null) {
    counters[ord] = newCounter(canonicalScheme, key);
  }
  return (C) counters[ord];
}
 
Example 12
Source Project: hadoop   Source File: FileSystemCounterGroup.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void readFields(DataInput in) throws IOException {
  int numSchemes = WritableUtils.readVInt(in);    // #scheme
  FileSystemCounter[] enums = FileSystemCounter.values();
  for (int i = 0; i < numSchemes; ++i) {
    String scheme = WritableUtils.readString(in); // scheme
    int numCounters = WritableUtils.readVInt(in); // #counter
    for (int j = 0; j < numCounters; ++j) {
      findCounter(scheme, enums[WritableUtils.readVInt(in)])  // key
          .setValue(WritableUtils.readVLong(in)); // value
    }
  }
}
 
Example 13
Source Project: hadoop   Source File: FileSystemCounterGroup.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized int hashCode() {
  // need to be deep as counters is an array
  int hash = FileSystemCounter.class.hashCode();
  for (Object[] counters : map.values()) {
    if (counters != null) hash ^= Arrays.hashCode(counters);
  }
  return hash;
}
 
Example 14
Source Project: hadoop   Source File: TestCounters.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
@Test
public void testReadWithLegacyNames() {
  Counters counters = new Counters();
  counters.incrCounter(TaskCounter.MAP_INPUT_RECORDS, 1);
  counters.incrCounter(JobCounter.DATA_LOCAL_MAPS, 1);
  counters.findCounter("file", FileSystemCounter.BYTES_READ).increment(1);
  
  checkLegacyNames(counters);
}
 
Example 15
Source Project: big-c   Source File: Task.java    License: Apache License 2.0 5 votes vote down vote up
void updateCounters() {
  if (readBytesCounter == null) {
    readBytesCounter = counters.findCounter(scheme,
        FileSystemCounter.BYTES_READ);
  }
  if (writeBytesCounter == null) {
    writeBytesCounter = counters.findCounter(scheme,
        FileSystemCounter.BYTES_WRITTEN);
  }
  if (readOpsCounter == null) {
    readOpsCounter = counters.findCounter(scheme,
        FileSystemCounter.READ_OPS);
  }
  if (largeReadOpsCounter == null) {
    largeReadOpsCounter = counters.findCounter(scheme,
        FileSystemCounter.LARGE_READ_OPS);
  }
  if (writeOpsCounter == null) {
    writeOpsCounter = counters.findCounter(scheme,
        FileSystemCounter.WRITE_OPS);
  }
  long readBytes = 0;
  long writeBytes = 0;
  long readOps = 0;
  long largeReadOps = 0;
  long writeOps = 0;
  for (FileSystem.Statistics stat: stats) {
    readBytes = readBytes + stat.getBytesRead();
    writeBytes = writeBytes + stat.getBytesWritten();
    readOps = readOps + stat.getReadOps();
    largeReadOps = largeReadOps + stat.getLargeReadOps();
    writeOps = writeOps + stat.getWriteOps();
  }
  readBytesCounter.setValue(readBytes);
  writeBytesCounter.setValue(writeBytes);
  readOpsCounter.setValue(readOps);
  largeReadOpsCounter.setValue(largeReadOps);
  writeOpsCounter.setValue(writeOps);
}
 
Example 16
Source Project: big-c   Source File: AbstractCounters.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Find the file system counter for the given scheme and enum.
 * @param scheme of the file system
 * @param key the enum of the counter
 * @return the file system counter
 */
@InterfaceAudience.Private
public synchronized C findCounter(String scheme, FileSystemCounter key) {
  return ((FileSystemCounterGroup<C>) getGroup(
      FileSystemCounter.class.getName()).getUnderlyingGroup()).
      findCounter(scheme, key);
}
 
Example 17
Source Project: big-c   Source File: FileSystemCounterGroup.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public C findCounter(String counterName, boolean create) {
  try {
    String[] pair = parseCounterName(counterName);
    return findCounter(pair[0], FileSystemCounter.valueOf(pair[1]));
  }
  catch (Exception e) {
    if (create) throw new IllegalArgumentException(e);
    LOG.warn(counterName + " is not a recognized counter.");
    return null;
  }
}
 
Example 18
Source Project: big-c   Source File: FileSystemCounterGroup.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public synchronized C findCounter(String scheme, FileSystemCounter key) {
  final String canonicalScheme = checkScheme(scheme);
  Object[] counters = map.get(canonicalScheme);
  int ord = key.ordinal();
  if (counters == null) {
    counters = new Object[FileSystemCounter.values().length];
    map.put(canonicalScheme, counters);
    counters[ord] = newCounter(canonicalScheme, key);
  }
  else if (counters[ord] == null) {
    counters[ord] = newCounter(canonicalScheme, key);
  }
  return (C) counters[ord];
}
 
Example 19
Source Project: big-c   Source File: FileSystemCounterGroup.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void readFields(DataInput in) throws IOException {
  int numSchemes = WritableUtils.readVInt(in);    // #scheme
  FileSystemCounter[] enums = FileSystemCounter.values();
  for (int i = 0; i < numSchemes; ++i) {
    String scheme = WritableUtils.readString(in); // scheme
    int numCounters = WritableUtils.readVInt(in); // #counter
    for (int j = 0; j < numCounters; ++j) {
      findCounter(scheme, enums[WritableUtils.readVInt(in)])  // key
          .setValue(WritableUtils.readVLong(in)); // value
    }
  }
}
 
Example 20
Source Project: big-c   Source File: FileSystemCounterGroup.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized int hashCode() {
  // need to be deep as counters is an array
  int hash = FileSystemCounter.class.hashCode();
  for (Object[] counters : map.values()) {
    if (counters != null) hash ^= Arrays.hashCode(counters);
  }
  return hash;
}
 
Example 21
Source Project: big-c   Source File: TestCounters.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
@Test
public void testReadWithLegacyNames() {
  Counters counters = new Counters();
  counters.incrCounter(TaskCounter.MAP_INPUT_RECORDS, 1);
  counters.incrCounter(JobCounter.DATA_LOCAL_MAPS, 1);
  counters.findCounter("file", FileSystemCounter.BYTES_READ).increment(1);
  
  checkLegacyNames(counters);
}
 
Example 22
Source Project: kylin-on-parquet-v2   Source File: HadoopCmdOutput.java    License: Apache License 2.0 4 votes vote down vote up
public void updateJobCounter() {
    try {
        Counters counters = job.getCounters();
        if (counters == null) {
            String errorMsg = "no counters for job " + getMrJobId();
            logger.warn(errorMsg);
            output.append(errorMsg);
        } else {
            this.output.append(counters.toString()).append("\n");
            logger.debug(counters.toString());

            mapInputRecords = String.valueOf(counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue());
            rawInputBytesRead = String.valueOf(counters.findCounter(RawDataCounter.BYTES).getValue());

            String outputFolder = job.getConfiguration().get("mapreduce.output.fileoutputformat.outputdir",
                    KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory());
            logger.debug("outputFolder is " + outputFolder);
            Path outputPath = new Path(outputFolder);
            String fsScheme = outputPath.getFileSystem(job.getConfiguration()).getScheme();
            long bytesWritten = counters.findCounter(fsScheme, FileSystemCounter.BYTES_WRITTEN).getValue();
            if (bytesWritten == 0) {
                logger.debug("Seems no counter found for " + fsScheme);
                bytesWritten = counters.findCounter("FileSystemCounters", "HDFS_BYTES_WRITTEN").getValue();
            }
            hdfsBytesWritten = String.valueOf(bytesWritten);
        }
        JobStatus jobStatus = job.getStatus();
        if (jobStatus.getState() == JobStatus.State.FAILED) {
            logger.warn("Job Diagnostics:" + jobStatus.getFailureInfo());
            output.append("Job Diagnostics:").append(jobStatus.getFailureInfo()).append("\n");
            TaskCompletionEvent taskEvent = getOneTaskFailure(job);
            if (taskEvent != null) {
                String[] fails = job.getTaskDiagnostics(taskEvent.getTaskAttemptId());
                logger.warn("Failure task Diagnostics:");
                output.append("Failure task Diagnostics:").append("\n");
                for (String failure : fails) {
                    logger.warn(failure);
                    output.append(failure).append("\n");
                }
            }
        }
    } catch (Exception e) {
        logger.error(e.getLocalizedMessage(), e);
        output.append(e.getLocalizedMessage());
    }
}
 
Example 23
Source Project: circus-train   Source File: DistCpCopier.java    License: Apache License 2.0 4 votes vote down vote up
private void registerRunningJobMetrics(final Job job, final String counter) {
  registry.remove(RunningMetrics.DIST_CP_BYTES_REPLICATED.name());
  registry
      .register(RunningMetrics.DIST_CP_BYTES_REPLICATED.name(),
          new JobCounterGauge(job, FileSystemCounter.class.getName(), counter));
}
 
Example 24
Source Project: hadoop   Source File: Counters.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected Counter newCounter(String scheme, FileSystemCounter key) {
  return new Counter(new FSCounter(scheme, key));
}
 
Example 25
Source Project: hadoop   Source File: FileSystemCounterGroup.java    License: Apache License 2.0 4 votes vote down vote up
public FSCounter(String scheme, FileSystemCounter ref) {
  this.scheme = scheme;
  key = ref;
}
 
Example 26
Source Project: hadoop   Source File: FileSystemCounterGroup.java    License: Apache License 2.0 4 votes vote down vote up
@Private
public FileSystemCounter getFileSystemCounter() {
  return key;
}
 
Example 27
Source Project: hadoop   Source File: FileSystemCounterGroup.java    License: Apache License 2.0 4 votes vote down vote up
protected String localizeCounterName(String counterName) {
  return ResourceBundles.getCounterName(FileSystemCounter.class.getName(),
                                        counterName, counterName);
}
 
Example 28
Source Project: hadoop   Source File: FileSystemCounterGroup.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public String getName() {
  return FileSystemCounter.class.getName();
}
 
Example 29
Source Project: big-c   Source File: Counters.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected Counter newCounter(String scheme, FileSystemCounter key) {
  return new Counter(new FSCounter(scheme, key));
}
 
Example 30
Source Project: big-c   Source File: FileSystemCounterGroup.java    License: Apache License 2.0 4 votes vote down vote up
public FSCounter(String scheme, FileSystemCounter ref) {
  this.scheme = scheme;
  key = ref;
}