Java Code Examples for org.apache.hadoop.fs.FileSystem#Statistics

The following examples show how to use org.apache.hadoop.fs.FileSystem#Statistics . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
/**
 * Constructs an instance of GoogleHadoopFSInputStream object.
 *
 * @param ghfs Instance of GoogleHadoopFileSystemBase.
 * @param gcsPath Path of the file to read from.
 * @param statistics File system statistics object.
 * @throws IOException if an IO error occurs.
 */
GoogleHadoopFSInputStream(
    GoogleHadoopFileSystemBase ghfs,
    URI gcsPath,
    GoogleCloudStorageReadOptions readOptions,
    FileSystem.Statistics statistics)
    throws IOException {
  logger.atFine().log(
      "GoogleHadoopFSInputStream(gcsPath: %s, readOptions: %s)", gcsPath, readOptions);
  this.ghfs = ghfs;
  this.gcsPath = gcsPath;
  this.statistics = statistics;
  this.initTime = System.nanoTime();
  this.totalBytesRead = 0;
  this.channel = ghfs.getGcsFs().open(gcsPath, readOptions);
}
 
Example 2
/**
 * Constructs an instance of GoogleHadoopOutputStream object.
 *
 * @param ghfs Instance of GoogleHadoopFileSystemBase.
 * @param gcsPath Path of the file to write to.
 * @param statistics File system statistics object.
 * @throws IOException if an IO error occurs.
 */
GoogleHadoopOutputStream(
    GoogleHadoopFileSystemBase ghfs,
    URI gcsPath,
    FileSystem.Statistics statistics,
    CreateFileOptions createFileOptions)
    throws IOException {
  logger.atFine().log(
      "GoogleHadoopOutputStream(gcsPath:%s, createFileOptions: %s)", gcsPath, createFileOptions);
  this.ghfs = ghfs;
  this.gcsPath = gcsPath;
  this.statistics = statistics;
  this.initTime = System.nanoTime();
  this.channel = createChannel(ghfs, gcsPath, createFileOptions);
  this.out =
      createOutputStream(this.channel, ghfs.getGcsFs().getOptions().getCloudStorageOptions());
}
 
Example 3
Source Project: big-c   File: Task.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Gets a handle to the Statistics instance based on the scheme associated
 * with path.
 * 
 * @param path the path.
 * @param conf the configuration to extract the scheme from if not part of 
 *   the path.
 * @return a Statistics instance, or null if none is found for the scheme.
 */
protected static List<Statistics> getFsStatistics(Path path, Configuration conf) throws IOException {
  List<Statistics> matchedStats = new ArrayList<FileSystem.Statistics>();
  path = path.getFileSystem(conf).makeQualified(path);
  String scheme = path.toUri().getScheme();
  for (Statistics stats : FileSystem.getAllStatistics()) {
    if (stats.getScheme().equals(scheme)) {
      matchedStats.add(stats);
    }
  }
  return matchedStats;
}
 
Example 4
Source Project: big-c   File: FTPInputStream.java    License: Apache License 2.0 5 votes vote down vote up
public FTPInputStream(InputStream stream, FTPClient client,
    FileSystem.Statistics stats) {
  if (stream == null) {
    throw new IllegalArgumentException("Null InputStream");
  }
  if (client == null || !client.isConnected()) {
    throw new IllegalArgumentException("FTP client null or not connected");
  }
  this.wrappedStream = stream;
  this.client = client;
  this.stats = stats;
  this.pos = 0;
  this.closed = false;
}
 
Example 5
Source Project: big-c   File: S3AOutputStream.java    License: Apache License 2.0 5 votes vote down vote up
public ProgressableProgressListener(Upload upload, Progressable progress, 
  FileSystem.Statistics statistics) {
  this.upload = upload;
  this.progress = progress;
  this.statistics = statistics;
  this.lastBytesTransferred = 0;
}
 
Example 6
private static GoogleHadoopFSInputStream createGhfsInputStream(
    GoogleHadoopFileSystem ghfs, URI path) throws IOException {
  GoogleCloudStorageReadOptions options =
      ghfs.getGcsFs().getOptions().getCloudStorageOptions().getReadChannelOptions();
  return new GoogleHadoopFSInputStream(
      ghfs, path, options, new FileSystem.Statistics(ghfs.getScheme()));
}
 
Example 7
void updateCounters() {
  if (readBytesCounter == null) {
    readBytesCounter = counters.findCounter(scheme, FileSystemCounter.BYTES_READ);
  }
  if (writeBytesCounter == null) {
    writeBytesCounter = counters.findCounter(scheme, FileSystemCounter.BYTES_WRITTEN);
  }
  if (readOpsCounter == null) {
    readOpsCounter = counters.findCounter(scheme, FileSystemCounter.READ_OPS);
  }
  if (largeReadOpsCounter == null) {
    largeReadOpsCounter = counters.findCounter(scheme, FileSystemCounter.LARGE_READ_OPS);
  }
  if (writeOpsCounter == null) {
    writeOpsCounter = counters.findCounter(scheme, FileSystemCounter.WRITE_OPS);
  }
  long readBytes = 0;
  long writeBytes = 0;
  long readOps = 0;
  long largeReadOps = 0;
  long writeOps = 0;
  for (FileSystem.Statistics stat : stats) {
    readBytes = readBytes + stat.getBytesRead();
    writeBytes = writeBytes + stat.getBytesWritten();
    readOps = readOps + stat.getReadOps();
    largeReadOps = largeReadOps + stat.getLargeReadOps();
    writeOps = writeOps + stat.getWriteOps();
  }
  readBytesCounter.setValue(readBytes);
  writeBytesCounter.setValue(writeBytes);
  readOpsCounter.setValue(readOps);
  largeReadOpsCounter.setValue(largeReadOps);
  writeOpsCounter.setValue(writeOps);
}
 
Example 8
Source Project: hadoop   File: S3AInputStream.java    License: Apache License 2.0 5 votes vote down vote up
public S3AInputStream(String bucket, String key, long contentLength, AmazonS3Client client,
                      FileSystem.Statistics stats) {
  this.bucket = bucket;
  this.key = key;
  this.contentLength = contentLength;
  this.client = client;
  this.stats = stats;
  this.pos = 0;
  this.closed = false;
  this.wrappedStream = null;
}
 
Example 9
Source Project: big-c   File: SwiftNativeInputStream.java    License: Apache License 2.0 5 votes vote down vote up
public SwiftNativeInputStream(SwiftNativeFileSystemStore storeNative,
    FileSystem.Statistics statistics, Path path, long bufferSize)
        throws IOException {
  this.nativeStore = storeNative;
  this.statistics = statistics;
  this.path = path;
  if (bufferSize <= 0) {
    throw new IllegalArgumentException("Invalid buffer size");
  }
  this.bufferSize = bufferSize;
  //initial buffer fill
  this.httpStream = storeNative.getObject(path).getInputStream();
  //fillBuffer(0);
}
 
Example 10
Source Project: tez   File: Utils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Gets a handle to the Statistics instance based on the scheme associated
 * with path.
 *
 * @param path the path.
 * @param conf the configuration to extract the scheme from if not part of
 *   the path.
 * @return a Statistics instance, or null if none is found for the scheme.
 */
@Private
public static List<Statistics> getFsStatistics(Path path, Configuration conf) throws IOException {
  List<Statistics> matchedStats = new ArrayList<FileSystem.Statistics>();
  path = path.getFileSystem(conf).makeQualified(path);
  String scheme = path.toUri().getScheme();
  for (Statistics stats : FileSystem.getAllStatistics()) {
    if (stats.getScheme().equals(scheme)) {
      matchedStats.add(stats);
    }
  }
  return matchedStats;
}
 
Example 11
@Test
public void testExceptionOnDelete() throws IOException {
  Path objectPath = new Path(ghfs.getFileSystemRoot(), "dir/object2.txt");
  GoogleHadoopSyncableOutputStream fout =
      new GoogleHadoopSyncableOutputStream(
          ghfs,
          ghfs.getGcsPath(objectPath),
          new FileSystem.Statistics(ghfs.getScheme()),
          CreateFileOptions.DEFAULT_OVERWRITE,
          SyncableOutputStreamOptions.DEFAULT,
          mockExecutorService);

  IOException fakeIoException = new IOException("fake io exception");
  when(mockExecutorService.submit(any(Callable.class)))
      .thenReturn(Futures.immediateFailedFuture(new ExecutionException(fakeIoException)));

  byte[] data1 = new byte[] { 0x0f, 0x0e, 0x0e, 0x0d };
  byte[] data2 = new byte[] { 0x0b, 0x0e, 0x0e, 0x0f };

  fout.write(data1, 0, data1.length);
  fout.sync();  // This one commits straight into destination.

  fout.write(data2, 0, data2.length);
  fout.sync();  // This one enqueues the delete, but doesn't propagate exception yet.

  verify(mockExecutorService).submit(any(Callable.class));

  IOException thrown = assertThrows(IOException.class, fout::close);
  assertThat(thrown).hasCauseThat().hasMessageThat().contains(fakeIoException.getMessage());

  verify(mockExecutorService, times(2)).submit(any(Callable.class));
}
 
Example 12
Source Project: big-c   File: DFSClient.java    License: Apache License 2.0 4 votes vote down vote up
/** 
 * Create a new DFSClient connected to the given nameNodeUri or rpcNamenode.
 * If HA is enabled and a positive value is set for 
 * {@link DFSConfigKeys#DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY} in the
 * configuration, the DFSClient will use {@link LossyRetryInvocationHandler}
 * as its RetryInvocationHandler. Otherwise one of nameNodeUri or rpcNamenode 
 * must be null.
 */
@VisibleForTesting
public DFSClient(URI nameNodeUri, ClientProtocol rpcNamenode,
    Configuration conf, FileSystem.Statistics stats)
  throws IOException {
  SpanReceiverHost.get(conf, DFSConfigKeys.DFS_CLIENT_HTRACE_PREFIX);
  traceSampler = new SamplerBuilder(TraceUtils.
      wrapHadoopConf(DFSConfigKeys.DFS_CLIENT_HTRACE_PREFIX, conf)).build();
  // Copy only the required DFSClient configuration
  this.dfsClientConf = new Conf(conf);
  if (this.dfsClientConf.useLegacyBlockReaderLocal) {
    LOG.debug("Using legacy short-circuit local reads.");
  }
  this.conf = conf;
  this.stats = stats;
  this.socketFactory = NetUtils.getSocketFactory(conf, ClientProtocol.class);
  this.dtpReplaceDatanodeOnFailure = ReplaceDatanodeOnFailure.get(conf);

  this.ugi = UserGroupInformation.getCurrentUser();
  
  this.authority = nameNodeUri == null? "null": nameNodeUri.getAuthority();
  this.clientName = "DFSClient_" + dfsClientConf.taskId + "_" + 
      DFSUtil.getRandom().nextInt()  + "_" + Thread.currentThread().getId();
  int numResponseToDrop = conf.getInt(
      DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY,
      DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_DEFAULT);
  NameNodeProxies.ProxyAndInfo<ClientProtocol> proxyInfo = null;
  AtomicBoolean nnFallbackToSimpleAuth = new AtomicBoolean(false);
  if (numResponseToDrop > 0) {
    // This case is used for testing.
    LOG.warn(DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY
        + " is set to " + numResponseToDrop
        + ", this hacked client will proactively drop responses");
    proxyInfo = NameNodeProxies.createProxyWithLossyRetryHandler(conf,
        nameNodeUri, ClientProtocol.class, numResponseToDrop,
        nnFallbackToSimpleAuth);
  }
  
  if (proxyInfo != null) {
    this.dtService = proxyInfo.getDelegationTokenService();
    this.namenode = proxyInfo.getProxy();
  } else if (rpcNamenode != null) {
    // This case is used for testing.
    Preconditions.checkArgument(nameNodeUri == null);
    this.namenode = rpcNamenode;
    dtService = null;
  } else {
    Preconditions.checkArgument(nameNodeUri != null,
        "null URI");
    proxyInfo = NameNodeProxies.createProxy(conf, nameNodeUri,
        ClientProtocol.class, nnFallbackToSimpleAuth);
    this.dtService = proxyInfo.getDelegationTokenService();
    this.namenode = proxyInfo.getProxy();
  }

  String localInterfaces[] =
    conf.getTrimmedStrings(DFSConfigKeys.DFS_CLIENT_LOCAL_INTERFACES);
  localInterfaceAddrs = getLocalInterfaceAddrs(localInterfaces);
  if (LOG.isDebugEnabled() && 0 != localInterfaces.length) {
    LOG.debug("Using local interfaces [" +
    Joiner.on(',').join(localInterfaces)+ "] with addresses [" +
    Joiner.on(',').join(localInterfaceAddrs) + "]");
  }
  
  Boolean readDropBehind = (conf.get(DFS_CLIENT_CACHE_DROP_BEHIND_READS) == null) ?
      null : conf.getBoolean(DFS_CLIENT_CACHE_DROP_BEHIND_READS, false);
  Long readahead = (conf.get(DFS_CLIENT_CACHE_READAHEAD) == null) ?
      null : conf.getLong(DFS_CLIENT_CACHE_READAHEAD, 0);
  Boolean writeDropBehind = (conf.get(DFS_CLIENT_CACHE_DROP_BEHIND_WRITES) == null) ?
      null : conf.getBoolean(DFS_CLIENT_CACHE_DROP_BEHIND_WRITES, false);
  this.defaultReadCachingStrategy =
      new CachingStrategy(readDropBehind, readahead);
  this.defaultWriteCachingStrategy =
      new CachingStrategy(writeDropBehind, readahead);
  this.clientContext = ClientContext.get(
      conf.get(DFS_CLIENT_CONTEXT, DFS_CLIENT_CONTEXT_DEFAULT),
      dfsClientConf);
  this.hedgedReadThresholdMillis = conf.getLong(
      DFSConfigKeys.DFS_DFSCLIENT_HEDGED_READ_THRESHOLD_MILLIS,
      DFSConfigKeys.DEFAULT_DFSCLIENT_HEDGED_READ_THRESHOLD_MILLIS);
  int numThreads = conf.getInt(
      DFSConfigKeys.DFS_DFSCLIENT_HEDGED_READ_THREADPOOL_SIZE,
      DFSConfigKeys.DEFAULT_DFSCLIENT_HEDGED_READ_THREADPOOL_SIZE);
  if (numThreads > 0) {
    this.initThreadsNumForHedgedReads(numThreads);
  }
  this.saslClient = new SaslDataTransferClient(
    conf, DataTransferSaslUtil.getSaslPropertiesResolver(conf),
    TrustedChannelResolver.getInstance(conf), nnFallbackToSimpleAuth);
}
 
Example 13
Source Project: hadoop-gpu   File: KFSImpl.java    License: Apache License 2.0 4 votes vote down vote up
public KFSImpl(String metaServerHost, int metaServerPort, 
               FileSystem.Statistics stats) throws IOException {
    kfsAccess = new KfsAccess(metaServerHost, metaServerPort);
    statistics = stats;
}
 
Example 14
Source Project: tez   File: FileSystemStatisticUpdater.java    License: Apache License 2.0 4 votes vote down vote up
FileSystemStatisticUpdater(TezCounters counters, List<FileSystem.Statistics> stats, String scheme) {
  this.stats = stats;
  this.scheme = scheme;
  this.counters = counters;
}
 
Example 15
Source Project: hadoop-gpu   File: Task.java    License: Apache License 2.0 4 votes vote down vote up
FileSystemStatisticUpdater(String uriScheme, FileSystem.Statistics stats) {
  this.stats = stats;
  this.counterNames = getFileSystemCounterNames(uriScheme);
}
 
Example 16
Source Project: big-c   File: DFSClient.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Same as this(nameNodeUri, null, conf, stats);
 * @see #DFSClient(URI, ClientProtocol, Configuration, FileSystem.Statistics) 
 */
public DFSClient(URI nameNodeUri, Configuration conf,
                 FileSystem.Statistics stats)
  throws IOException {
  this(nameNodeUri, null, conf, stats);
}
 
Example 17
Source Project: hadoop   File: HdfsDataOutputStream.java    License: Apache License 2.0 4 votes vote down vote up
public HdfsDataOutputStream(CryptoOutputStream out, FileSystem.Statistics stats,
    long startPosition) throws IOException {
  super(out, stats, startPosition);
  Preconditions.checkArgument(out.getWrappedStream() instanceof DFSOutputStream,
      "CryptoOutputStream should wrap a DFSOutputStream");
}
 
Example 18
Source Project: big-c   File: HdfsDataOutputStream.java    License: Apache License 2.0 4 votes vote down vote up
public HdfsDataOutputStream(DFSOutputStream out, FileSystem.Statistics stats,
    long startPosition) throws IOException {
  super(out, stats, startPosition);
}
 
Example 19
/**
 * Writes a file with the given buffer repeated numWrites times.
 *
 * @param hadoopPath Path of the file to create.
 * @param buffer Data to write.
 * @param numWrites Number of times to repeat the data.
 * @param overwrite If true, overwrite any existing file.
 * @return Number of bytes written.
 */
public int writeFile(Path hadoopPath, byte[] buffer, int numWrites, boolean overwrite)
    throws IOException {
  int numBytesWritten = -1;
  int totalBytesWritten = 0;

  long fileSystemBytesWritten = 0;
  FileSystem.Statistics stats = FileSystem.getStatistics(
      ghfsFileSystemDescriptor.getScheme(), ghfs.getClass());
  if (stats != null) {
    // Let it be null in case no stats have been added for our scheme yet.
    fileSystemBytesWritten =
        stats.getBytesWritten();
  }
  try (FSDataOutputStream writeStream =
      ghfs.create(
          hadoopPath,
          FsPermission.getDefault(),
          overwrite,
          GoogleHadoopFileSystemConfiguration.GCS_OUTPUT_STREAM_BUFFER_SIZE.getDefault(),
          GoogleHadoopFileSystemBase.REPLICATION_FACTOR_DEFAULT,
          GoogleHadoopFileSystemConfiguration.BLOCK_SIZE.getDefault(),
          /* progress= */ null)) {
    for (int i = 0; i < numWrites; i++) {
      writeStream.write(buffer, 0, buffer.length);
      numBytesWritten = buffer.length;
      totalBytesWritten += numBytesWritten;
    }
  }

  // After the write, the stats better be non-null for our ghfs scheme.
  stats = FileSystem.getStatistics(ghfsFileSystemDescriptor.getScheme(), ghfs.getClass());
  assertThat(stats).isNotNull();
  long endFileSystemBytesWritten =
      stats.getBytesWritten();
  int bytesWrittenStats = (int) (endFileSystemBytesWritten - fileSystemBytesWritten);
  if (statistics == FileSystemStatistics.EXACT) {
    assertWithMessage("FS statistics mismatch fetched from class '%s'", ghfs.getClass())
        .that(bytesWrittenStats)
        .isEqualTo(totalBytesWritten);
  } else if (statistics == FileSystemStatistics.GREATER_OR_EQUAL) {
    assertWithMessage("Expected %d <= %d", totalBytesWritten, bytesWrittenStats)
        .that(totalBytesWritten <= bytesWrittenStats)
        .isTrue();
  } else if (statistics == FileSystemStatistics.NONE) {
    // Do not perform any check because stats are either not maintained or are erratic.
  } else if (statistics == FileSystemStatistics.IGNORE) {
    // NO-OP
  }

  return totalBytesWritten;
}
 
Example 20
Source Project: RDFS   File: Task.java    License: Apache License 2.0 4 votes vote down vote up
FileSystemStatisticUpdater(String uriScheme, FileSystem.Statistics stats) {
  this.stats = stats;
  this.counterNames = getFileSystemCounterNames(uriScheme);
}