Java Code Examples for org.apache.hadoop.fs.CacheFlag

The following examples show how to use org.apache.hadoop.fs.CacheFlag. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: FSNDNCacheOp.java    License: Apache License 2.0 6 votes vote down vote up
static CacheDirectiveInfo addCacheDirective(
    FSNamesystem fsn, CacheManager cacheManager,
    CacheDirectiveInfo directive, EnumSet<CacheFlag> flags,
    boolean logRetryCache)
    throws IOException {

  final FSPermissionChecker pc = getFsPermissionChecker(fsn);

  if (directive.getId() != null) {
    throw new IOException("addDirective: you cannot specify an ID " +
        "for this operation.");
  }
  CacheDirectiveInfo effectiveDirective =
      cacheManager.addDirective(directive, pc, flags);
  fsn.getEditLog().logAddCacheDirectiveInfo(effectiveDirective,
      logRetryCache);
  return effectiveDirective;
}
 
Example 2
Source Project: hadoop   Source File: NameNodeRpcServer.java    License: Apache License 2.0 6 votes vote down vote up
@Override // ClientProtocol
public long addCacheDirective(
    CacheDirectiveInfo path, EnumSet<CacheFlag> flags) throws IOException {
  checkNNStartup();
  CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion
    (retryCache, null);
  if (cacheEntry != null && cacheEntry.isSuccess()) {
    return (Long) cacheEntry.getPayload();
  }

  boolean success = false;
  long ret = 0;
  try {
    ret = namesystem.addCacheDirective(path, flags, cacheEntry != null);
    success = true;
  } finally {
    RetryCache.setState(cacheEntry, success, ret);
  }
  return ret;
}
 
Example 3
Source Project: hadoop   Source File: NameNodeRpcServer.java    License: Apache License 2.0 6 votes vote down vote up
@Override // ClientProtocol
public void modifyCacheDirective(
    CacheDirectiveInfo directive, EnumSet<CacheFlag> flags) throws IOException {
  checkNNStartup();
  CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
  if (cacheEntry != null && cacheEntry.isSuccess()) {
    return;
  }

  boolean success = false;
  try {
    namesystem.modifyCacheDirective(directive, flags, cacheEntry != null);
    success = true;
  } finally {
    RetryCache.setState(cacheEntry, success);
  }
}
 
Example 4
Source Project: hadoop   Source File: TestRetryCacheWithHA.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Add a list of cache directives, list cache directives,
 * switch active NN, and list cache directives again.
 */
@Test (timeout=60000)
public void testListCacheDirectives() throws Exception {
  final int poolCount = 7;
  HashSet<String> poolNames = new HashSet<String>(poolCount);
  Path path = new Path("/p");
  for (int i=0; i<poolCount; i++) {
    String poolName = "testListCacheDirectives-" + i;
    CacheDirectiveInfo directiveInfo =
      new CacheDirectiveInfo.Builder().setPool(poolName).setPath(path).build();
    dfs.addCachePool(new CachePoolInfo(poolName));
    dfs.addCacheDirective(directiveInfo, EnumSet.of(CacheFlag.FORCE));
    poolNames.add(poolName);
  }
  listCacheDirectives(poolNames, 0);

  cluster.transitionToStandby(0);
  cluster.transitionToActive(1);
  cluster.waitActive(1);
  listCacheDirectives(poolNames, 1);
}
 
Example 5
Source Project: big-c   Source File: FSNDNCacheOp.java    License: Apache License 2.0 6 votes vote down vote up
static CacheDirectiveInfo addCacheDirective(
    FSNamesystem fsn, CacheManager cacheManager,
    CacheDirectiveInfo directive, EnumSet<CacheFlag> flags,
    boolean logRetryCache)
    throws IOException {

  final FSPermissionChecker pc = getFsPermissionChecker(fsn);

  if (directive.getId() != null) {
    throw new IOException("addDirective: you cannot specify an ID " +
        "for this operation.");
  }
  CacheDirectiveInfo effectiveDirective =
      cacheManager.addDirective(directive, pc, flags);
  fsn.getEditLog().logAddCacheDirectiveInfo(effectiveDirective,
      logRetryCache);
  return effectiveDirective;
}
 
Example 6
Source Project: big-c   Source File: NameNodeRpcServer.java    License: Apache License 2.0 6 votes vote down vote up
@Override // ClientProtocol
public long addCacheDirective(
    CacheDirectiveInfo path, EnumSet<CacheFlag> flags) throws IOException {
  checkNNStartup();
  CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion
    (retryCache, null);
  if (cacheEntry != null && cacheEntry.isSuccess()) {
    return (Long) cacheEntry.getPayload();
  }

  boolean success = false;
  long ret = 0;
  try {
    ret = namesystem.addCacheDirective(path, flags, cacheEntry != null);
    success = true;
  } finally {
    RetryCache.setState(cacheEntry, success, ret);
  }
  return ret;
}
 
Example 7
Source Project: big-c   Source File: NameNodeRpcServer.java    License: Apache License 2.0 6 votes vote down vote up
@Override // ClientProtocol
public void modifyCacheDirective(
    CacheDirectiveInfo directive, EnumSet<CacheFlag> flags) throws IOException {
  checkNNStartup();
  CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
  if (cacheEntry != null && cacheEntry.isSuccess()) {
    return;
  }

  boolean success = false;
  try {
    namesystem.modifyCacheDirective(directive, flags, cacheEntry != null);
    success = true;
  } finally {
    RetryCache.setState(cacheEntry, success);
  }
}
 
Example 8
Source Project: big-c   Source File: TestRetryCacheWithHA.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Add a list of cache directives, list cache directives,
 * switch active NN, and list cache directives again.
 */
@Test (timeout=60000)
public void testListCacheDirectives() throws Exception {
  final int poolCount = 7;
  HashSet<String> poolNames = new HashSet<String>(poolCount);
  Path path = new Path("/p");
  for (int i=0; i<poolCount; i++) {
    String poolName = "testListCacheDirectives-" + i;
    CacheDirectiveInfo directiveInfo =
      new CacheDirectiveInfo.Builder().setPool(poolName).setPath(path).build();
    dfs.addCachePool(new CachePoolInfo(poolName));
    dfs.addCacheDirective(directiveInfo, EnumSet.of(CacheFlag.FORCE));
    poolNames.add(poolName);
  }
  listCacheDirectives(poolNames, 0);

  cluster.transitionToStandby(0);
  cluster.transitionToActive(1);
  cluster.waitActive(1);
  listCacheDirectives(poolNames, 1);
}
 
Example 9
Source Project: hadoop   Source File: DFSClient.java    License: Apache License 2.0 5 votes vote down vote up
public long addCacheDirective(
    CacheDirectiveInfo info, EnumSet<CacheFlag> flags) throws IOException {
  checkOpen();
  TraceScope scope = Trace.startSpan("addCacheDirective", traceSampler);
  try {
    return namenode.addCacheDirective(info, flags);
  } catch (RemoteException re) {
    throw re.unwrapRemoteException();
  } finally {
    scope.close();
  }
}
 
Example 10
Source Project: hadoop   Source File: DFSClient.java    License: Apache License 2.0 5 votes vote down vote up
public void modifyCacheDirective(
    CacheDirectiveInfo info, EnumSet<CacheFlag> flags) throws IOException {
  checkOpen();
  TraceScope scope = Trace.startSpan("modifyCacheDirective", traceSampler);
  try {
    namenode.modifyCacheDirective(info, flags);
  } catch (RemoteException re) {
    throw re.unwrapRemoteException();
  } finally {
    scope.close();
  }
}
 
Example 11
Source Project: hadoop   Source File: DistributedFileSystem.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Add a new CacheDirective.
 * 
 * @param info Information about a directive to add.
 * @param flags {@link CacheFlag}s to use for this operation.
 * @return the ID of the directive that was created.
 * @throws IOException if the directive could not be added
 */
public long addCacheDirective(
    CacheDirectiveInfo info, EnumSet<CacheFlag> flags) throws IOException {
  Preconditions.checkNotNull(info.getPath());
  Path path = new Path(getPathName(fixRelativePart(info.getPath()))).
      makeQualified(getUri(), getWorkingDirectory());
  return dfs.addCacheDirective(
      new CacheDirectiveInfo.Builder(info).
          setPath(path).
          build(),
      flags);
}
 
Example 12
Source Project: hadoop   Source File: DistributedFileSystem.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Modify a CacheDirective.
 * 
 * @param info Information about the directive to modify. You must set the ID
 *          to indicate which CacheDirective you want to modify.
 * @param flags {@link CacheFlag}s to use for this operation.
 * @throws IOException if the directive could not be modified
 */
public void modifyCacheDirective(
    CacheDirectiveInfo info, EnumSet<CacheFlag> flags) throws IOException {
  if (info.getPath() != null) {
    info = new CacheDirectiveInfo.Builder(info).
        setPath(new Path(getPathName(fixRelativePart(info.getPath()))).
            makeQualified(getUri(), getWorkingDirectory())).build();
  }
  dfs.modifyCacheDirective(info, flags);
}
 
Example 13
Source Project: hadoop   Source File: FSNDNCacheOp.java    License: Apache License 2.0 5 votes vote down vote up
static void modifyCacheDirective(
    FSNamesystem fsn, CacheManager cacheManager, CacheDirectiveInfo directive,
    EnumSet<CacheFlag> flags, boolean logRetryCache) throws IOException {
  final FSPermissionChecker pc = getFsPermissionChecker(fsn);

  cacheManager.modifyDirective(directive, pc, flags);
  fsn.getEditLog().logModifyCacheDirectiveInfo(directive, logRetryCache);
}
 
Example 14
Source Project: hadoop   Source File: CacheManager.java    License: Apache License 2.0 5 votes vote down vote up
public CacheDirectiveInfo addDirective(
    CacheDirectiveInfo info, FSPermissionChecker pc, EnumSet<CacheFlag> flags)
    throws IOException {
  assert namesystem.hasWriteLock();
  CacheDirective directive;
  try {
    CachePool pool = getCachePool(validatePoolName(info));
    checkWritePermission(pc, pool);
    String path = validatePath(info);
    short replication = validateReplication(info, (short)1);
    long expiryTime = validateExpiryTime(info, pool.getMaxRelativeExpiryMs());
    // Do quota validation if required
    if (!flags.contains(CacheFlag.FORCE)) {
      checkLimit(pool, path, replication);
    }
    // All validation passed
    // Add a new entry with the next available ID.
    long id = getNextDirectiveId();
    directive = new CacheDirective(id, path, replication, expiryTime);
    addInternal(directive, pool);
  } catch (IOException e) {
    LOG.warn("addDirective of " + info + " failed: ", e);
    throw e;
  }
  LOG.info("addDirective of {} successful.", info);
  return directive.toInfo();
}
 
Example 15
Source Project: hadoop   Source File: PBHelper.java    License: Apache License 2.0 5 votes vote down vote up
public static int convertCacheFlags(EnumSet<CacheFlag> flags) {
  int value = 0;
  if (flags.contains(CacheFlag.FORCE)) {
    value |= CacheFlagProto.FORCE.getNumber();
  }
  return value;
}
 
Example 16
Source Project: hadoop   Source File: PBHelper.java    License: Apache License 2.0 5 votes vote down vote up
public static EnumSet<CacheFlag> convertCacheFlags(int flags) {
  EnumSet<CacheFlag> result = EnumSet.noneOf(CacheFlag.class);
  if ((flags & CacheFlagProto.FORCE_VALUE) == CacheFlagProto.FORCE_VALUE) {
    result.add(CacheFlag.FORCE);
  }
  return result;
}
 
Example 17
Source Project: hadoop   Source File: ClientNamenodeProtocolTranslatorPB.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public long addCacheDirective(CacheDirectiveInfo directive,
    EnumSet<CacheFlag> flags) throws IOException {
  try {
    AddCacheDirectiveRequestProto.Builder builder =
        AddCacheDirectiveRequestProto.newBuilder().
            setInfo(PBHelper.convert(directive));
    if (!flags.isEmpty()) {
      builder.setCacheFlags(PBHelper.convertCacheFlags(flags));
    }
    return rpcProxy.addCacheDirective(null, builder.build()).getId();
  } catch (ServiceException e) {
    throw ProtobufHelper.getRemoteException(e);
  }
}
 
Example 18
Source Project: hadoop   Source File: ClientNamenodeProtocolTranslatorPB.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void modifyCacheDirective(CacheDirectiveInfo directive,
    EnumSet<CacheFlag> flags) throws IOException {
  try {
    ModifyCacheDirectiveRequestProto.Builder builder =
        ModifyCacheDirectiveRequestProto.newBuilder().
            setInfo(PBHelper.convert(directive));
    if (!flags.isEmpty()) {
      builder.setCacheFlags(PBHelper.convertCacheFlags(flags));
    }
    rpcProxy.modifyCacheDirective(null, builder.build());
  } catch (ServiceException e) {
    throw ProtobufHelper.getRemoteException(e);
  }
}
 
Example 19
Source Project: hadoop   Source File: TestRetryCacheWithHA.java    License: Apache License 2.0 5 votes vote down vote up
@Override
void invoke() throws Exception {
  client.modifyCacheDirective(
      new CacheDirectiveInfo.Builder().
          setId(id).
          setReplication(newReplication).
          build(), EnumSet.of(CacheFlag.FORCE));
}
 
Example 20
Source Project: nnproxy   Source File: CacheRegistry.java    License: Apache License 2.0 5 votes vote down vote up
public long addCacheDirective(CacheDirectiveInfo directive, EnumSet<CacheFlag> flags)
        throws IOException {
    UpstreamManager.Upstream upstream = getUpstream(directive.getPool());
    long id = maskDirectiveId(upstream.protocol.addCacheDirective(directive, flags), upstream.fsIndex);
    reloadThread.interrupt();
    return id;
}
 
Example 21
Source Project: big-c   Source File: DFSClient.java    License: Apache License 2.0 5 votes vote down vote up
public long addCacheDirective(
    CacheDirectiveInfo info, EnumSet<CacheFlag> flags) throws IOException {
  checkOpen();
  TraceScope scope = Trace.startSpan("addCacheDirective", traceSampler);
  try {
    return namenode.addCacheDirective(info, flags);
  } catch (RemoteException re) {
    throw re.unwrapRemoteException();
  } finally {
    scope.close();
  }
}
 
Example 22
Source Project: big-c   Source File: DFSClient.java    License: Apache License 2.0 5 votes vote down vote up
public void modifyCacheDirective(
    CacheDirectiveInfo info, EnumSet<CacheFlag> flags) throws IOException {
  checkOpen();
  TraceScope scope = Trace.startSpan("modifyCacheDirective", traceSampler);
  try {
    namenode.modifyCacheDirective(info, flags);
  } catch (RemoteException re) {
    throw re.unwrapRemoteException();
  } finally {
    scope.close();
  }
}
 
Example 23
Source Project: big-c   Source File: DistributedFileSystem.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Add a new CacheDirective.
 * 
 * @param info Information about a directive to add.
 * @param flags {@link CacheFlag}s to use for this operation.
 * @return the ID of the directive that was created.
 * @throws IOException if the directive could not be added
 */
public long addCacheDirective(
    CacheDirectiveInfo info, EnumSet<CacheFlag> flags) throws IOException {
  Preconditions.checkNotNull(info.getPath());
  Path path = new Path(getPathName(fixRelativePart(info.getPath()))).
      makeQualified(getUri(), getWorkingDirectory());
  return dfs.addCacheDirective(
      new CacheDirectiveInfo.Builder(info).
          setPath(path).
          build(),
      flags);
}
 
Example 24
Source Project: big-c   Source File: DistributedFileSystem.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Modify a CacheDirective.
 * 
 * @param info Information about the directive to modify. You must set the ID
 *          to indicate which CacheDirective you want to modify.
 * @param flags {@link CacheFlag}s to use for this operation.
 * @throws IOException if the directive could not be modified
 */
public void modifyCacheDirective(
    CacheDirectiveInfo info, EnumSet<CacheFlag> flags) throws IOException {
  if (info.getPath() != null) {
    info = new CacheDirectiveInfo.Builder(info).
        setPath(new Path(getPathName(fixRelativePart(info.getPath()))).
            makeQualified(getUri(), getWorkingDirectory())).build();
  }
  dfs.modifyCacheDirective(info, flags);
}
 
Example 25
Source Project: big-c   Source File: FSNDNCacheOp.java    License: Apache License 2.0 5 votes vote down vote up
static void modifyCacheDirective(
    FSNamesystem fsn, CacheManager cacheManager, CacheDirectiveInfo directive,
    EnumSet<CacheFlag> flags, boolean logRetryCache) throws IOException {
  final FSPermissionChecker pc = getFsPermissionChecker(fsn);

  cacheManager.modifyDirective(directive, pc, flags);
  fsn.getEditLog().logModifyCacheDirectiveInfo(directive, logRetryCache);
}
 
Example 26
Source Project: big-c   Source File: CacheManager.java    License: Apache License 2.0 5 votes vote down vote up
public CacheDirectiveInfo addDirective(
    CacheDirectiveInfo info, FSPermissionChecker pc, EnumSet<CacheFlag> flags)
    throws IOException {
  assert namesystem.hasWriteLock();
  CacheDirective directive;
  try {
    CachePool pool = getCachePool(validatePoolName(info));
    checkWritePermission(pc, pool);
    String path = validatePath(info);
    short replication = validateReplication(info, (short)1);
    long expiryTime = validateExpiryTime(info, pool.getMaxRelativeExpiryMs());
    // Do quota validation if required
    if (!flags.contains(CacheFlag.FORCE)) {
      checkLimit(pool, path, replication);
    }
    // All validation passed
    // Add a new entry with the next available ID.
    long id = getNextDirectiveId();
    directive = new CacheDirective(id, path, replication, expiryTime);
    addInternal(directive, pool);
  } catch (IOException e) {
    LOG.warn("addDirective of " + info + " failed: ", e);
    throw e;
  }
  LOG.info("addDirective of {} successful.", info);
  return directive.toInfo();
}
 
Example 27
Source Project: big-c   Source File: PBHelper.java    License: Apache License 2.0 5 votes vote down vote up
public static int convertCacheFlags(EnumSet<CacheFlag> flags) {
  int value = 0;
  if (flags.contains(CacheFlag.FORCE)) {
    value |= CacheFlagProto.FORCE.getNumber();
  }
  return value;
}
 
Example 28
Source Project: big-c   Source File: PBHelper.java    License: Apache License 2.0 5 votes vote down vote up
public static EnumSet<CacheFlag> convertCacheFlags(int flags) {
  EnumSet<CacheFlag> result = EnumSet.noneOf(CacheFlag.class);
  if ((flags & CacheFlagProto.FORCE_VALUE) == CacheFlagProto.FORCE_VALUE) {
    result.add(CacheFlag.FORCE);
  }
  return result;
}
 
Example 29
Source Project: big-c   Source File: ClientNamenodeProtocolTranslatorPB.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public long addCacheDirective(CacheDirectiveInfo directive,
    EnumSet<CacheFlag> flags) throws IOException {
  try {
    AddCacheDirectiveRequestProto.Builder builder =
        AddCacheDirectiveRequestProto.newBuilder().
            setInfo(PBHelper.convert(directive));
    if (!flags.isEmpty()) {
      builder.setCacheFlags(PBHelper.convertCacheFlags(flags));
    }
    return rpcProxy.addCacheDirective(null, builder.build()).getId();
  } catch (ServiceException e) {
    throw ProtobufHelper.getRemoteException(e);
  }
}
 
Example 30
Source Project: big-c   Source File: ClientNamenodeProtocolTranslatorPB.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void modifyCacheDirective(CacheDirectiveInfo directive,
    EnumSet<CacheFlag> flags) throws IOException {
  try {
    ModifyCacheDirectiveRequestProto.Builder builder =
        ModifyCacheDirectiveRequestProto.newBuilder().
            setInfo(PBHelper.convert(directive));
    if (!flags.isEmpty()) {
      builder.setCacheFlags(PBHelper.convertCacheFlags(flags));
    }
    rpcProxy.modifyCacheDirective(null, builder.build());
  } catch (ServiceException e) {
    throw ProtobufHelper.getRemoteException(e);
  }
}