Java Code Examples for org.apache.hadoop.fs.StorageType#getTypesSupportingQuota()

The following examples show how to use org.apache.hadoop.fs.StorageType#getTypesSupportingQuota() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: DirectoryWithQuotaFeature.java From hadoop with Apache License 2.0

6 votes

private void verifyQuotaByStorageType(EnumCounters<StorageType> typeDelta)
    throws QuotaByStorageTypeExceededException {
  if (!isQuotaByStorageTypeSet()) {
    return;
  }
  for (StorageType t: StorageType.getTypesSupportingQuota()) {
    if (!isQuotaByStorageTypeSet(t)) {
      continue;
    }
    if (Quota.isViolated(quota.getTypeSpace(t), usage.getTypeSpace(t),
        typeDelta.get(t))) {
      throw new QuotaByStorageTypeExceededException(
        quota.getTypeSpace(t), usage.getTypeSpace(t) + typeDelta.get(t), t);
    }
  }
}

Example 2

Source File: PBHelper.java From big-c with Apache License 2.0

6 votes

public static ContentSummaryProto convert(ContentSummary cs) {
  if (cs == null) return null;
  ContentSummaryProto.Builder builder = ContentSummaryProto.newBuilder();
      builder.setLength(cs.getLength()).
      setFileCount(cs.getFileCount()).
      setDirectoryCount(cs.getDirectoryCount()).
      setQuota(cs.getQuota()).
      setSpaceConsumed(cs.getSpaceConsumed()).
      setSpaceQuota(cs.getSpaceQuota());

  if (cs.isTypeQuotaSet() || cs.isTypeConsumedAvailable()) {
    HdfsProtos.StorageTypeQuotaInfosProto.Builder isb =
        HdfsProtos.StorageTypeQuotaInfosProto.newBuilder();
    for (StorageType t: StorageType.getTypesSupportingQuota()) {
      HdfsProtos.StorageTypeQuotaInfoProto info =
          HdfsProtos.StorageTypeQuotaInfoProto.newBuilder().
              setType(convertStorageType(t)).
              setConsumed(cs.getTypeConsumed(t)).
              setQuota(cs.getTypeQuota(t)).
              build();
      isb.addTypeQuotaInfo(info);
    }
    builder.setTypeQuotaInfos(isb);
  }
  return builder.build();
}

Example 3

Source File: PBHelper.java From hadoop with Apache License 2.0

6 votes

public static ContentSummaryProto convert(ContentSummary cs) {
  if (cs == null) return null;
  ContentSummaryProto.Builder builder = ContentSummaryProto.newBuilder();
      builder.setLength(cs.getLength()).
      setFileCount(cs.getFileCount()).
      setDirectoryCount(cs.getDirectoryCount()).
      setQuota(cs.getQuota()).
      setSpaceConsumed(cs.getSpaceConsumed()).
      setSpaceQuota(cs.getSpaceQuota());

  if (cs.isTypeQuotaSet() || cs.isTypeConsumedAvailable()) {
    HdfsProtos.StorageTypeQuotaInfosProto.Builder isb =
        HdfsProtos.StorageTypeQuotaInfosProto.newBuilder();
    for (StorageType t: StorageType.getTypesSupportingQuota()) {
      HdfsProtos.StorageTypeQuotaInfoProto info =
          HdfsProtos.StorageTypeQuotaInfoProto.newBuilder().
              setType(convertStorageType(t)).
              setConsumed(cs.getTypeConsumed(t)).
              setQuota(cs.getTypeQuota(t)).
              build();
      isb.addTypeQuotaInfo(info);
    }
    builder.setTypeQuotaInfos(isb);
  }
  return builder.build();
}

Example 4

Source File: DirectoryWithQuotaFeature.java From big-c with Apache License 2.0

6 votes

private void verifyQuotaByStorageType(EnumCounters<StorageType> typeDelta)
    throws QuotaByStorageTypeExceededException {
  if (!isQuotaByStorageTypeSet()) {
    return;
  }
  for (StorageType t: StorageType.getTypesSupportingQuota()) {
    if (!isQuotaByStorageTypeSet(t)) {
      continue;
    }
    if (Quota.isViolated(quota.getTypeSpace(t), usage.getTypeSpace(t),
        typeDelta.get(t))) {
      throw new QuotaByStorageTypeExceededException(
        quota.getTypeSpace(t), usage.getTypeSpace(t) + typeDelta.get(t), t);
    }
  }
}

Example 5

Source File: ContentSummary.java From big-c with Apache License 2.0

5 votes

/** Returns true if any storage type consumption information is available*/
public boolean isTypeConsumedAvailable() {
  if (typeConsumed == null) {
    return false;
  }
  for (StorageType t : StorageType.getTypesSupportingQuota()) {
    if (typeConsumed[t.ordinal()] > 0) {
      return true;
    }
  }
  return false;
}

Example 6

Source File: ContentSummary.java From big-c with Apache License 2.0

5 votes

/** Returns true if any storage type quota has been set*/
public boolean isTypeQuotaSet() {
  if (typeQuota == null) {
    return false;
  }
  for (StorageType t : StorageType.getTypesSupportingQuota()) {
    if (typeQuota[t.ordinal()] > 0) {
      return true;
    }
  }
  return false;
}

Example 7

Source File: FSImageFormatPBINode.java From big-c with Apache License 2.0

5 votes

private static QuotaByStorageTypeFeatureProto.Builder
    buildQuotaByStorageTypeEntries(QuotaCounts q) {
  QuotaByStorageTypeFeatureProto.Builder b =
      QuotaByStorageTypeFeatureProto.newBuilder();
  for (StorageType t: StorageType.getTypesSupportingQuota()) {
    if (q.getTypeSpace(t) >= 0) {
      QuotaByStorageTypeEntryProto.Builder eb =
          QuotaByStorageTypeEntryProto.newBuilder().
          setStorageType(PBHelper.convertStorageType(t)).
          setQuota(q.getTypeSpace(t));
      b.addQuotas(eb);
    }
  }
  return b;
}

Example 8

Source File: DirectoryWithQuotaFeature.java From big-c with Apache License 2.0

5 votes

private String typeSpaceString() {
  StringBuilder sb = new StringBuilder();
  for (StorageType t : StorageType.getTypesSupportingQuota()) {
    sb.append("StorageType: " + t +
        (quota.getTypeSpace(t) < 0? "-":
        usage.getTypeSpace(t) + "/" + usage.getTypeSpace(t)));
  }
  return sb.toString();
}

Example 9

Source File: ContentSummary.java From hadoop with Apache License 2.0

5 votes

/** Returns true if any storage type consumption information is available*/
public boolean isTypeConsumedAvailable() {
  if (typeConsumed == null) {
    return false;
  }
  for (StorageType t : StorageType.getTypesSupportingQuota()) {
    if (typeConsumed[t.ordinal()] > 0) {
      return true;
    }
  }
  return false;
}

Example 10

Source File: ContentSummary.java From hadoop with Apache License 2.0

5 votes

/** Returns true if any storage type quota has been set*/
public boolean isTypeQuotaSet() {
  if (typeQuota == null) {
    return false;
  }
  for (StorageType t : StorageType.getTypesSupportingQuota()) {
    if (typeQuota[t.ordinal()] > 0) {
      return true;
    }
  }
  return false;
}

Example 11

Source File: FSImageFormatPBINode.java From hadoop with Apache License 2.0

5 votes

private static QuotaByStorageTypeFeatureProto.Builder
    buildQuotaByStorageTypeEntries(QuotaCounts q) {
  QuotaByStorageTypeFeatureProto.Builder b =
      QuotaByStorageTypeFeatureProto.newBuilder();
  for (StorageType t: StorageType.getTypesSupportingQuota()) {
    if (q.getTypeSpace(t) >= 0) {
      QuotaByStorageTypeEntryProto.Builder eb =
          QuotaByStorageTypeEntryProto.newBuilder().
          setStorageType(PBHelper.convertStorageType(t)).
          setQuota(q.getTypeSpace(t));
      b.addQuotas(eb);
    }
  }
  return b;
}

Example 12

Source File: DirectoryWithQuotaFeature.java From hadoop with Apache License 2.0

5 votes

private String typeSpaceString() {
  StringBuilder sb = new StringBuilder();
  for (StorageType t : StorageType.getTypesSupportingQuota()) {
    sb.append("StorageType: " + t +
        (quota.getTypeSpace(t) < 0? "-":
        usage.getTypeSpace(t) + "/" + usage.getTypeSpace(t)));
  }
  return sb.toString();
}

Example 13

Source File: TestQuotaByStorageType.java From hadoop with Apache License 2.0

4 votes

@Test
public void testQuotaByStorageTypePersistenceInFsImage() throws IOException {
  final String METHOD_NAME = GenericTestUtils.getMethodName();
  final Path testDir = new Path(dir, METHOD_NAME);
  Path createdFile1 = new Path(testDir, "created_file1.data");
  dfs.mkdirs(testDir);

  // set storage policy on testDir to ONESSD
  dfs.setStoragePolicy(testDir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);

  // set quota by storage type on testDir
  final long SSD_QUOTA = BLOCKSIZE * 4;
  dfs.setQuotaByStorageType(testDir, StorageType.SSD, SSD_QUOTA);
  INode testDirNode = fsdir.getINode4Write(testDir.toString());
  assertTrue(testDirNode.isDirectory());
  assertTrue(testDirNode.isQuotaSet());

  // Create file of size 2 * BLOCKSIZE under testDir
  long file1Len = BLOCKSIZE * 2;
  int bufLen = BLOCKSIZE / 16;
  DFSTestUtil.createFile(dfs, createdFile1, bufLen, file1Len, BLOCKSIZE, REPLICATION, seed);

  // Verify SSD consumed before namenode restart
  long ssdConsumed = testDirNode.asDirectory().getDirectoryWithQuotaFeature()
      .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
  assertEquals(file1Len, ssdConsumed);

  // Restart the namenode with checkpoint to make sure fsImage is correct
  dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
  dfs.saveNamespace();
  dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
  cluster.restartNameNode(true);

  INode testDirNodeAfterNNRestart = fsdir.getINode4Write(testDir.toString());
  assertTrue(testDirNode.isDirectory());
  assertTrue(testDirNode.isQuotaSet());

  QuotaCounts qc = testDirNodeAfterNNRestart.getQuotaCounts();
  assertEquals(SSD_QUOTA, qc.getTypeSpace(StorageType.SSD));
  for (StorageType t: StorageType.getTypesSupportingQuota()) {
    if (t != StorageType.SSD) {
      assertEquals(HdfsConstants.QUOTA_RESET, qc.getTypeSpace(t));
    }
  }

  long ssdConsumedAfterNNRestart = testDirNodeAfterNNRestart.asDirectory()
      .getDirectoryWithQuotaFeature()
      .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
  assertEquals(file1Len, ssdConsumedAfterNNRestart);
}

Example 14

Source File: FSImage.java From big-c with Apache License 2.0

4 votes

private static void updateCountForQuotaRecursively(BlockStoragePolicySuite bsps,
    byte blockStoragePolicyId, INodeDirectory dir, QuotaCounts counts) {
  final long parentNamespace = counts.getNameSpace();
  final long parentStoragespace = counts.getStorageSpace();
  final EnumCounters<StorageType> parentTypeSpaces = counts.getTypeSpaces();

  dir.computeQuotaUsage4CurrentDirectory(bsps, blockStoragePolicyId, counts);
  
  for (INode child : dir.getChildrenList(Snapshot.CURRENT_STATE_ID)) {
    final byte childPolicyId = child.getStoragePolicyIDForQuota(blockStoragePolicyId);
    if (child.isDirectory()) {
      updateCountForQuotaRecursively(bsps, childPolicyId,
          child.asDirectory(), counts);
    } else {
      // file or symlink: count here to reduce recursive calls.
      child.computeQuotaUsage(bsps, childPolicyId, counts, false,
          Snapshot.CURRENT_STATE_ID);
    }
  }
    
  if (dir.isQuotaSet()) {
    // check if quota is violated. It indicates a software bug.
    final QuotaCounts q = dir.getQuotaCounts();

    final long namespace = counts.getNameSpace() - parentNamespace;
    final long nsQuota = q.getNameSpace();
    if (Quota.isViolated(nsQuota, namespace)) {
      LOG.warn("Namespace quota violation in image for "
          + dir.getFullPathName()
          + " quota = " + nsQuota + " < consumed = " + namespace);
    }

    final long ssConsumed = counts.getStorageSpace() - parentStoragespace;
    final long ssQuota = q.getStorageSpace();
    if (Quota.isViolated(ssQuota, ssConsumed)) {
      LOG.warn("Storagespace quota violation in image for "
          + dir.getFullPathName()
          + " quota = " + ssQuota + " < consumed = " + ssConsumed);
    }

    final EnumCounters<StorageType> typeSpaces =
        new EnumCounters<StorageType>(StorageType.class);
    for (StorageType t : StorageType.getTypesSupportingQuota()) {
      final long typeSpace = counts.getTypeSpaces().get(t) -
          parentTypeSpaces.get(t);
      final long typeQuota = q.getTypeSpaces().get(t);
      if (Quota.isViolated(typeQuota, typeSpace)) {
        LOG.warn("Storage type quota violation in image for "
            + dir.getFullPathName()
            + " type = " + t.toString() + " quota = "
            + typeQuota + " < consumed " + typeSpace);
      }
    }

    dir.getDirectoryWithQuotaFeature().setSpaceConsumed(namespace, ssConsumed,
        typeSpaces);
  }
}

Example 15

Source File: TestQuotaByStorageType.java From hadoop with Apache License 2.0

4 votes

@Test
public void testQuotaByStorageTypePersistenceInEditLog() throws IOException {
  final String METHOD_NAME = GenericTestUtils.getMethodName();
  final Path testDir = new Path(dir, METHOD_NAME);
  Path createdFile1 = new Path(testDir, "created_file1.data");
  dfs.mkdirs(testDir);

  // set storage policy on testDir to ONESSD
  dfs.setStoragePolicy(testDir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);

  // set quota by storage type on testDir
  final long SSD_QUOTA = BLOCKSIZE * 4;
  dfs.setQuotaByStorageType(testDir, StorageType.SSD, SSD_QUOTA);
  INode testDirNode = fsdir.getINode4Write(testDir.toString());
  assertTrue(testDirNode.isDirectory());
  assertTrue(testDirNode.isQuotaSet());

  // Create file of size 2 * BLOCKSIZE under testDir
  long file1Len = BLOCKSIZE * 2;
  int bufLen = BLOCKSIZE / 16;
  DFSTestUtil.createFile(dfs, createdFile1, bufLen, file1Len, BLOCKSIZE, REPLICATION, seed);

  // Verify SSD consumed before namenode restart
  long ssdConsumed = testDirNode.asDirectory().getDirectoryWithQuotaFeature()
      .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
  assertEquals(file1Len, ssdConsumed);

  // Restart namenode to make sure the editlog is correct
  cluster.restartNameNode(true);

  INode testDirNodeAfterNNRestart = fsdir.getINode4Write(testDir.toString());
  // Verify quota is still set
  assertTrue(testDirNode.isDirectory());
  assertTrue(testDirNode.isQuotaSet());

  QuotaCounts qc = testDirNodeAfterNNRestart.getQuotaCounts();
  assertEquals(SSD_QUOTA, qc.getTypeSpace(StorageType.SSD));
  for (StorageType t: StorageType.getTypesSupportingQuota()) {
    if (t != StorageType.SSD) {
      assertEquals(HdfsConstants.QUOTA_RESET, qc.getTypeSpace(t));
    }
  }

  long ssdConsumedAfterNNRestart = testDirNodeAfterNNRestart.asDirectory()
      .getDirectoryWithQuotaFeature()
      .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
  assertEquals(file1Len, ssdConsumedAfterNNRestart);
}

Example 16

Source File: TestQuotaByStorageType.java From big-c with Apache License 2.0

4 votes

@Test
public void testQuotaByStorageTypePersistenceInEditLog() throws IOException {
  final String METHOD_NAME = GenericTestUtils.getMethodName();
  final Path testDir = new Path(dir, METHOD_NAME);
  Path createdFile1 = new Path(testDir, "created_file1.data");
  dfs.mkdirs(testDir);

  // set storage policy on testDir to ONESSD
  dfs.setStoragePolicy(testDir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);

  // set quota by storage type on testDir
  final long SSD_QUOTA = BLOCKSIZE * 4;
  dfs.setQuotaByStorageType(testDir, StorageType.SSD, SSD_QUOTA);
  INode testDirNode = fsdir.getINode4Write(testDir.toString());
  assertTrue(testDirNode.isDirectory());
  assertTrue(testDirNode.isQuotaSet());

  // Create file of size 2 * BLOCKSIZE under testDir
  long file1Len = BLOCKSIZE * 2;
  int bufLen = BLOCKSIZE / 16;
  DFSTestUtil.createFile(dfs, createdFile1, bufLen, file1Len, BLOCKSIZE, REPLICATION, seed);

  // Verify SSD consumed before namenode restart
  long ssdConsumed = testDirNode.asDirectory().getDirectoryWithQuotaFeature()
      .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
  assertEquals(file1Len, ssdConsumed);

  // Restart namenode to make sure the editlog is correct
  cluster.restartNameNode(true);

  INode testDirNodeAfterNNRestart = fsdir.getINode4Write(testDir.toString());
  // Verify quota is still set
  assertTrue(testDirNode.isDirectory());
  assertTrue(testDirNode.isQuotaSet());

  QuotaCounts qc = testDirNodeAfterNNRestart.getQuotaCounts();
  assertEquals(SSD_QUOTA, qc.getTypeSpace(StorageType.SSD));
  for (StorageType t: StorageType.getTypesSupportingQuota()) {
    if (t != StorageType.SSD) {
      assertEquals(HdfsConstants.QUOTA_RESET, qc.getTypeSpace(t));
    }
  }

  long ssdConsumedAfterNNRestart = testDirNodeAfterNNRestart.asDirectory()
      .getDirectoryWithQuotaFeature()
      .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
  assertEquals(file1Len, ssdConsumedAfterNNRestart);
}

Example 17

Source File: TestQuotaByStorageType.java From big-c with Apache License 2.0

4 votes

@Test
public void testQuotaByStorageTypePersistenceInFsImage() throws IOException {
  final String METHOD_NAME = GenericTestUtils.getMethodName();
  final Path testDir = new Path(dir, METHOD_NAME);
  Path createdFile1 = new Path(testDir, "created_file1.data");
  dfs.mkdirs(testDir);

  // set storage policy on testDir to ONESSD
  dfs.setStoragePolicy(testDir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);

  // set quota by storage type on testDir
  final long SSD_QUOTA = BLOCKSIZE * 4;
  dfs.setQuotaByStorageType(testDir, StorageType.SSD, SSD_QUOTA);
  INode testDirNode = fsdir.getINode4Write(testDir.toString());
  assertTrue(testDirNode.isDirectory());
  assertTrue(testDirNode.isQuotaSet());

  // Create file of size 2 * BLOCKSIZE under testDir
  long file1Len = BLOCKSIZE * 2;
  int bufLen = BLOCKSIZE / 16;
  DFSTestUtil.createFile(dfs, createdFile1, bufLen, file1Len, BLOCKSIZE, REPLICATION, seed);

  // Verify SSD consumed before namenode restart
  long ssdConsumed = testDirNode.asDirectory().getDirectoryWithQuotaFeature()
      .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
  assertEquals(file1Len, ssdConsumed);

  // Restart the namenode with checkpoint to make sure fsImage is correct
  dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
  dfs.saveNamespace();
  dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
  cluster.restartNameNode(true);

  INode testDirNodeAfterNNRestart = fsdir.getINode4Write(testDir.toString());
  assertTrue(testDirNode.isDirectory());
  assertTrue(testDirNode.isQuotaSet());

  QuotaCounts qc = testDirNodeAfterNNRestart.getQuotaCounts();
  assertEquals(SSD_QUOTA, qc.getTypeSpace(StorageType.SSD));
  for (StorageType t: StorageType.getTypesSupportingQuota()) {
    if (t != StorageType.SSD) {
      assertEquals(HdfsConstants.QUOTA_RESET, qc.getTypeSpace(t));
    }
  }

  long ssdConsumedAfterNNRestart = testDirNodeAfterNNRestart.asDirectory()
      .getDirectoryWithQuotaFeature()
      .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
  assertEquals(file1Len, ssdConsumedAfterNNRestart);
}

Example 18

Source File: FSImage.java From hadoop with Apache License 2.0

4 votes

private static void updateCountForQuotaRecursively(BlockStoragePolicySuite bsps,
    byte blockStoragePolicyId, INodeDirectory dir, QuotaCounts counts) {
  final long parentNamespace = counts.getNameSpace();
  final long parentStoragespace = counts.getStorageSpace();
  final EnumCounters<StorageType> parentTypeSpaces = counts.getTypeSpaces();

  dir.computeQuotaUsage4CurrentDirectory(bsps, blockStoragePolicyId, counts);
  
  for (INode child : dir.getChildrenList(Snapshot.CURRENT_STATE_ID)) {
    final byte childPolicyId = child.getStoragePolicyIDForQuota(blockStoragePolicyId);
    if (child.isDirectory()) {
      updateCountForQuotaRecursively(bsps, childPolicyId,
          child.asDirectory(), counts);
    } else {
      // file or symlink: count here to reduce recursive calls.
      child.computeQuotaUsage(bsps, childPolicyId, counts, false,
          Snapshot.CURRENT_STATE_ID);
    }
  }
    
  if (dir.isQuotaSet()) {
    // check if quota is violated. It indicates a software bug.
    final QuotaCounts q = dir.getQuotaCounts();

    final long namespace = counts.getNameSpace() - parentNamespace;
    final long nsQuota = q.getNameSpace();
    if (Quota.isViolated(nsQuota, namespace)) {
      LOG.warn("Namespace quota violation in image for "
          + dir.getFullPathName()
          + " quota = " + nsQuota + " < consumed = " + namespace);
    }

    final long ssConsumed = counts.getStorageSpace() - parentStoragespace;
    final long ssQuota = q.getStorageSpace();
    if (Quota.isViolated(ssQuota, ssConsumed)) {
      LOG.warn("Storagespace quota violation in image for "
          + dir.getFullPathName()
          + " quota = " + ssQuota + " < consumed = " + ssConsumed);
    }

    final EnumCounters<StorageType> typeSpaces =
        new EnumCounters<StorageType>(StorageType.class);
    for (StorageType t : StorageType.getTypesSupportingQuota()) {
      final long typeSpace = counts.getTypeSpaces().get(t) -
          parentTypeSpaces.get(t);
      final long typeQuota = q.getTypeSpaces().get(t);
      if (Quota.isViolated(typeQuota, typeSpace)) {
        LOG.warn("Storage type quota violation in image for "
            + dir.getFullPathName()
            + " type = " + t.toString() + " quota = "
            + typeQuota + " < consumed " + typeSpace);
      }
    }

    dir.getDirectoryWithQuotaFeature().setSpaceConsumed(namespace, ssConsumed,
        typeSpaces);
  }
}