Java Code Examples for org.apache.hadoop.hbase.CellUtil#cloneValue()

/**
 * Gets a list of {@link RegionPruneInfo} for given regions. Returns all regions if the given regions set is null.
 *
 * @param regions a set of regions
 * @return list of {@link RegionPruneInfo}s.
 * @throws IOException when not able to read the data from HBase
 */
public List<RegionPruneInfo> getPruneInfoForRegions(@Nullable SortedSet<byte[]> regions) throws IOException {
  List<RegionPruneInfo> regionPruneInfos = new ArrayList<>();
  try (Table stateTable = stateTableSupplier.get()) {
    byte[] startRow = makeRegionKey(EMPTY_BYTE_ARRAY);
    Scan scan = new Scan(startRow, REGION_KEY_PREFIX_STOP);
    scan.addColumn(FAMILY, PRUNE_UPPER_BOUND_COL);

    try (ResultScanner scanner = stateTable.getScanner(scan)) {
      Result next;
      while ((next = scanner.next()) != null) {
        byte[] region = getRegionFromKey(next.getRow());
        if (regions == null || regions.contains(region)) {
          Cell cell = next.getColumnLatestCell(FAMILY, PRUNE_UPPER_BOUND_COL);
          if (cell != null) {
            byte[] pruneUpperBoundBytes = CellUtil.cloneValue(cell);
            long timestamp = cell.getTimestamp();
            regionPruneInfos.add(new RegionPruneInfo(region, Bytes.toStringBinary(region),
                                                     Bytes.toLong(pruneUpperBoundBytes), timestamp));
          }
        }
      }
    }
  }
  return Collections.unmodifiableList(regionPruneInfos);
}
 

public static InternalDataStatistics<?, ?, ?> getMergedStats(final List<Cell> rowCells) {
  InternalDataStatistics<?, ?, ?> mergedStats = null;
  for (final Cell cell : rowCells) {
    final byte[] byteValue = CellUtil.cloneValue(cell);
    final InternalDataStatistics<?, ?, ?> stats =
        (InternalDataStatistics) URLClassloaderUtils.fromBinary(byteValue);

    if (mergedStats != null) {
      mergedStats.merge(stats);
    } else {
      mergedStats = stats;
    }
  }

  return mergedStats;
}
 

private AgentLifeCycleBo createAgentLifeCycleBo(Cell valueCell) {
    if (valueCell == null) {
        return null;
    }
    byte[] value = CellUtil.cloneValue(valueCell);
    final Buffer buffer = new FixedBuffer(value);
    
    final int version = buffer.readInt();
    if (version == 0) {
        final String agentId = buffer.readPrefixedString();
        final long startTimestamp = buffer.readLong();
        final long eventTimestamp = buffer.readLong();
        final long eventIdentifier = buffer.readLong();
        final AgentLifeCycleState agentLifeCycleState = AgentLifeCycleState.getStateByCode(buffer.readShort());
        final AgentLifeCycleBo agentLifeCycleBo = new AgentLifeCycleBo(agentId, startTimestamp, eventTimestamp, eventIdentifier, agentLifeCycleState);
        return agentLifeCycleBo;
    }
    return null;
}
 

public String[] getListOfBackupIdsFromDeleteOperation() throws IOException {
  LOG.trace("Get delete operation for backup ids");

  Get get = createGetForDeleteOperation();
  try (Table table = connection.getTable(tableName)) {
    Result res = table.get(get);
    if (res.isEmpty()) {
      return null;
    }
    Cell cell = res.listCells().get(0);
    byte[] val = CellUtil.cloneValue(cell);
    if (val.length == 0) {
      return null;
    }
    return new String(val).split(",");
  }
}
 

/**
 * attempts to approximately set the job submit time based on the last
 * modification time of the job history file
 * @param value result
 * @return approximate job submit time
 * @throws MissingColumnInResultException
 */
public long getApproxSubmitTime(Result value)
    throws MissingColumnInResultException {
  if (value == null) {
    throw new IllegalArgumentException(
        "Cannot get last modification time from " + "a null hbase result");
  }

  Cell cell = value.getColumnLatestCell(Constants.INFO_FAM_BYTES,
      Constants.JOBHISTORY_LAST_MODIFIED_COL_BYTES);

  if (cell == null) {
    throw new MissingColumnInResultException(Constants.INFO_FAM_BYTES,
        Constants.JOBHISTORY_LAST_MODIFIED_COL_BYTES);
  }

  byte[] lastModTimeBytes = CellUtil.cloneValue(cell);
  // we try to approximately set the job submit time based on when the job
  // history file
  // was last modified and an average job duration
  long lastModTime = Bytes.toLong(lastModTimeBytes);
  long jobSubmitTimeMillis = lastModTime - Constants.AVERGAE_JOB_DURATION;
  LOG.debug("Approximate job submit time is " + jobSubmitTimeMillis
      + " based on " + lastModTime);
  return jobSubmitTimeMillis;
}
 

/**
 * Gets a list of {@link RegionPruneInfo} for given regions. Returns all regions if the given regions set is null.
 *
 * @param regions a set of regions
 * @return list of {@link RegionPruneInfo}s.
 * @throws IOException when not able to read the data from HBase
 */
public List<RegionPruneInfo> getPruneInfoForRegions(@Nullable SortedSet<byte[]> regions) throws IOException {
  List<RegionPruneInfo> regionPruneInfos = new ArrayList<>();
  try (Table stateTable = stateTableSupplier.get()) {
    byte[] startRow = makeRegionKey(EMPTY_BYTE_ARRAY);
    Scan scan = new Scan(startRow, REGION_KEY_PREFIX_STOP);
    scan.addColumn(FAMILY, PRUNE_UPPER_BOUND_COL);

    try (ResultScanner scanner = stateTable.getScanner(scan)) {
      Result next;
      while ((next = scanner.next()) != null) {
        byte[] region = getRegionFromKey(next.getRow());
        if (regions == null || regions.contains(region)) {
          Cell cell = next.getColumnLatestCell(FAMILY, PRUNE_UPPER_BOUND_COL);
          if (cell != null) {
            byte[] pruneUpperBoundBytes = CellUtil.cloneValue(cell);
            long timestamp = cell.getTimestamp();
            regionPruneInfos.add(new RegionPruneInfo(region, Bytes.toStringBinary(region),
                                                     Bytes.toLong(pruneUpperBoundBytes), timestamp));
          }
        }
      }
    }
  }
  return Collections.unmodifiableList(regionPruneInfos);
}
 

/**
 * Gets a list of {@link RegionPruneInfo} for given regions. Returns all regions if the given regions set is null.
 *
 * @param regions a set of regions
 * @return list of {@link RegionPruneInfo}s.
 * @throws IOException when not able to read the data from HBase
 */
public List<RegionPruneInfo> getPruneInfoForRegions(@Nullable SortedSet<byte[]> regions) throws IOException {
  List<RegionPruneInfo> regionPruneInfos = new ArrayList<>();
  try (Table stateTable = stateTableSupplier.get()) {
    byte[] startRow = makeRegionKey(EMPTY_BYTE_ARRAY);
    Scan scan = new Scan(startRow, REGION_KEY_PREFIX_STOP);
    scan.addColumn(FAMILY, PRUNE_UPPER_BOUND_COL);

    try (ResultScanner scanner = stateTable.getScanner(scan)) {
      Result next;
      while ((next = scanner.next()) != null) {
        byte[] region = getRegionFromKey(next.getRow());
        if (regions == null || regions.contains(region)) {
          Cell cell = next.getColumnLatestCell(FAMILY, PRUNE_UPPER_BOUND_COL);
          if (cell != null) {
            byte[] pruneUpperBoundBytes = CellUtil.cloneValue(cell);
            long timestamp = cell.getTimestamp();
            regionPruneInfos.add(new RegionPruneInfo(region, Bytes.toStringBinary(region),
                                                     Bytes.toLong(pruneUpperBoundBytes), timestamp));
          }
        }
      }
    }
  }
  return Collections.unmodifiableList(regionPruneInfos);
}
 

private void updateMutationAddingTags(final Mutation m) {
  byte[] attribute = m.getAttribute("visibility");
  byte[] cf = null;
  List<Cell> updatedCells = new ArrayList<>();
  if (attribute != null) {
    for (List<? extends Cell> edits : m.getFamilyCellMap().values()) {
      for (Cell cell : edits) {
        KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
        if (cf == null) {
          cf = CellUtil.cloneFamily(kv);
        }
        Tag tag = new ArrayBackedTag((byte) 1, attribute);
        List<Tag> tagList = new ArrayList<>();
        tagList.add(tag);

        KeyValue newKV = new KeyValue(CellUtil.cloneRow(kv), 0, kv.getRowLength(),
            CellUtil.cloneFamily(kv), 0, kv.getFamilyLength(), CellUtil.cloneQualifier(kv), 0,
            kv.getQualifierLength(), kv.getTimestamp(),
            KeyValue.Type.codeToType(kv.getTypeByte()), CellUtil.cloneValue(kv), 0,
            kv.getValueLength(), tagList);
        ((List<Cell>) updatedCells).add(newKV);
      }
    }
    m.getFamilyCellMap().remove(cf);
    // Update the family map
    m.getFamilyCellMap().put(cf, updatedCells);
  }
}
 

protected void assertNResult(Result result, byte [] row,
  byte [][] families, byte [][] qualifiers, byte [][] values, int [][] idxs) {
  assertTrue("Expected row [" + Bytes.toString(row) + "] " +
      "Got row [" + Bytes.toString(result.getRow()) +"]",
    equals(row, result.getRow()));
  assertEquals("Expected " + idxs.length + " keys but result contains " + result.size(),
    result.size(), idxs.length);

  Cell [] keys = result.rawCells();

  for(int i=0;i<keys.length;i++) {
    byte [] family = families[idxs[i][0]];
    byte [] qualifier = qualifiers[idxs[i][1]];
    byte [] value = values[idxs[i][2]];
    Cell key = keys[i];

    byte[] famb = CellUtil.cloneFamily(key);
    byte[] qualb = CellUtil.cloneQualifier(key);
    byte[] valb = CellUtil.cloneValue(key);
    assertTrue("(" + i + ") Expected family [" + Bytes.toString(family)
        + "] " + "Got family [" + Bytes.toString(famb) + "]",
      equals(family, famb));
    assertTrue("(" + i + ") Expected qualifier [" + Bytes.toString(qualifier)
        + "] " + "Got qualifier [" + Bytes.toString(qualb) + "]",
      equals(qualifier, qualb));
    assertTrue("(" + i + ") Expected value [" + Bytes.toString(value) + "] "
        + "Got value [" + Bytes.toString(valb) + "]",
      equals(value, valb));
  }
}
 

/**
 * Returns the value of the first column in the Result.
 * @return value of the first column
 */
public byte [] value() {
  if (isEmpty()) {
    return null;
  }
  return CellUtil.cloneValue(cells[0]);
}
 

@Override
public void emitTuples()
{
  try {
    ResultScanner scanner = getResultScanner();

    for (int i = 0; i < batchSize; ++i) {
      Result result = scanner.next();
      if (result == null) {
        break;
      }

      nameValueMap.clear();

      //row is imsi
      nameValueMap.put("imsi", result.getRow());

      List<Cell> cells = result.listCells();
      for (Cell cell : cells) {
        String columnName = Bytes.toString(CellUtil.cloneQualifier(cell));
        byte[] value = CellUtil.cloneValue(cell);
        nameValueMap.put(columnName, value);
      }
      EnrichedCDR cdr = new EnrichedCDR(nameValueMap);

      outputPort.emit(cdr);
    }

  } catch (Exception e) {
    logger.error("emitTuples() exception", e);
  }
}
 

@Test
public void testRowAndValueFilterWithBinaryComponentComparator() throws IOException {
  //SELECT * from table where a=1 and b > 10 and b < 20 and c > 90 and c < 100 and d=1
  //and value has 'y' at position 1"
  tableName = TableName.valueOf(name.getMethodName());
  Table ht = TEST_UTIL.createTable(tableName, family, Integer.MAX_VALUE);
  generateRows(ht, family, qf);
  FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
  setRowFilters(filterList);
  setValueFilters(filterList);
  Scan scan = new Scan();
  scan.setFilter(filterList);
  List<Cell> result = getResults(ht,scan);
  for(Cell cell: result){
    byte[] key = CellUtil.cloneRow(cell);
    int a = Bytes.readAsInt(key,aOffset,4);
    int b = Bytes.readAsInt(key,bOffset,4);
    int c = Bytes.readAsInt(key,cOffset,4);
    int d = Bytes.readAsInt(key,dOffset,4);
    assertTrue(a == 1 &&
               b > 10 &&
               b < 20 &&
               c > 90 &&
               c < 100 &&
               d == 1);
    byte[] value = CellUtil.cloneValue(cell);
    assertTrue(Bytes.toString(value).charAt(1) == 'y');
  }
  ht.close();
}
 

private void copyAcl(String origCubeId, String newCubeId, String projectName) throws Exception {
    String projectResPath = ProjectInstance.concatResourcePath(projectName);
    Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
    ProjectInstance project = store.getResource(projectResPath, projectSerializer);
    String projUUID = project.getUuid();
    Table aclHtable = null;
    try {
        aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));

        // cube acl
        Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
        if (result.listCells() != null) {
            for (Cell cell : result.listCells()) {
                byte[] family = CellUtil.cloneFamily(cell);
                byte[] column = CellUtil.cloneQualifier(cell);
                byte[] value = CellUtil.cloneValue(cell);

                // use the target project uuid as the parent
                if (Bytes.toString(family).equals(ACL_INFO_FAMILY) && Bytes.toString(column).equals(ACL_INFO_FAMILY_PARENT_COLUMN)) {
                    String valueString = "{\"id\":\"" + projUUID + "\",\"type\":\"org.apache.kylin.metadata.project.ProjectInstance\"}";
                    value = Bytes.toBytes(valueString);
                }
                Put put = new Put(Bytes.toBytes(newCubeId));
                put.add(family, column, value);
                aclHtable.put(put);
            }
        }
    } finally {
        IOUtils.closeQuietly(aclHtable);
    }
}
 

/**
 * Read the timestamp for each region server log after the last successful backup. Each table has
 * its own set of the timestamps. The info is stored for each table as a concatenated string of
 * rs->timestapmp
 * @param backupRoot root directory path to backup
 * @return the timestamp for each region server. key: tableName value:
 *         RegionServer,PreviousTimeStamp
 * @throws IOException exception
 */
public HashMap<TableName, HashMap<String, Long>> readLogTimestampMap(String backupRoot)
    throws IOException {
  if (LOG.isTraceEnabled()) {
    LOG.trace("read RS log ts from backup system table for root=" + backupRoot);
  }

  HashMap<TableName, HashMap<String, Long>> tableTimestampMap = new HashMap<>();

  Scan scan = createScanForReadLogTimestampMap(backupRoot);
  try (Table table = connection.getTable(tableName);
      ResultScanner scanner = table.getScanner(scan)) {
    Result res;
    while ((res = scanner.next()) != null) {
      res.advance();
      Cell cell = res.current();
      byte[] row = CellUtil.cloneRow(cell);
      String tabName = getTableNameForReadLogTimestampMap(row);
      TableName tn = TableName.valueOf(tabName);
      byte[] data = CellUtil.cloneValue(cell);
      if (data == null) {
        throw new IOException("Data of last backup data from backup system table "
            + "is empty. Create a backup first.");
      }
      if (data != null && data.length > 0) {
        HashMap<String, Long> lastBackup =
            fromTableServerTimestampProto(BackupProtos.TableServerTimestamp.parseFrom(data));
        tableTimestampMap.put(tn, lastBackup);
      }
    }
    return tableTimestampMap;
  }
}
 

/**
 * Returns the value of a column from a Result object.
 *
 * @param result HBase table row
 * @param column HBase column to be retrieved
 * @return HBase column value
 */
byte[] getColumnValue(Result result, HBaseColumnDescriptor column) {
    // if column does not contain a value, return null
    if (!result.containsColumn(column.columnFamilyBytes(),
            column.qualifierBytes())) {
        return null;
    }

    // else, get the latest version of the requested column
    Cell cell = result.getColumnLatestCell(column.columnFamilyBytes(), column.qualifierBytes());
    return CellUtil.cloneValue(cell);
}
 

@Override
public List<SpanBo> mapRow(Result result, int rowNum) throws Exception {
    if (result.isEmpty()) {
        return Collections.emptyList();
    }


    byte[] rowKey = result.getRow();
    final TransactionId transactionId = this.rowKeyDecoder.decodeRowKey(rowKey);

    final Cell[] rawCells = result.rawCells();

    ListMultimap<AgentKey, SpanBo> spanMap = LinkedListMultimap.create();
    List<SpanChunkBo> spanChunkList = new ArrayList<>();

    final SpanDecodingContext decodingContext = new SpanDecodingContext();
    decodingContext.setTransactionId(transactionId);

    for (Cell cell : rawCells) {
        SpanDecoder spanDecoder = null;
        // only if family name is "span"
        if (CellUtil.matchingFamily(cell, HbaseColumnFamily.TRACE_V2_SPAN.getName())) {

            decodingContext.setCollectorAcceptedTime(cell.getTimestamp());

            final Buffer qualifier = new FixedBuffer(CellUtil.cloneQualifier(cell));
            final Buffer columnValue = new FixedBuffer(CellUtil.cloneValue(cell));

            spanDecoder = resolveDecoder(columnValue);
            final Object decodeObject = spanDecoder.decode(qualifier, columnValue, decodingContext);
            if (decodeObject instanceof SpanBo) {
                SpanBo spanBo = (SpanBo) decodeObject;
                if (logger.isDebugEnabled()) {
                    logger.debug("spanBo:{}", spanBo);
                }
                AgentKey agentKey = newAgentKey(spanBo);
                spanMap.put(agentKey, spanBo);
            } else if (decodeObject instanceof SpanChunkBo) {
                SpanChunkBo spanChunkBo = (SpanChunkBo) decodeObject;
                if (logger.isDebugEnabled()) {
                    logger.debug("spanChunkBo:{}", spanChunkBo);
                }
                spanChunkList.add(spanChunkBo);
            }

        } else {
            if (logger.isWarnEnabled()) {
                String columnFamily = Bytes.toStringBinary(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength());
                logger.warn("Unknown ColumnFamily :{}", columnFamily);
            }
        }
        nextCell(spanDecoder, decodingContext);
    }
    decodingContext.finish();


    return buildSpanBoList(spanMap, spanChunkList);

}
 

/**
 * Constructor from KeyValue
 * @param cell
 */
public CellModel(org.apache.hadoop.hbase.Cell cell) {
  this(CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell), cell.getTimestamp(), CellUtil
      .cloneValue(cell));
}
 

/**
 * @param result from the {@link Scan} from
 *          {@link getHistoryRawTableScan(String, String, String, boolean,
 *          boolean, boolean)}
 * @return the configuration part.
 * @throws MissingColumnInResultException when the result does not contain
 *           {@link Constants#RAW_FAM}, {@link Constants#JOBCONF_COL}.
 */
public Configuration createConfigurationFromResult(Result result)
    throws MissingColumnInResultException {

  if (result == null) {
    throw new IllegalArgumentException("Cannot create InputStream from null");
  }

  Cell cell = result.getColumnLatestCell(Constants.RAW_FAM_BYTES,
      Constants.JOBCONF_COL_BYTES);

  // Create a jobConf from the raw input
  Configuration jobConf = new Configuration(false);

  byte[] jobConfRawBytes = null;
  if (cell != null) {
    jobConfRawBytes = CellUtil.cloneValue(cell);
  }
  if (jobConfRawBytes == null || jobConfRawBytes.length == 0) {
    throw new MissingColumnInResultException(Constants.RAW_FAM_BYTES,
        Constants.JOBCONF_COL_BYTES);
  }

  InputStream in = new ByteArrayInputStream(jobConfRawBytes);
  jobConf.addResource(in);

  // Configuration property loading is lazy, so we need to force a load from
  // the input stream
  try {
    int size = jobConf.size();
    if (LOG.isDebugEnabled()) {
      LOG.info(
          "Loaded " + size + " job configuration properties from result");
    }
  } catch (Exception e) {
    throw new ProcessingException("Invalid configuration from result "
        + Bytes.toStringBinary(result.getRow()), e);
  }

  return jobConf;
}
 

/**
 * Looks at every value of the mapreduce output and verifies that indeed
 * the values have been reversed.
 *
 * @param table Table to scan.
 * @throws IOException
 * @throws NullPointerException if we failed to find a cell value
 */
private void verifyAttempt(final Table table)
    throws IOException, NullPointerException {
  Scan scan = new Scan();
  scan.addFamily(INPUT_FAMILY);
  scan.addFamily(OUTPUT_FAMILY);
  ResultScanner scanner = table.getScanner(scan);
  try {
    Iterator<Result> itr = scanner.iterator();
    assertTrue(itr.hasNext());
    while(itr.hasNext()) {
      Result r = itr.next();
      if (LOG.isDebugEnabled()) {
        if (r.size() > 2 ) {
          throw new IOException("Too many results, expected 2 got " +
              r.size());
        }
      }
      byte[] firstValue = null;
      byte[] secondValue = null;
      int count = 0;
      for(Cell kv : r.listCells()) {
        if (count == 0) {
          firstValue = CellUtil.cloneValue(kv);
        }else if (count == 1) {
          secondValue = CellUtil.cloneValue(kv);
        }else if (count == 2) {
          break;
        }
        count++;
      }
      String first = "";
      if (firstValue == null) {
        throw new NullPointerException(Bytes.toString(r.getRow()) +
            ": first value is null");
      }
      first = Bytes.toString(firstValue);
      String second = "";
      if (secondValue == null) {
        throw new NullPointerException(Bytes.toString(r.getRow()) +
            ": second value is null");
      }
      byte[] secondReversed = new byte[secondValue.length];
      for (int i = 0, j = secondValue.length - 1; j >= 0; j--, i++) {
        secondReversed[i] = secondValue[j];
      }
      second = Bytes.toString(secondReversed);
      if (first.compareTo(second) != 0) {
        if (LOG.isDebugEnabled()) {
          LOG.debug("second key is not the reverse of first. row=" +
              Bytes.toStringBinary(r.getRow()) + ", first value=" + first +
              ", second value=" + second);
        }
        fail();
      }
    }
  } finally {
    scanner.close();
  }
}
 

@Override
public byte[] getValueArray() {
  return CellUtil.cloneValue(this);
}