Java Code Examples for org.apache.hadoop.hbase.client.Put#getFamilyCellMap()

/**
 * @param put an instance of Put
 * @param timestamp  timestamp to be used as cells version
 * @param commitTimestamp  timestamp to be used as commit timestamp
 * @throws IOException if a remote or network exception occurs.
 */
static public Put markPutAsCommitted(Put put, long timestamp, long commitTimestamp) {
    final Put tsput = new Put(put.getRow(), timestamp);
    propagateAttributes(put, tsput);

    Map<byte[], List<Cell>> kvs = put.getFamilyCellMap();
    for (List<Cell> kvl : kvs.values()) {
        for (Cell c : kvl) {
            KeyValue kv = KeyValueUtil.ensureKeyValue(c);
            Bytes.putLong(kv.getValueArray(), kv.getTimestampOffset(), timestamp);
            try {
                tsput.add(kv);
            } catch (IOException e) {
                // The existing Put has this Cell, so the cloned one
                // will never throw an IOException when it's added.
                throw new RuntimeException(e);
            }
            tsput.addColumn(CellUtil.cloneFamily(kv),
                    CellUtils.addShadowCellSuffixPrefix(CellUtil.cloneQualifier(kv), 0, CellUtil.cloneQualifier(kv).length),
                    kv.getTimestamp(),
                    Bytes.toBytes(commitTimestamp));
        }
    }

    return tsput;
}
 

private <R extends Serializable & Comparable<R>, T extends HBRecord<R>> T readValueFromRowAndPut(byte[] rowKeyBytes, Put put, Class<T> clazz) {
    Map<byte[], List<Cell>> rawMap = put.getFamilyCellMap();
    NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    for (Map.Entry<byte[], List<Cell>> familyNameAndColumnValues : rawMap.entrySet()) {
        byte[] family = familyNameAndColumnValues.getKey();
        if (!map.containsKey(family)) {
            map.put(family, new TreeMap<>(Bytes.BYTES_COMPARATOR));
        }
        List<Cell> cellList = familyNameAndColumnValues.getValue();
        for (Cell cell : cellList) {
            byte[] column = CellUtil.cloneQualifier(cell);
            if (!map.get(family).containsKey(column)) {
                map.get(family).put(column, new TreeMap<>());
            }
            map.get(family).get(column).put(cell.getTimestamp(), CellUtil.cloneValue(cell));
        }
    }
    return convertMapToRecord(rowKeyBytes, map, clazz);
}
 

public static void mutatePutValue(Put somePut, byte[] family, byte[] qualifier, byte[] newValue) {
    NavigableMap<byte[], List<Cell>> familyCellMap = somePut.getFamilyCellMap();
    List<Cell> cells = familyCellMap.get(family);
    List<Cell> newCells = Lists.newArrayList();
    if (cells != null) {
        for (Cell cell : cells) {
            if (Bytes.compareTo(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(),
                qualifier, 0, qualifier.length) == 0) {
                Cell replacementCell = new KeyValue(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
                    cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(), cell.getQualifierArray(),
                    cell.getQualifierOffset(), cell.getQualifierLength(), cell.getTimestamp(),
                    KeyValue.Type.codeToType(cell.getType().getCode()), newValue, 0, newValue.length);
                newCells.add(replacementCell);
            } else {
                newCells.add(cell);
            }
        }
        familyCellMap.put(family, newCells);
    }
}
 

private void verifyPut(String row, String columnFamily, String columnQualifier, String content, Put put) {
    assertEquals(row, new String(put.getRow()));

    NavigableMap<byte [], List<Cell>> familyCells = put.getFamilyCellMap();
    assertEquals(1, familyCells.size());

    Map.Entry<byte[], List<Cell>> entry = familyCells.firstEntry();
    assertEquals(columnFamily, new String(entry.getKey()));
    assertEquals(1, entry.getValue().size());

    Cell cell = entry.getValue().get(0);
    assertEquals(columnQualifier, new String(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()));
    assertEquals(content, new String(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
}
 

private Put putInternal(Transaction tx, Put put, boolean addShadowCell) throws IOException {

        throwExceptionIfOpSetsTimerange(put);

        HBaseTransaction transaction = enforceHBaseTransactionAsParam(tx);

        final long writeTimestamp = transaction.getWriteTimestamp();

        // create put with correct ts
        final Put tsput = new Put(put.getRow(), writeTimestamp);
        propagateAttributes(put, tsput);
        Map<byte[], List<Cell>> kvs = put.getFamilyCellMap();
        for (List<Cell> kvl : kvs.values()) {
            for (Cell c : kvl) {
                CellUtils.validateCell(c, writeTimestamp);
                // Reach into keyvalue to update timestamp.
                // It's not nice to reach into keyvalue internals,
                // but we want to avoid having to copy the whole thing
                KeyValue kv = KeyValueUtil.ensureKeyValue(c);
                Bytes.putLong(kv.getValueArray(), kv.getTimestampOffset(), writeTimestamp);
                tsput.add(kv);

                if (addShadowCell) {
                    tsput.addColumn(CellUtil.cloneFamily(kv),
                            CellUtils.addShadowCellSuffixPrefix(CellUtil.cloneQualifier(kv), 0, CellUtil.cloneQualifier(kv).length),
                            kv.getTimestamp(),
                            Bytes.toBytes(kv.getTimestamp()));
                } else {
                    HBaseCellId cellId = new HBaseCellId(this,
                            CellUtil.cloneRow(kv),
                            CellUtil.cloneFamily(kv),
                            CellUtil.cloneQualifier(kv),
                            kv.getTimestamp());

                    addWriteSetElement(transaction, cellId);
                }
            }
        }
        return tsput;
    }
 

@Override
public void prePut(final ObserverContext<RegionCoprocessorEnvironment> c,
    final Put put, final WALEdit edit, final Durability durability)
    throws IOException {
  User user = getActiveUser(c);
  checkForReservedTagPresence(user, put);

  // Require WRITE permission to the table, CF, or top visible value, if any.
  // NOTE: We don't need to check the permissions for any earlier Puts
  // because we treat the ACLs in each Put as timestamped like any other
  // HBase value. A new ACL in a new Put applies to that Put. It doesn't
  // change the ACL of any previous Put. This allows simple evolution of
  // security policy over time without requiring expensive updates.
  RegionCoprocessorEnvironment env = c.getEnvironment();
  Map<byte[],? extends Collection<Cell>> families = put.getFamilyCellMap();
  AuthResult authResult = permissionGranted(OpType.PUT,
      user, env, families, Action.WRITE);
  AccessChecker.logResult(authResult);
  if (!authResult.isAllowed()) {
    if (cellFeaturesEnabled && !compatibleEarlyTermination) {
      put.setAttribute(CHECK_COVERING_PERM, TRUE);
    } else if (authorizationEnabled) {
      throw new AccessDeniedException("Insufficient permissions " + authResult.toContextString());
    }
  }

  // Add cell ACLs from the operation to the cells themselves
  byte[] bytes = put.getAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL);
  if (bytes != null) {
    if (cellFeaturesEnabled) {
      addCellPermissions(bytes, put.getFamilyCellMap());
    } else {
      throw new DoNotRetryIOException("Cell ACLs cannot be persisted");
    }
  }
}
 

@Override
public void prePut(final ObserverContext<RegionCoprocessorEnvironment> c,
    final Put put, final WALEdit edit,
    final Durability durability) throws IOException {
  Map<byte[], List<Cell>> familyMap  = put.getFamilyCellMap();
  RegionCoprocessorEnvironment e = c.getEnvironment();
  assertNotNull(e);
  assertNotNull(e.getRegion());
  assertNotNull(familyMap);
  if (e.getRegion().getTableDescriptor().getTableName().equals(
      TestRegionObserverInterface.TEST_TABLE)) {
    List<Cell> cells = familyMap.get(TestRegionObserverInterface.A);
    assertNotNull(cells);
    assertNotNull(cells.get(0));
    Cell cell = cells.get(0);
    assertTrue(Bytes.equals(cell.getQualifierArray(), cell.getQualifierOffset(),
      cell.getQualifierLength(), TestRegionObserverInterface.A, 0,
      TestRegionObserverInterface.A.length));
    cells = familyMap.get(TestRegionObserverInterface.B);
    assertNotNull(cells);
    assertNotNull(cells.get(0));
    cell = cells.get(0);
    assertTrue(Bytes.equals(cell.getQualifierArray(), cell.getQualifierOffset(),
      cell.getQualifierLength(), TestRegionObserverInterface.B, 0,
      TestRegionObserverInterface.B.length));
    cells = familyMap.get(TestRegionObserverInterface.C);
    assertNotNull(cells);
    assertNotNull(cells.get(0));
    cell = cells.get(0);
    assertTrue(Bytes.equals(cell.getQualifierArray(), cell.getQualifierOffset(),
      cell.getQualifierLength(), TestRegionObserverInterface.C, 0,
      TestRegionObserverInterface.C.length));
  }
  ctPrePut.incrementAndGet();
}
 

@Override
public void postPut(final ObserverContext<RegionCoprocessorEnvironment> c,
    final Put put, final WALEdit edit,
    final Durability durability) throws IOException {
  Map<byte[], List<Cell>> familyMap  = put.getFamilyCellMap();
  RegionCoprocessorEnvironment e = c.getEnvironment();
  assertNotNull(e);
  assertNotNull(e.getRegion());
  assertNotNull(familyMap);
  List<Cell> cells = familyMap.get(TestRegionObserverInterface.A);
  if (e.getRegion().getTableDescriptor().getTableName().equals(
      TestRegionObserverInterface.TEST_TABLE)) {
    assertNotNull(cells);
    assertNotNull(cells.get(0));
    // KeyValue v1 expectation.  Cast for now until we go all Cell all the time. TODO
    Cell cell = cells.get(0);
    assertTrue(Bytes.equals(cell.getQualifierArray(), cell.getQualifierOffset(),
      cell.getQualifierLength(), TestRegionObserverInterface.A, 0,
      TestRegionObserverInterface.A.length));
    cells = familyMap.get(TestRegionObserverInterface.B);
    assertNotNull(cells);
    assertNotNull(cells.get(0));
    // KeyValue v1 expectation.  Cast for now until we go all Cell all the time. TODO
    cell = cells.get(0);
    assertTrue(Bytes.equals(cell.getQualifierArray(), cell.getQualifierOffset(),
      cell.getQualifierLength(), TestRegionObserverInterface.B, 0,
      TestRegionObserverInterface.B.length));
    cells = familyMap.get(TestRegionObserverInterface.C);
    assertNotNull(cells);
    assertNotNull(cells.get(0));
    // KeyValue v1 expectation.  Cast for now until we go all Cell all the time. TODO
    cell = cells.get(0);
    assertTrue(Bytes.equals(cell.getQualifierArray(), cell.getQualifierOffset(),
      cell.getQualifierLength(), TestRegionObserverInterface.C, 0,
      TestRegionObserverInterface.C.length));
  }
  ctPostPut.incrementAndGet();
}
 

@Override
public void prePut(final ObserverContext<RegionCoprocessorEnvironment> e,
    final Put put, final WALEdit edit, final Durability durability)
    throws IOException {
  PREPUT_INVOCATIONS.incrementAndGet();
  Map<byte[], List<Cell>> familyMap = put.getFamilyCellMap();
  if (familyMap.containsKey(test)) {
    PREPUT_BYPASSES.incrementAndGet();
    e.bypass();
  }
}
 

/**
 * Iterates over the cells that are mutated by the put operation for the given column family and
 * column qualifier and conditionally modifies those cells to add a tags list. We only add tags
 * if the cell value does not match the passed valueArray. If we always want to add tags to
 * these cells, we can pass in a null valueArray
 * @param somePut Put operation
 * @param family column family of the cells
 * @param qualifier column qualifier of the cells
 * @param cellBuilder ExtendedCellBuilder object
 * @param valueArray byte array of values or null
 * @param tagArray byte array of tags to add to the cells
 */
public static void conditionallyAddTagsToPutCells(Put somePut, byte[] family, byte[] qualifier,
        ExtendedCellBuilder cellBuilder, byte[] valueArray, byte[] tagArray) {
    NavigableMap<byte[], List<Cell>> familyCellMap = somePut.getFamilyCellMap();
    List<Cell> cells = familyCellMap.get(family);
    List<Cell> newCells = Lists.newArrayList();
    if (cells != null) {
        for (Cell cell : cells) {
            if (Bytes.compareTo(cell.getQualifierArray(), cell.getQualifierOffset(),
                    cell.getQualifierLength(), qualifier, 0, qualifier.length) == 0 &&
                    (valueArray == null || !CellUtil.matchingValue(cell, valueArray))) {
                ExtendedCell extendedCell = cellBuilder
                        .setRow(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())
                        .setFamily(cell.getFamilyArray(), cell.getFamilyOffset(),
                                cell.getFamilyLength())
                        .setQualifier(cell.getQualifierArray(), cell.getQualifierOffset(),
                                cell.getQualifierLength())
                        .setValue(cell.getValueArray(), cell.getValueOffset(),
                                cell.getValueLength())
                        .setTimestamp(cell.getTimestamp())
                        .setType(cell.getType())
                        .setTags(TagUtil.concatTags(tagArray, cell))
                        .build();
                // Replace existing cell with a cell that has the custom tags list
                newCells.add(extendedCell);
            } else {
                // Add cell as is
                newCells.add(cell);
            }
        }
        familyCellMap.put(family, newCells);
    }
}
 

private void verifyPut(String row, String columnFamily, String columnQualifier, String content, Put put) {
    assertEquals(row, new String(put.getRow()));

    NavigableMap<byte [], List<Cell>> familyCells = put.getFamilyCellMap();
    assertEquals(1, familyCells.size());

    Map.Entry<byte[], List<Cell>> entry = familyCells.firstEntry();
    assertEquals(columnFamily, new String(entry.getKey()));
    assertEquals(1, entry.getValue().size());

    Cell cell = entry.getValue().get(0);
    assertEquals(columnQualifier, new String(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()));
    assertEquals(content, new String(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
}
 

private void verifyPut(String row, String columnFamily, String columnQualifier, String content, Put put) {
    assertEquals(row, new String(put.getRow()));

    NavigableMap<byte [], List<Cell>> familyCells = put.getFamilyCellMap();
    assertEquals(1, familyCells.size());

    Map.Entry<byte[], List<Cell>> entry = familyCells.firstEntry();
    assertEquals(columnFamily, new String(entry.getKey()));
    assertEquals(1, entry.getValue().size());

    Cell cell = entry.getValue().get(0);
    assertEquals(columnQualifier, new String(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()));
    assertEquals(content, new String(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
}
 

private void verifyPut(String row, String columnFamily, String columnQualifier, String content, Put put) {
    assertEquals(row, new String(put.getRow()));

    NavigableMap<byte [], List<Cell>> familyCells = put.getFamilyCellMap();
    assertEquals(1, familyCells.size());

    Map.Entry<byte[], List<Cell>> entry = familyCells.firstEntry();
    assertEquals(columnFamily, new String(entry.getKey()));
    assertEquals(1, entry.getValue().size());

    Cell cell = entry.getValue().get(0);
    assertEquals(columnQualifier, new String(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()));
    assertEquals(content, new String(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
}
 

private void verifyPut(String row, String columnFamily, String columnQualifier, String content, Put put) {
    assertEquals(row, new String(put.getRow()));

    NavigableMap<byte [], List<Cell>> familyCells = put.getFamilyCellMap();
    assertEquals(1, familyCells.size());

    Map.Entry<byte[], List<Cell>> entry = familyCells.firstEntry();
    assertEquals(columnFamily, new String(entry.getKey()));
    assertEquals(1, entry.getValue().size());

    Cell cell = entry.getValue().get(0);
    assertEquals(columnQualifier, new String(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()));
    assertEquals(content, new String(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
}
 

@Override
public void postPut(ObserverContext<RegionCoprocessorEnvironment> e, Put put, WALEdit edit, Durability durability)
    throws IOException {
	String tableName = e.getEnvironment().getRegion().getRegionInfo().getTable().getNameAsString();
	if (tableName.startsWith("hbase:")) { //Ԫ���ݱ�,����!
		return;
	}
	String rowKey = Bytes.toString(put.getRow());

	String cFamily = null;
	String cQualifier = null;
	String cValue = null;
	NavigableMap<byte[], List<Cell>> map = put.getFamilyCellMap();
	JsonObject jsonSet = new JsonObject();
	for (List<Cell> cells : map.values()) {
		for (Cell cell : cells) {
			cFamily = new String(CellUtil.cloneFamily(cell));
			cQualifier = new String(CellUtil.cloneQualifier(cell));
			cValue = new String(CellUtil.cloneValue(cell), SolrTools.UTF_8);
			if (cQualifier.endsWith("_s")) { //string
				jsonSet.putObject(cFamily + F_SEPARATOR + cQualifier, (new JsonObject()).putString("set", cValue));
			} else if (cQualifier.endsWith("_t")) { //text_general
				jsonSet.putObject(cFamily + F_SEPARATOR + cQualifier, (new JsonObject()).putString("set", cValue));
			} else if (cQualifier.endsWith("_dt")) { //date
				jsonSet.putObject(cFamily + F_SEPARATOR + cQualifier, (new JsonObject()).putString("set", cValue));
			} else if (cQualifier.endsWith("_i")) { //int
				jsonSet.putObject(cFamily + F_SEPARATOR + cQualifier, (new JsonObject()).putNumber("set", Integer.valueOf(cValue)));
			} else if (cQualifier.endsWith("_l")) { //long
				jsonSet.putObject(cFamily + F_SEPARATOR + cQualifier, (new JsonObject()).putNumber("set", Long.valueOf(cValue)));
			} else if (cQualifier.endsWith("_f")) { //float
				jsonSet.putObject(cFamily + F_SEPARATOR + cQualifier, (new JsonObject()).putNumber("set", Float.valueOf(cValue)));
			} else if (cQualifier.endsWith("_d")) { //double
				jsonSet.putObject(cFamily + F_SEPARATOR + cQualifier, (new JsonObject()).putNumber("set", Double.valueOf(cValue)));
			} else if (cQualifier.endsWith("_b")) { //boolean
				jsonSet.putObject(cFamily + F_SEPARATOR + cQualifier, (new JsonObject()).putBoolean("set", Boolean.valueOf(cValue)));
			} else { //������Ҫ������,����!
				continue;
			}
		}
	}
	if (jsonSet.size() == 0) { //˵��û��solr��ѯ�ֶ�
		return;
	}

	jsonSet.putString(F_ID, tableName + F_SEPARATOR + rowKey);
	jsonSet.putObject(F_TABLENAME, (new JsonObject()).putString("set", tableName));
	jsonSet.putObject(F_ROWKEY, (new JsonObject()).putString("set", rowKey));
	jsonSet.putObject(F_UPDATETIME, (new JsonObject()).putString("set", SolrTools.solrDateFormat.format(new java.util.Date())));

	log.debug("postPut!!! " + jsonSet.encode());
	_bqUpdate.enqueue(jsonSet.encode().getBytes(SolrTools.UTF_8));
}
 

@Override
protected void reduce(K row, Iterable<Put> vals, Context context)
    throws IOException, InterruptedException {
  // Using HeapSize to create an upper bound on the memory size of
  // the puts and flush some portion of the content while looping. This
  // flush could result in multiple Puts for a single rowkey. That is
  // acceptable because Combiner is run as an optimization and it's not
  // critical that all Puts are grouped perfectly.
  long threshold = context.getConfiguration().getLong(
      "putcombiner.row.threshold", 1L * (1<<30));
  int cnt = 0;
  long curSize = 0;
  Put put = null;
  Map<byte[], List<Cell>> familyMap = null;
  for (Put p : vals) {
    cnt++;
    if (put == null) {
      put = p;
      familyMap = put.getFamilyCellMap();
    } else {
      for (Entry<byte[], List<Cell>> entry : p.getFamilyCellMap()
          .entrySet()) {
        List<Cell> cells = familyMap.get(entry.getKey());
        List<Cell> kvs = (cells != null) ? (List<Cell>) cells : null;
        for (Cell cell : entry.getValue()) {
          KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
          curSize += kv.heapSize();
          if (kvs != null) {
            kvs.add(kv);
          }
        }
        if (cells == null) {
          familyMap.put(entry.getKey(), entry.getValue());
        }
      }
      if (cnt % 10 == 0) context.setStatus("Combine " + cnt);
      if (curSize > threshold) {
        if (LOG.isDebugEnabled()) {
          LOG.debug(String.format("Combined %d Put(s) into %d.", cnt, 1));
        }
        context.write(row, put);
        put = null;
        curSize = 0;
        cnt = 0;
      }
    }
  }
  if (put != null) {
    if (LOG.isDebugEnabled()) {
      LOG.debug(String.format("Combined %d Put(s) into %d.", cnt, 1));
    }
    context.write(row, put);
  }
}