Java Code Examples for org.apache.hadoop.hbase.client.Put#getFamilyCellMap()

The following examples show how to use org.apache.hadoop.hbase.client.Put#getFamilyCellMap() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: phoenix-omid   File: TTable.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * @param put an instance of Put
 * @param timestamp  timestamp to be used as cells version
 * @param commitTimestamp  timestamp to be used as commit timestamp
 * @throws IOException if a remote or network exception occurs.
 */
static public Put markPutAsCommitted(Put put, long timestamp, long commitTimestamp) {
    final Put tsput = new Put(put.getRow(), timestamp);
    propagateAttributes(put, tsput);

    Map<byte[], List<Cell>> kvs = put.getFamilyCellMap();
    for (List<Cell> kvl : kvs.values()) {
        for (Cell c : kvl) {
            KeyValue kv = KeyValueUtil.ensureKeyValue(c);
            Bytes.putLong(kv.getValueArray(), kv.getTimestampOffset(), timestamp);
            try {
                tsput.add(kv);
            } catch (IOException e) {
                // The existing Put has this Cell, so the cloned one
                // will never throw an IOException when it's added.
                throw new RuntimeException(e);
            }
            tsput.addColumn(CellUtil.cloneFamily(kv),
                    CellUtils.addShadowCellSuffixPrefix(CellUtil.cloneQualifier(kv), 0, CellUtil.cloneQualifier(kv).length),
                    kv.getTimestamp(),
                    Bytes.toBytes(commitTimestamp));
        }
    }

    return tsput;
}
 
Example 2
Source Project: hbase-orm   File: HBObjectMapper.java    License: Apache License 2.0 6 votes vote down vote up
private <R extends Serializable & Comparable<R>, T extends HBRecord<R>> T readValueFromRowAndPut(byte[] rowKeyBytes, Put put, Class<T> clazz) {
    Map<byte[], List<Cell>> rawMap = put.getFamilyCellMap();
    NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    for (Map.Entry<byte[], List<Cell>> familyNameAndColumnValues : rawMap.entrySet()) {
        byte[] family = familyNameAndColumnValues.getKey();
        if (!map.containsKey(family)) {
            map.put(family, new TreeMap<>(Bytes.BYTES_COMPARATOR));
        }
        List<Cell> cellList = familyNameAndColumnValues.getValue();
        for (Cell cell : cellList) {
            byte[] column = CellUtil.cloneQualifier(cell);
            if (!map.get(family).containsKey(column)) {
                map.get(family).put(column, new TreeMap<>());
            }
            map.get(family).get(column).put(cell.getTimestamp(), CellUtil.cloneValue(cell));
        }
    }
    return convertMapToRecord(rowKeyBytes, map, clazz);
}
 
Example 3
Source Project: phoenix   File: MetaDataUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static void mutatePutValue(Put somePut, byte[] family, byte[] qualifier, byte[] newValue) {
    NavigableMap<byte[], List<Cell>> familyCellMap = somePut.getFamilyCellMap();
    List<Cell> cells = familyCellMap.get(family);
    List<Cell> newCells = Lists.newArrayList();
    if (cells != null) {
        for (Cell cell : cells) {
            if (Bytes.compareTo(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(),
                qualifier, 0, qualifier.length) == 0) {
                Cell replacementCell = new KeyValue(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
                    cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(), cell.getQualifierArray(),
                    cell.getQualifierOffset(), cell.getQualifierLength(), cell.getTimestamp(),
                    KeyValue.Type.codeToType(cell.getType().getCode()), newValue, 0, newValue.length);
                newCells.add(replacementCell);
            } else {
                newCells.add(cell);
            }
        }
        familyCellMap.put(family, newCells);
    }
}
 
Example 4
private void verifyPut(String row, String columnFamily, String columnQualifier, String content, Put put) {
    assertEquals(row, new String(put.getRow()));

    NavigableMap<byte [], List<Cell>> familyCells = put.getFamilyCellMap();
    assertEquals(1, familyCells.size());

    Map.Entry<byte[], List<Cell>> entry = familyCells.firstEntry();
    assertEquals(columnFamily, new String(entry.getKey()));
    assertEquals(1, entry.getValue().size());

    Cell cell = entry.getValue().get(0);
    assertEquals(columnQualifier, new String(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()));
    assertEquals(content, new String(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
}
 
Example 5
Source Project: phoenix-omid   File: TTable.java    License: Apache License 2.0 5 votes vote down vote up
private Put putInternal(Transaction tx, Put put, boolean addShadowCell) throws IOException {

        throwExceptionIfOpSetsTimerange(put);

        HBaseTransaction transaction = enforceHBaseTransactionAsParam(tx);

        final long writeTimestamp = transaction.getWriteTimestamp();

        // create put with correct ts
        final Put tsput = new Put(put.getRow(), writeTimestamp);
        propagateAttributes(put, tsput);
        Map<byte[], List<Cell>> kvs = put.getFamilyCellMap();
        for (List<Cell> kvl : kvs.values()) {
            for (Cell c : kvl) {
                CellUtils.validateCell(c, writeTimestamp);
                // Reach into keyvalue to update timestamp.
                // It's not nice to reach into keyvalue internals,
                // but we want to avoid having to copy the whole thing
                KeyValue kv = KeyValueUtil.ensureKeyValue(c);
                Bytes.putLong(kv.getValueArray(), kv.getTimestampOffset(), writeTimestamp);
                tsput.add(kv);

                if (addShadowCell) {
                    tsput.addColumn(CellUtil.cloneFamily(kv),
                            CellUtils.addShadowCellSuffixPrefix(CellUtil.cloneQualifier(kv), 0, CellUtil.cloneQualifier(kv).length),
                            kv.getTimestamp(),
                            Bytes.toBytes(kv.getTimestamp()));
                } else {
                    HBaseCellId cellId = new HBaseCellId(this,
                            CellUtil.cloneRow(kv),
                            CellUtil.cloneFamily(kv),
                            CellUtil.cloneQualifier(kv),
                            kv.getTimestamp());

                    addWriteSetElement(transaction, cellId);
                }
            }
        }
        return tsput;
    }
 
Example 6
Source Project: hbase   File: AccessController.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void prePut(final ObserverContext<RegionCoprocessorEnvironment> c,
    final Put put, final WALEdit edit, final Durability durability)
    throws IOException {
  User user = getActiveUser(c);
  checkForReservedTagPresence(user, put);

  // Require WRITE permission to the table, CF, or top visible value, if any.
  // NOTE: We don't need to check the permissions for any earlier Puts
  // because we treat the ACLs in each Put as timestamped like any other
  // HBase value. A new ACL in a new Put applies to that Put. It doesn't
  // change the ACL of any previous Put. This allows simple evolution of
  // security policy over time without requiring expensive updates.
  RegionCoprocessorEnvironment env = c.getEnvironment();
  Map<byte[],? extends Collection<Cell>> families = put.getFamilyCellMap();
  AuthResult authResult = permissionGranted(OpType.PUT,
      user, env, families, Action.WRITE);
  AccessChecker.logResult(authResult);
  if (!authResult.isAllowed()) {
    if (cellFeaturesEnabled && !compatibleEarlyTermination) {
      put.setAttribute(CHECK_COVERING_PERM, TRUE);
    } else if (authorizationEnabled) {
      throw new AccessDeniedException("Insufficient permissions " + authResult.toContextString());
    }
  }

  // Add cell ACLs from the operation to the cells themselves
  byte[] bytes = put.getAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL);
  if (bytes != null) {
    if (cellFeaturesEnabled) {
      addCellPermissions(bytes, put.getFamilyCellMap());
    } else {
      throw new DoNotRetryIOException("Cell ACLs cannot be persisted");
    }
  }
}
 
Example 7
Source Project: hbase   File: SimpleRegionObserver.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void prePut(final ObserverContext<RegionCoprocessorEnvironment> c,
    final Put put, final WALEdit edit,
    final Durability durability) throws IOException {
  Map<byte[], List<Cell>> familyMap  = put.getFamilyCellMap();
  RegionCoprocessorEnvironment e = c.getEnvironment();
  assertNotNull(e);
  assertNotNull(e.getRegion());
  assertNotNull(familyMap);
  if (e.getRegion().getTableDescriptor().getTableName().equals(
      TestRegionObserverInterface.TEST_TABLE)) {
    List<Cell> cells = familyMap.get(TestRegionObserverInterface.A);
    assertNotNull(cells);
    assertNotNull(cells.get(0));
    Cell cell = cells.get(0);
    assertTrue(Bytes.equals(cell.getQualifierArray(), cell.getQualifierOffset(),
      cell.getQualifierLength(), TestRegionObserverInterface.A, 0,
      TestRegionObserverInterface.A.length));
    cells = familyMap.get(TestRegionObserverInterface.B);
    assertNotNull(cells);
    assertNotNull(cells.get(0));
    cell = cells.get(0);
    assertTrue(Bytes.equals(cell.getQualifierArray(), cell.getQualifierOffset(),
      cell.getQualifierLength(), TestRegionObserverInterface.B, 0,
      TestRegionObserverInterface.B.length));
    cells = familyMap.get(TestRegionObserverInterface.C);
    assertNotNull(cells);
    assertNotNull(cells.get(0));
    cell = cells.get(0);
    assertTrue(Bytes.equals(cell.getQualifierArray(), cell.getQualifierOffset(),
      cell.getQualifierLength(), TestRegionObserverInterface.C, 0,
      TestRegionObserverInterface.C.length));
  }
  ctPrePut.incrementAndGet();
}
 
Example 8
Source Project: hbase   File: SimpleRegionObserver.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void postPut(final ObserverContext<RegionCoprocessorEnvironment> c,
    final Put put, final WALEdit edit,
    final Durability durability) throws IOException {
  Map<byte[], List<Cell>> familyMap  = put.getFamilyCellMap();
  RegionCoprocessorEnvironment e = c.getEnvironment();
  assertNotNull(e);
  assertNotNull(e.getRegion());
  assertNotNull(familyMap);
  List<Cell> cells = familyMap.get(TestRegionObserverInterface.A);
  if (e.getRegion().getTableDescriptor().getTableName().equals(
      TestRegionObserverInterface.TEST_TABLE)) {
    assertNotNull(cells);
    assertNotNull(cells.get(0));
    // KeyValue v1 expectation.  Cast for now until we go all Cell all the time. TODO
    Cell cell = cells.get(0);
    assertTrue(Bytes.equals(cell.getQualifierArray(), cell.getQualifierOffset(),
      cell.getQualifierLength(), TestRegionObserverInterface.A, 0,
      TestRegionObserverInterface.A.length));
    cells = familyMap.get(TestRegionObserverInterface.B);
    assertNotNull(cells);
    assertNotNull(cells.get(0));
    // KeyValue v1 expectation.  Cast for now until we go all Cell all the time. TODO
    cell = cells.get(0);
    assertTrue(Bytes.equals(cell.getQualifierArray(), cell.getQualifierOffset(),
      cell.getQualifierLength(), TestRegionObserverInterface.B, 0,
      TestRegionObserverInterface.B.length));
    cells = familyMap.get(TestRegionObserverInterface.C);
    assertNotNull(cells);
    assertNotNull(cells.get(0));
    // KeyValue v1 expectation.  Cast for now until we go all Cell all the time. TODO
    cell = cells.get(0);
    assertTrue(Bytes.equals(cell.getQualifierArray(), cell.getQualifierOffset(),
      cell.getQualifierLength(), TestRegionObserverInterface.C, 0,
      TestRegionObserverInterface.C.length));
  }
  ctPostPut.incrementAndGet();
}
 
Example 9
Source Project: hbase   File: TestRegionObserverBypass.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void prePut(final ObserverContext<RegionCoprocessorEnvironment> e,
    final Put put, final WALEdit edit, final Durability durability)
    throws IOException {
  PREPUT_INVOCATIONS.incrementAndGet();
  Map<byte[], List<Cell>> familyMap = put.getFamilyCellMap();
  if (familyMap.containsKey(test)) {
    PREPUT_BYPASSES.incrementAndGet();
    e.bypass();
  }
}
 
Example 10
Source Project: phoenix   File: MetaDataUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Iterates over the cells that are mutated by the put operation for the given column family and
 * column qualifier and conditionally modifies those cells to add a tags list. We only add tags
 * if the cell value does not match the passed valueArray. If we always want to add tags to
 * these cells, we can pass in a null valueArray
 * @param somePut Put operation
 * @param family column family of the cells
 * @param qualifier column qualifier of the cells
 * @param cellBuilder ExtendedCellBuilder object
 * @param valueArray byte array of values or null
 * @param tagArray byte array of tags to add to the cells
 */
public static void conditionallyAddTagsToPutCells(Put somePut, byte[] family, byte[] qualifier,
        ExtendedCellBuilder cellBuilder, byte[] valueArray, byte[] tagArray) {
    NavigableMap<byte[], List<Cell>> familyCellMap = somePut.getFamilyCellMap();
    List<Cell> cells = familyCellMap.get(family);
    List<Cell> newCells = Lists.newArrayList();
    if (cells != null) {
        for (Cell cell : cells) {
            if (Bytes.compareTo(cell.getQualifierArray(), cell.getQualifierOffset(),
                    cell.getQualifierLength(), qualifier, 0, qualifier.length) == 0 &&
                    (valueArray == null || !CellUtil.matchingValue(cell, valueArray))) {
                ExtendedCell extendedCell = cellBuilder
                        .setRow(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())
                        .setFamily(cell.getFamilyArray(), cell.getFamilyOffset(),
                                cell.getFamilyLength())
                        .setQualifier(cell.getQualifierArray(), cell.getQualifierOffset(),
                                cell.getQualifierLength())
                        .setValue(cell.getValueArray(), cell.getValueOffset(),
                                cell.getValueLength())
                        .setTimestamp(cell.getTimestamp())
                        .setType(cell.getType())
                        .setTags(TagUtil.concatTags(tagArray, cell))
                        .build();
                // Replace existing cell with a cell that has the custom tags list
                newCells.add(extendedCell);
            } else {
                // Add cell as is
                newCells.add(cell);
            }
        }
        familyCellMap.put(family, newCells);
    }
}
 
Example 11
private void verifyPut(String row, String columnFamily, String columnQualifier, String content, Put put) {
    assertEquals(row, new String(put.getRow()));

    NavigableMap<byte [], List<Cell>> familyCells = put.getFamilyCellMap();
    assertEquals(1, familyCells.size());

    Map.Entry<byte[], List<Cell>> entry = familyCells.firstEntry();
    assertEquals(columnFamily, new String(entry.getKey()));
    assertEquals(1, entry.getValue().size());

    Cell cell = entry.getValue().get(0);
    assertEquals(columnQualifier, new String(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()));
    assertEquals(content, new String(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
}
 
Example 12
private void verifyPut(String row, String columnFamily, String columnQualifier, String content, Put put) {
    assertEquals(row, new String(put.getRow()));

    NavigableMap<byte [], List<Cell>> familyCells = put.getFamilyCellMap();
    assertEquals(1, familyCells.size());

    Map.Entry<byte[], List<Cell>> entry = familyCells.firstEntry();
    assertEquals(columnFamily, new String(entry.getKey()));
    assertEquals(1, entry.getValue().size());

    Cell cell = entry.getValue().get(0);
    assertEquals(columnQualifier, new String(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()));
    assertEquals(content, new String(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
}
 
Example 13
private void verifyPut(String row, String columnFamily, String columnQualifier, String content, Put put) {
    assertEquals(row, new String(put.getRow()));

    NavigableMap<byte [], List<Cell>> familyCells = put.getFamilyCellMap();
    assertEquals(1, familyCells.size());

    Map.Entry<byte[], List<Cell>> entry = familyCells.firstEntry();
    assertEquals(columnFamily, new String(entry.getKey()));
    assertEquals(1, entry.getValue().size());

    Cell cell = entry.getValue().get(0);
    assertEquals(columnQualifier, new String(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()));
    assertEquals(content, new String(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
}
 
Example 14
Source Project: nifi   File: TestHBase_2_ClientService.java    License: Apache License 2.0 5 votes vote down vote up
private void verifyPut(String row, String columnFamily, String columnQualifier, String content, Put put) {
    assertEquals(row, new String(put.getRow()));

    NavigableMap<byte [], List<Cell>> familyCells = put.getFamilyCellMap();
    assertEquals(1, familyCells.size());

    Map.Entry<byte[], List<Cell>> entry = familyCells.firstEntry();
    assertEquals(columnFamily, new String(entry.getKey()));
    assertEquals(1, entry.getValue().size());

    Cell cell = entry.getValue().get(0);
    assertEquals(columnQualifier, new String(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()));
    assertEquals(content, new String(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
}
 
Example 15
@Override
public void postPut(ObserverContext<RegionCoprocessorEnvironment> e, Put put, WALEdit edit, Durability durability)
    throws IOException {
	String tableName = e.getEnvironment().getRegion().getRegionInfo().getTable().getNameAsString();
	if (tableName.startsWith("hbase:")) { //Ԫ���ݱ�,����!
		return;
	}
	String rowKey = Bytes.toString(put.getRow());

	String cFamily = null;
	String cQualifier = null;
	String cValue = null;
	NavigableMap<byte[], List<Cell>> map = put.getFamilyCellMap();
	JsonObject jsonSet = new JsonObject();
	for (List<Cell> cells : map.values()) {
		for (Cell cell : cells) {
			cFamily = new String(CellUtil.cloneFamily(cell));
			cQualifier = new String(CellUtil.cloneQualifier(cell));
			cValue = new String(CellUtil.cloneValue(cell), SolrTools.UTF_8);
			if (cQualifier.endsWith("_s")) { //string
				jsonSet.putObject(cFamily + F_SEPARATOR + cQualifier, (new JsonObject()).putString("set", cValue));
			} else if (cQualifier.endsWith("_t")) { //text_general
				jsonSet.putObject(cFamily + F_SEPARATOR + cQualifier, (new JsonObject()).putString("set", cValue));
			} else if (cQualifier.endsWith("_dt")) { //date
				jsonSet.putObject(cFamily + F_SEPARATOR + cQualifier, (new JsonObject()).putString("set", cValue));
			} else if (cQualifier.endsWith("_i")) { //int
				jsonSet.putObject(cFamily + F_SEPARATOR + cQualifier, (new JsonObject()).putNumber("set", Integer.valueOf(cValue)));
			} else if (cQualifier.endsWith("_l")) { //long
				jsonSet.putObject(cFamily + F_SEPARATOR + cQualifier, (new JsonObject()).putNumber("set", Long.valueOf(cValue)));
			} else if (cQualifier.endsWith("_f")) { //float
				jsonSet.putObject(cFamily + F_SEPARATOR + cQualifier, (new JsonObject()).putNumber("set", Float.valueOf(cValue)));
			} else if (cQualifier.endsWith("_d")) { //double
				jsonSet.putObject(cFamily + F_SEPARATOR + cQualifier, (new JsonObject()).putNumber("set", Double.valueOf(cValue)));
			} else if (cQualifier.endsWith("_b")) { //boolean
				jsonSet.putObject(cFamily + F_SEPARATOR + cQualifier, (new JsonObject()).putBoolean("set", Boolean.valueOf(cValue)));
			} else { //������Ҫ������,����!
				continue;
			}
		}
	}
	if (jsonSet.size() == 0) { //˵��û��solr��ѯ�ֶ�
		return;
	}

	jsonSet.putString(F_ID, tableName + F_SEPARATOR + rowKey);
	jsonSet.putObject(F_TABLENAME, (new JsonObject()).putString("set", tableName));
	jsonSet.putObject(F_ROWKEY, (new JsonObject()).putString("set", rowKey));
	jsonSet.putObject(F_UPDATETIME, (new JsonObject()).putString("set", SolrTools.solrDateFormat.format(new java.util.Date())));

	log.debug("postPut!!! " + jsonSet.encode());
	_bqUpdate.enqueue(jsonSet.encode().getBytes(SolrTools.UTF_8));
}
 
Example 16
Source Project: hbase   File: PutCombiner.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected void reduce(K row, Iterable<Put> vals, Context context)
    throws IOException, InterruptedException {
  // Using HeapSize to create an upper bound on the memory size of
  // the puts and flush some portion of the content while looping. This
  // flush could result in multiple Puts for a single rowkey. That is
  // acceptable because Combiner is run as an optimization and it's not
  // critical that all Puts are grouped perfectly.
  long threshold = context.getConfiguration().getLong(
      "putcombiner.row.threshold", 1L * (1<<30));
  int cnt = 0;
  long curSize = 0;
  Put put = null;
  Map<byte[], List<Cell>> familyMap = null;
  for (Put p : vals) {
    cnt++;
    if (put == null) {
      put = p;
      familyMap = put.getFamilyCellMap();
    } else {
      for (Entry<byte[], List<Cell>> entry : p.getFamilyCellMap()
          .entrySet()) {
        List<Cell> cells = familyMap.get(entry.getKey());
        List<Cell> kvs = (cells != null) ? (List<Cell>) cells : null;
        for (Cell cell : entry.getValue()) {
          KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
          curSize += kv.heapSize();
          if (kvs != null) {
            kvs.add(kv);
          }
        }
        if (cells == null) {
          familyMap.put(entry.getKey(), entry.getValue());
        }
      }
      if (cnt % 10 == 0) context.setStatus("Combine " + cnt);
      if (curSize > threshold) {
        if (LOG.isDebugEnabled()) {
          LOG.debug(String.format("Combined %d Put(s) into %d.", cnt, 1));
        }
        context.write(row, put);
        put = null;
        curSize = 0;
        cnt = 0;
      }
    }
  }
  if (put != null) {
    if (LOG.isDebugEnabled()) {
      LOG.debug(String.format("Combined %d Put(s) into %d.", cnt, 1));
    }
    context.write(row, put);
  }
}