org.hbase.async.KeyValue Java Examples

The following examples show how to use org.hbase.async.KeyValue. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HBaseRecordStore.java    From qmq with Apache License 2.0 6 votes vote down vote up
private List<RecordQueryResult.Record> findDeadRecord(final String subject, final String messageId) {
    final String subjectId = dicService.name2Id(subject);
    final String keyRegexp = BackupMessageKeyRegexpBuilder.buildDeadRecordRegexp(subjectId, messageId);
    final String startKey = BackupMessageKeyRangeBuilder.buildDeadRecordStartKey(subjectId, messageId);
    final String endKey = BackupMessageKeyRangeBuilder.buildDeadRecordEndKey(subjectId, messageId);

    try {
        return scan(table, keyRegexp, startKey, endKey, 100, 0, R_FAMILY, B_RECORD_QUALIFIERS, kvs -> {
            final KeyValueList<KeyValue> kvl = new KeyValueListImpl(kvs);
            final byte[] value = kvl.getValue(RECORDS);
            final long sequence = Bytes.getLong(value, 0);
            final long timestamp = Bytes.getLong(value, 8);
            final int consumerGroupLength = value.length - 16;
            final byte[] consumerGroupBytes = new byte[consumerGroupLength];
            System.arraycopy(value, 16, consumerGroupBytes, 0, consumerGroupLength);
            final String consumerGroup = CharsetUtils.toUTF8String(consumerGroupBytes);
            return new RecordQueryResult.Record(consumerGroup, ActionEnum.OMIT.getCode(), RecordEnum.DEAD_RECORD.getCode(), timestamp, "", sequence);
        });
    } catch (Exception e) {
        LOG.error("Failed to find dead records.", e);
        return Collections.emptyList();
    }
}
 
Example #2
Source File: HBaseValueDecoder.java    From qmq with Apache License 2.0 6 votes vote down vote up
public static RecordQueryResult.Record getRecord(List<KeyValue> kvs, byte type) {
    KeyValueList<KeyValue> kvl = new KeyValueListImpl(kvs);
    byte[] rowKey = kvl.getKey();
    String row = CharsetUtils.toUTF8String(rowKey);
    long sequence = Long.parseLong(row.substring(MESSAGE_SUBJECT_LENGTH, MESSAGE_SUBJECT_LENGTH + RECORD_SEQUENCE_LENGTH));
    byte action = Byte.parseByte(row.substring(row.length() - 1));

    byte[] value = kvl.getValue(RECORDS);
    long timestamp = Bytes.getLong(value, 0);
    short consumerIdLength = Bytes.getShort(value, 8);
    byte[] consumerIdBytes = new byte[consumerIdLength];
    System.arraycopy(value, 10, consumerIdBytes, 0, consumerIdLength);
    String consumerId = CharsetUtils.toUTF8String(consumerIdBytes);
    short consumerGroupLength = Bytes.getShort(value, 10 + consumerIdLength);
    byte[] consumerGroupBytes = new byte[consumerGroupLength];
    System.arraycopy(value, 12 + consumerIdLength, consumerGroupBytes, 0, consumerGroupLength);
    String consumerGroup = CharsetUtils.toUTF8String(consumerGroupBytes);
    return new RecordQueryResult.Record(consumerGroup, action, type, timestamp, consumerId, sequence);
}
 
Example #3
Source File: KeyValueListImpl.java    From qmq with Apache License 2.0 5 votes vote down vote up
public KeyValueListImpl(List<KeyValue> keyValues) {
    if (keyValues != null && !keyValues.isEmpty()) {
        this.keyValues = keyValues;
        key = keyValues.get(0).key();
        for (KeyValue keyValue : keyValues) {
            data.put(new String(keyValue.qualifier(), CharsetUtil.UTF_8), keyValue);
        }
    }
}
 
Example #4
Source File: HBaseParserTest.java    From Elasticsearch-HBase-River with Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testBase() {
	final HBaseParser parser = new HBaseParser(new HBaseRiver(null, null, null));

	final ArrayList<KeyValue> input = new ArrayList<KeyValue>();

	input.add(getKeyValue("family1", "category1", "value1"));
	input.add(getKeyValue("family1", "category2", "value2"));
	input.add(getKeyValue("family1", "category3", "value3"));
	input.add(getKeyValue("family2", "category1", "value4"));
	input.add(getKeyValue("family2", "category4", "value5"));
	input.add(getKeyValue("family3", "category5", "value6"));
	input.add(getKeyValue("family2", "category6", "value7"));

	final Map<String, Object> output = parser.readDataTree(input);

	assertNotNull(output.get("family1"));
	final Map<String, Object> family1 = (Map<String, Object>) output.get("family1");
	assertEquals(family1.get("category1"), "value1");
	assertEquals(family1.get("category2"), "value2");
	assertEquals(family1.get("category3"), "value3");
	assertNotNull(output.get("family2"));
	final Map<String, Object> family2 = (Map<String, Object>) output.get("family2");
	assertEquals(family2.get("category1"), "value4");
	assertEquals(family2.get("category4"), "value5");
	assertEquals(family2.get("category6"), "value7");
	assertNotNull(output.get("family3"));
	final Map<String, Object> family3 = (Map<String, Object>) output.get("family3");
	assertEquals(family3.get("category5"), "value6");
}
 
Example #5
Source File: HBaseParser.java    From Elasticsearch-HBase-River with Apache License 2.0 5 votes vote down vote up
/**
 * Generate a tree structure that ElasticSearch can read and index from one of the rows that has been returned from
 * HBase.
 * 
 * @param row
 * @return
 */
@SuppressWarnings("unchecked")
protected Map<String, Object> readDataTree(final ArrayList<KeyValue> row) {
	final Map<String, Object> dataTree = new HashMap<String, Object>();
	for (final KeyValue column : row) {
		final String family = this.river.normalizeField(new String(column.family(), this.river.getCharset()));
		final String qualifier = new String(column.qualifier(), this.river.getCharset());
		final String value = new String(column.value(), this.river.getCharset());
		if (!dataTree.containsKey(family)) {
			dataTree.put(family, new HashMap<String, Object>());
		}
		readQualifierStructure((Map<String, Object>) dataTree.get(family), qualifier, value);
	}
	return dataTree;
}
 
Example #6
Source File: GenServerGetCallback.java    From diver with Apache License 2.0 5 votes vote down vote up
@Override
public Object call(final ArrayList<KeyValue> data) throws Exception {
  final OtpErlangObject[] items = new OtpErlangObject[data.size()];
  int i = 0;
  for (final KeyValue keyValue : data) {
    final OtpErlangObject[] erldata = new OtpErlangObject[] {
        new OtpErlangBinary(keyValue.key()),
        new OtpErlangBinary(keyValue.family()),
        new OtpErlangBinary(keyValue.qualifier()),
        new OtpErlangBinary(keyValue.value()),
        new OtpErlangLong(keyValue.timestamp())
    };
    items[i] = new OtpErlangTuple(erldata);
    i++;
  }

  final OtpErlangObject[] body = new OtpErlangObject[] {
      OK_ATOM,
      new OtpErlangList(items)
  };

  final OtpErlangObject[] resp = new OtpErlangObject[] {
      from.elementAt(1),  // Ref
      new OtpErlangTuple(body)
  };

  mbox.send((OtpErlangPid) from.elementAt(0), new OtpErlangTuple(resp));
  return null;
}
 
Example #7
Source File: HBaseRecordStore.java    From qmq with Apache License 2.0 5 votes vote down vote up
private List<BackupMessageMeta> scanMessageMeta(String subject, String messageId) {
    final LocalDateTime now = LocalDateTime.now();
    final Date createTimeEnd = localDateTime2Date(now);
    final Date createTimeBegin = localDateTime2Date(now.minusDays(30));

    try {
        final String subjectId = dicService.name2Id(subject);
        final String keyRegexp = BackupMessageKeyRegexpBuilder.buildRetryRegexp(subjectId, messageId);
        final String startKey = BackupMessageKeyRangeBuilder.buildRetryRangeKey(subjectId, messageId, createTimeEnd);
        final String endKey = BackupMessageKeyRangeBuilder.buildRetryRangeKey(subjectId, messageId, createTimeBegin);
        final List<BackupMessageMeta> metas = scan(indexTable, keyRegexp, startKey, endKey, 1000, 0, B_FAMILY, B_MESSAGE_QUALIFIERS, kvs -> {
            KeyValueList<KeyValue> kvl = new KeyValueListImpl(kvs);
            byte[] value = kvl.getValue(CONTENT);
            byte[] rowKey = kvl.getKey();
            BackupMessageMeta meta = getMessageMeta(value);
            if (meta != null && rowKey.length > CONSUMER_GROUP_INDEX_IN_RETRY_MESSAGE) {
                byte[] consumerGroupId = new byte[CONSUMER_GROUP_LENGTH];
                System.arraycopy(rowKey, CONSUMER_GROUP_INDEX_IN_RETRY_MESSAGE, consumerGroupId, 0, CONSUMER_GROUP_LENGTH);
                meta.setConsumerGroupId(new String(consumerGroupId, CharsetUtil.UTF_8));
            }
            return meta;
        });
        return Lists.newArrayList(Sets.newHashSet(metas));
    } catch (Exception e) {
        LOG.error("Failed to scan messages meta.", e);
        return Lists.newArrayList();
    }
}
 
Example #8
Source File: AbstractHBaseMessageStore.java    From qmq with Apache License 2.0 5 votes vote down vote up
void getMessageFromHBase(final String subject, final byte[] table, final MessageQueryResult messageQueryResult, final String keyRegexp, final String startKey, final String endKey
        , final int maxResults) {
    List<BackupMessageMeta> metas;
    try {
        metas = scan(table, keyRegexp, startKey, endKey, maxResults + 1, 0, B_FAMILY, B_MESSAGE_QUALIFIERS, kvs -> {
            KeyValueList<KeyValue> kvl = new KeyValueListImpl(kvs);
            messageQueryResult.setNext(new String(kvl.getKey(), CharsetUtil.UTF_8));
            byte[] value = kvl.getValue(CONTENT);
            BackupMessageMeta meta = getMessageMeta(value);
            if (meta == null) {
                Metrics.counter("message.content.missing").inc();
                LOG.info("Message content missing");
            }
            return meta;
        });
    } catch (Exception e) {
        LOG.error("Failed to get messages from hbase.", e);
        messageQueryResult.setList(Collections.emptyList());
        return;
    }
    int size = metas.size();
    LOG.info("Found {} metas from HBase.", size);
    slim(metas, messageQueryResult, maxResults);

    List<MessageQueryResult.MessageMeta> messages = getMessagesWithMeta(subject, metas);
    messageQueryResult.setList(messages);
}
 
Example #9
Source File: KeyValueListImpl.java    From qmq with Apache License 2.0 4 votes vote down vote up
@Override
public Long getTimestamp(String qualifier) {
    KeyValue keyValue = getKeyValue(qualifier);
    return keyValue == null ? null : keyValue.timestamp();
}
 
Example #10
Source File: KeyValueListImpl.java    From qmq with Apache License 2.0 4 votes vote down vote up
@Override
public byte[] getValue(String qualifier) {
    KeyValue keyValue = getKeyValue(qualifier);
    return (keyValue == null) ? null : keyValue.value();
}
 
Example #11
Source File: KeyValueListImpl.java    From qmq with Apache License 2.0 4 votes vote down vote up
public KeyValue getKeyValue(String qualifier) {
    return data.get(qualifier);
}
 
Example #12
Source File: KeyValueListImpl.java    From qmq with Apache License 2.0 4 votes vote down vote up
@Override
public Map<String, KeyValue> getData() {
    return data;
}
 
Example #13
Source File: KeyValueListImpl.java    From qmq with Apache License 2.0 4 votes vote down vote up
public void setKeyValues(ArrayList<KeyValue> keyValues) {
    this.keyValues = keyValues;
}
 
Example #14
Source File: KeyValueListImpl.java    From qmq with Apache License 2.0 4 votes vote down vote up
@Override
public List<KeyValue> getKeyValues() {
    return keyValues;
}
 
Example #15
Source File: HBaseParserTest.java    From Elasticsearch-HBase-River with Apache License 2.0 4 votes vote down vote up
private KeyValue getKeyValue(final String family, final String qualifier, final String value) {
	return new KeyValue(String.valueOf(this.rowCounter++).getBytes(this.charset),
		family.getBytes(this.charset),
		qualifier.getBytes(this.charset),
		value.getBytes(this.charset));
}
 
Example #16
Source File: AbstractHBaseStore.java    From qmq with Apache License 2.0 votes vote down vote up
T extractData(List<KeyValue> kvs); 
Example #17
Source File: KeyValueList.java    From qmq with Apache License 2.0 votes vote down vote up
List<KeyValue> getKeyValues();