Java Code Examples for org.hbase.async.KeyValue

The following examples show how to use org.hbase.async.KeyValue. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: qmq   Source File: HBaseValueDecoder.java    License: Apache License 2.0 6 votes vote down vote up
public static RecordQueryResult.Record getRecord(List<KeyValue> kvs, byte type) {
    KeyValueList<KeyValue> kvl = new KeyValueListImpl(kvs);
    byte[] rowKey = kvl.getKey();
    String row = CharsetUtils.toUTF8String(rowKey);
    long sequence = Long.parseLong(row.substring(MESSAGE_SUBJECT_LENGTH, MESSAGE_SUBJECT_LENGTH + RECORD_SEQUENCE_LENGTH));
    byte action = Byte.parseByte(row.substring(row.length() - 1));

    byte[] value = kvl.getValue(RECORDS);
    long timestamp = Bytes.getLong(value, 0);
    short consumerIdLength = Bytes.getShort(value, 8);
    byte[] consumerIdBytes = new byte[consumerIdLength];
    System.arraycopy(value, 10, consumerIdBytes, 0, consumerIdLength);
    String consumerId = CharsetUtils.toUTF8String(consumerIdBytes);
    short consumerGroupLength = Bytes.getShort(value, 10 + consumerIdLength);
    byte[] consumerGroupBytes = new byte[consumerGroupLength];
    System.arraycopy(value, 12 + consumerIdLength, consumerGroupBytes, 0, consumerGroupLength);
    String consumerGroup = CharsetUtils.toUTF8String(consumerGroupBytes);
    return new RecordQueryResult.Record(consumerGroup, action, type, timestamp, consumerId, sequence);
}
 
Example 2
Source Project: qmq   Source File: HBaseRecordStore.java    License: Apache License 2.0 6 votes vote down vote up
private List<RecordQueryResult.Record> findDeadRecord(final String subject, final String messageId) {
    final String subjectId = dicService.name2Id(subject);
    final String keyRegexp = BackupMessageKeyRegexpBuilder.buildDeadRecordRegexp(subjectId, messageId);
    final String startKey = BackupMessageKeyRangeBuilder.buildDeadRecordStartKey(subjectId, messageId);
    final String endKey = BackupMessageKeyRangeBuilder.buildDeadRecordEndKey(subjectId, messageId);

    try {
        return scan(table, keyRegexp, startKey, endKey, 100, 0, R_FAMILY, B_RECORD_QUALIFIERS, kvs -> {
            final KeyValueList<KeyValue> kvl = new KeyValueListImpl(kvs);
            final byte[] value = kvl.getValue(RECORDS);
            final long sequence = Bytes.getLong(value, 0);
            final long timestamp = Bytes.getLong(value, 8);
            final int consumerGroupLength = value.length - 16;
            final byte[] consumerGroupBytes = new byte[consumerGroupLength];
            System.arraycopy(value, 16, consumerGroupBytes, 0, consumerGroupLength);
            final String consumerGroup = CharsetUtils.toUTF8String(consumerGroupBytes);
            return new RecordQueryResult.Record(consumerGroup, ActionEnum.OMIT.getCode(), RecordEnum.DEAD_RECORD.getCode(), timestamp, "", sequence);
        });
    } catch (Exception e) {
        LOG.error("Failed to find dead records.", e);
        return Collections.emptyList();
    }
}
 
Example 3
Source Project: qmq   Source File: KeyValueListImpl.java    License: Apache License 2.0 5 votes vote down vote up
public KeyValueListImpl(List<KeyValue> keyValues) {
    if (keyValues != null && !keyValues.isEmpty()) {
        this.keyValues = keyValues;
        key = keyValues.get(0).key();
        for (KeyValue keyValue : keyValues) {
            data.put(new String(keyValue.qualifier(), CharsetUtil.UTF_8), keyValue);
        }
    }
}
 
Example 4
Source Project: qmq   Source File: AbstractHBaseMessageStore.java    License: Apache License 2.0 5 votes vote down vote up
void getMessageFromHBase(final String subject, final byte[] table, final MessageQueryResult messageQueryResult, final String keyRegexp, final String startKey, final String endKey
        , final int maxResults) {
    List<BackupMessageMeta> metas;
    try {
        metas = scan(table, keyRegexp, startKey, endKey, maxResults + 1, 0, B_FAMILY, B_MESSAGE_QUALIFIERS, kvs -> {
            KeyValueList<KeyValue> kvl = new KeyValueListImpl(kvs);
            messageQueryResult.setNext(new String(kvl.getKey(), CharsetUtil.UTF_8));
            byte[] value = kvl.getValue(CONTENT);
            BackupMessageMeta meta = getMessageMeta(value);
            if (meta == null) {
                Metrics.counter("message.content.missing").inc();
                LOG.info("Message content missing");
            }
            return meta;
        });
    } catch (Exception e) {
        LOG.error("Failed to get messages from hbase.", e);
        messageQueryResult.setList(Collections.emptyList());
        return;
    }
    int size = metas.size();
    LOG.info("Found {} metas from HBase.", size);
    slim(metas, messageQueryResult, maxResults);

    List<MessageQueryResult.MessageMeta> messages = getMessagesWithMeta(subject, metas);
    messageQueryResult.setList(messages);
}
 
Example 5
Source Project: qmq   Source File: HBaseRecordStore.java    License: Apache License 2.0 5 votes vote down vote up
private List<BackupMessageMeta> scanMessageMeta(String subject, String messageId) {
    final LocalDateTime now = LocalDateTime.now();
    final Date createTimeEnd = localDateTime2Date(now);
    final Date createTimeBegin = localDateTime2Date(now.minusDays(30));

    try {
        final String subjectId = dicService.name2Id(subject);
        final String keyRegexp = BackupMessageKeyRegexpBuilder.buildRetryRegexp(subjectId, messageId);
        final String startKey = BackupMessageKeyRangeBuilder.buildRetryRangeKey(subjectId, messageId, createTimeEnd);
        final String endKey = BackupMessageKeyRangeBuilder.buildRetryRangeKey(subjectId, messageId, createTimeBegin);
        final List<BackupMessageMeta> metas = scan(indexTable, keyRegexp, startKey, endKey, 1000, 0, B_FAMILY, B_MESSAGE_QUALIFIERS, kvs -> {
            KeyValueList<KeyValue> kvl = new KeyValueListImpl(kvs);
            byte[] value = kvl.getValue(CONTENT);
            byte[] rowKey = kvl.getKey();
            BackupMessageMeta meta = getMessageMeta(value);
            if (meta != null && rowKey.length > CONSUMER_GROUP_INDEX_IN_RETRY_MESSAGE) {
                byte[] consumerGroupId = new byte[CONSUMER_GROUP_LENGTH];
                System.arraycopy(rowKey, CONSUMER_GROUP_INDEX_IN_RETRY_MESSAGE, consumerGroupId, 0, CONSUMER_GROUP_LENGTH);
                meta.setConsumerGroupId(new String(consumerGroupId, CharsetUtil.UTF_8));
            }
            return meta;
        });
        return Lists.newArrayList(Sets.newHashSet(metas));
    } catch (Exception e) {
        LOG.error("Failed to scan messages meta.", e);
        return Lists.newArrayList();
    }
}
 
Example 6
Source Project: diver   Source File: GenServerGetCallback.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Object call(final ArrayList<KeyValue> data) throws Exception {
  final OtpErlangObject[] items = new OtpErlangObject[data.size()];
  int i = 0;
  for (final KeyValue keyValue : data) {
    final OtpErlangObject[] erldata = new OtpErlangObject[] {
        new OtpErlangBinary(keyValue.key()),
        new OtpErlangBinary(keyValue.family()),
        new OtpErlangBinary(keyValue.qualifier()),
        new OtpErlangBinary(keyValue.value()),
        new OtpErlangLong(keyValue.timestamp())
    };
    items[i] = new OtpErlangTuple(erldata);
    i++;
  }

  final OtpErlangObject[] body = new OtpErlangObject[] {
      OK_ATOM,
      new OtpErlangList(items)
  };

  final OtpErlangObject[] resp = new OtpErlangObject[] {
      from.elementAt(1),  // Ref
      new OtpErlangTuple(body)
  };

  mbox.send((OtpErlangPid) from.elementAt(0), new OtpErlangTuple(resp));
  return null;
}
 
Example 7
Source Project: Elasticsearch-HBase-River   Source File: HBaseParser.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Generate a tree structure that ElasticSearch can read and index from one of the rows that has been returned from
 * HBase.
 * 
 * @param row
 * @return
 */
@SuppressWarnings("unchecked")
protected Map<String, Object> readDataTree(final ArrayList<KeyValue> row) {
	final Map<String, Object> dataTree = new HashMap<String, Object>();
	for (final KeyValue column : row) {
		final String family = this.river.normalizeField(new String(column.family(), this.river.getCharset()));
		final String qualifier = new String(column.qualifier(), this.river.getCharset());
		final String value = new String(column.value(), this.river.getCharset());
		if (!dataTree.containsKey(family)) {
			dataTree.put(family, new HashMap<String, Object>());
		}
		readQualifierStructure((Map<String, Object>) dataTree.get(family), qualifier, value);
	}
	return dataTree;
}
 
Example 8
Source Project: Elasticsearch-HBase-River   Source File: HBaseParserTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testBase() {
	final HBaseParser parser = new HBaseParser(new HBaseRiver(null, null, null));

	final ArrayList<KeyValue> input = new ArrayList<KeyValue>();

	input.add(getKeyValue("family1", "category1", "value1"));
	input.add(getKeyValue("family1", "category2", "value2"));
	input.add(getKeyValue("family1", "category3", "value3"));
	input.add(getKeyValue("family2", "category1", "value4"));
	input.add(getKeyValue("family2", "category4", "value5"));
	input.add(getKeyValue("family3", "category5", "value6"));
	input.add(getKeyValue("family2", "category6", "value7"));

	final Map<String, Object> output = parser.readDataTree(input);

	assertNotNull(output.get("family1"));
	final Map<String, Object> family1 = (Map<String, Object>) output.get("family1");
	assertEquals(family1.get("category1"), "value1");
	assertEquals(family1.get("category2"), "value2");
	assertEquals(family1.get("category3"), "value3");
	assertNotNull(output.get("family2"));
	final Map<String, Object> family2 = (Map<String, Object>) output.get("family2");
	assertEquals(family2.get("category1"), "value4");
	assertEquals(family2.get("category4"), "value5");
	assertEquals(family2.get("category6"), "value7");
	assertNotNull(output.get("family3"));
	final Map<String, Object> family3 = (Map<String, Object>) output.get("family3");
	assertEquals(family3.get("category5"), "value6");
}
 
Example 9
Source Project: qmq   Source File: KeyValueListImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public List<KeyValue> getKeyValues() {
    return keyValues;
}
 
Example 10
Source Project: qmq   Source File: KeyValueListImpl.java    License: Apache License 2.0 4 votes vote down vote up
public void setKeyValues(ArrayList<KeyValue> keyValues) {
    this.keyValues = keyValues;
}
 
Example 11
Source Project: qmq   Source File: KeyValueListImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public Map<String, KeyValue> getData() {
    return data;
}
 
Example 12
Source Project: qmq   Source File: KeyValueListImpl.java    License: Apache License 2.0 4 votes vote down vote up
public KeyValue getKeyValue(String qualifier) {
    return data.get(qualifier);
}
 
Example 13
Source Project: qmq   Source File: KeyValueListImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public byte[] getValue(String qualifier) {
    KeyValue keyValue = getKeyValue(qualifier);
    return (keyValue == null) ? null : keyValue.value();
}
 
Example 14
Source Project: qmq   Source File: KeyValueListImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public Long getTimestamp(String qualifier) {
    KeyValue keyValue = getKeyValue(qualifier);
    return keyValue == null ? null : keyValue.timestamp();
}
 
Example 15
Source Project: Elasticsearch-HBase-River   Source File: HBaseParserTest.java    License: Apache License 2.0 4 votes vote down vote up
private KeyValue getKeyValue(final String family, final String qualifier, final String value) {
	return new KeyValue(String.valueOf(this.rowCounter++).getBytes(this.charset),
		family.getBytes(this.charset),
		qualifier.getBytes(this.charset),
		value.getBytes(this.charset));
}
 
Example 16
Source Project: qmq   Source File: KeyValueList.java    License: Apache License 2.0 votes vote down vote up
List<KeyValue> getKeyValues(); 
Example 17
Source Project: qmq   Source File: AbstractHBaseStore.java    License: Apache License 2.0 votes vote down vote up
T extractData(List<KeyValue> kvs);