Java Code Examples for org.apache.hadoop.hbase.KeyValue

The following are top voted examples for showing how to use org.apache.hadoop.hbase.KeyValue. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: ditb   File: TestSeekToBlockWithEncoders.java   View source code 7 votes vote down vote up
/**
 * Test seeking while file is encoded.
 */
@Test
public void testSeekToBlockWithDecreasingCommonPrefix() throws IOException {
  List<KeyValue> sampleKv = new ArrayList<KeyValue>();
  KeyValue kv1 = new KeyValue(Bytes.toBytes("row10aaa"), Bytes.toBytes("f1"),
      Bytes.toBytes("q1"), Bytes.toBytes("val"));
  sampleKv.add(kv1);
  KeyValue kv2 = new KeyValue(Bytes.toBytes("row10aaa"), Bytes.toBytes("f1"),
      Bytes.toBytes("q2"), Bytes.toBytes("val"));
  sampleKv.add(kv2);
  KeyValue kv3 = new KeyValue(Bytes.toBytes("row10aaa"), Bytes.toBytes("f1"),
      Bytes.toBytes("q3"), Bytes.toBytes("val"));
  sampleKv.add(kv3);
  KeyValue kv4 = new KeyValue(Bytes.toBytes("row11baa"), Bytes.toBytes("f1"),
      Bytes.toBytes("q1"), Bytes.toBytes("val"));
  sampleKv.add(kv4);
  KeyValue toSeek = KeyValueUtil.createLastOnRow(kv3.getRowArray(), kv3.getRowOffset(),
      kv3.getRowLength(), null, 0, 0, null, 0, 0);
  seekToTheKey(kv3, sampleKv, toSeek);
}
 
Example 2
Project: ditb   File: ThriftServerRunner.java   View source code 7 votes vote down vote up
@Override
public void deleteAllTs(ByteBuffer tableName,
                        ByteBuffer row,
                        ByteBuffer column,
    long timestamp, Map<ByteBuffer, ByteBuffer> attributes) throws IOError {
  Table table = null;
  try {
    table = getTable(tableName);
    Delete delete  = new Delete(getBytes(row));
    addAttributes(delete, attributes);
    byte [][] famAndQf = KeyValue.parseColumn(getBytes(column));
    if (famAndQf.length == 1) {
      delete.deleteFamily(famAndQf[0], timestamp);
    } else {
      delete.deleteColumns(famAndQf[0], famAndQf[1], timestamp);
    }
    table.delete(delete);

  } catch (IOException e) {
    LOG.warn(e.getMessage(), e);
    throw new IOError(Throwables.getStackTraceAsString(e));
  } finally {
    closeTable(table);
  }
}
 
Example 3
Project: SparkDemo   File: HBaseTest.java   View source code 6 votes vote down vote up
/**
 * show data
 */
public static void getAllRecord(String tableName) {
    try {
        Table table = connection.getTable(TableName.valueOf(tableName));
        Scan s = new Scan();

        ResultScanner ss = table.getScanner(s);
        for (Result r : ss) {
            for (KeyValue kv : r.raw()) {
                System.out.print(new String(kv.getRow()) + " ");
                System.out.print(new String(kv.getFamily()) + ":");
                System.out.print(new String(kv.getQualifier()) + " ");
                System.out.print(kv.getTimestamp() + " ");
                System.out.println(new String(kv.getValue()));
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 
Example 4
Project: ditb   File: Range.java   View source code 6 votes vote down vote up
public static Range[] fromFilter(SingleColumnValueFilter filter) {
  if (!(filter.getComparator() instanceof BinaryComparator)) {
    return new Range[0];
  }

  byte[] column = KeyValue.makeColumn(filter.getFamily(), filter.getQualifier());
  CompareOp compareOp = filter.getOperator();
  byte[] value = filter.getComparator().getValue();

  if (compareOp == CompareOp.NOT_EQUAL) {
    return new Range[] { new Range(column, null, CompareOp.NO_OP, value, CompareOp.LESS),
        new Range(column, value, CompareOp.GREATER, null, CompareOp.NO_OP) };
  } else {
    switch (compareOp) {
    case EQUAL:
    case GREATER_OR_EQUAL:
    case GREATER:
      return new Range[] { new Range(column, value, compareOp, null, CompareOp.NO_OP) };
    case LESS:
    case LESS_OR_EQUAL:
      return new Range[] { new Range(column, null, CompareOp.NO_OP, value, compareOp) };
    default:
      return new Range[0];
    }
  }
}
 
Example 5
Project: HBase-High-Performance-Cookbook   File: HBaseRegularClient.java   View source code 6 votes vote down vote up
/**
 * Getting all records  a row from an existing SS tables 
 * @method getAllRecord
 * @inputParameters hbaseBtable Name used
 * @return type: no return type as its a void method 
 * 
 **/
@SuppressWarnings({ "deprecation", "resource" })
public static void getAllRecord(String myHbaseBtableName) {
  ResultScanner hbaseBSs = null;
  try {
    HTable hbaseBtable = new HTable(hbaseBconf, myHbaseBtableName);
    Scan hbaseBScan = new Scan();
    hbaseBSs = hbaseBtable.getScanner(hbaseBScan);
    for (Result r : hbaseBSs) {
      for (KeyValue hbaseBkv : r.raw()) {
        System.out.print(new String(hbaseBkv.getRow()) + " ");
        System.out.print(new String(hbaseBkv.getFamily()) + ":");
        System.out.print(new String(hbaseBkv.getQualifier()) + " ");
        System.out.print(hbaseBkv.getTimestamp() + " ");
        System.out.println(new String(hbaseBkv.getValue()));
      }
    }
  } catch (IOException eio) {
    eip.printStackTrace();
  } finally {
    if (hbaseBSs != null) hbaseBSs.close();
    // closing the ss hbaseBtable 
  }
}
 
Example 6
Project: ditb   File: TestSeekToBlockWithEncoders.java   View source code 6 votes vote down vote up
@Test
public void testSeekToBlockWithDiffFamilyAndQualifer() throws IOException {
  List<KeyValue> sampleKv = new ArrayList<KeyValue>();
  KeyValue kv1 = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("fam1"), Bytes.toBytes("q1"),
      Bytes.toBytes("val"));
  sampleKv.add(kv1);
  KeyValue kv2 = new KeyValue(Bytes.toBytes("aab"), Bytes.toBytes("fam1"), Bytes.toBytes("q1"),
      Bytes.toBytes("val"));
  sampleKv.add(kv2);
  KeyValue kv4 = new KeyValue(Bytes.toBytes("aac"), Bytes.toBytes("fam1"), Bytes.toBytes("q1"),
      Bytes.toBytes("val"));
  sampleKv.add(kv4);
  KeyValue kv5 = new KeyValue(Bytes.toBytes("aac"), Bytes.toBytes("fam1"), Bytes.toBytes("q2"),
      Bytes.toBytes("val"));
  sampleKv.add(kv5);
  KeyValue toSeek = new KeyValue(Bytes.toBytes("aac"), Bytes.toBytes("fam2"),
      Bytes.toBytes("q2"), Bytes.toBytes("val"));
  seekToTheKey(kv5, sampleKv, toSeek);
}
 
Example 7
Project: ditb   File: TestSeekToBlockWithEncoders.java   View source code 6 votes vote down vote up
@Test
public void testSeekToBlockWithDiffQualiferOnSameRowButDescendingInSize() throws IOException {
  List<KeyValue> sampleKv = new ArrayList<KeyValue>();
  KeyValue kv1 = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("qual1"),
      Bytes.toBytes("val"));
  sampleKv.add(kv1);
  KeyValue kv2 = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("qual2"),
      Bytes.toBytes("val"));
  sampleKv.add(kv2);
  KeyValue kv4 = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("qual3"),
      Bytes.toBytes("val"));
  sampleKv.add(kv4);
  KeyValue kv5 = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("qual4"),
      Bytes.toBytes("val"));
  sampleKv.add(kv5);
  KeyValue kv6 = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("qz"),
      Bytes.toBytes("val"));
  sampleKv.add(kv6);
  KeyValue toSeek = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("qz"),
      Bytes.toBytes("val"));
  seekToTheKey(kv6, sampleKv, toSeek);
}
 
Example 8
Project: ditb   File: BaseIndexScanner.java   View source code 6 votes vote down vote up
public static List<Cell> recoverClusteringResult(List<Cell> cells, byte[] family,
    byte[] qualifier) {
  if (cells == null || cells.size() == 0) return cells;
  byte[][] indexColumn = IndexPutParser.parseIndexRowKey(cells.get(0).getRow());
  List<Cell> list = new ArrayList<>(cells.size() + 1);
  for (Cell cell : cells) {
    byte[] tag = cell.getTagsArray();
    if (tag != null && tag.length > KeyValue.MAX_TAGS_LENGTH) tag = null;
    KeyValue kv =
        new KeyValue(indexColumn[0], CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell),
            cell.getTimestamp(), KeyValue.Type.codeToType(cell.getTypeByte()),
            CellUtil.cloneValue(cell), tag);
    list.add(kv);
  }
  list.add(new KeyValue(indexColumn[0], family, qualifier, indexColumn[1]));
  Collections.sort(list, KeyValue.COMPARATOR);
  return list;
}
 
Example 9
Project: ditb   File: TestReplicationSink.java   View source code 6 votes vote down vote up
/**
 * Insert a mix of puts and deletes
 * @throws Exception
 */
@Test
public void testMixedPutDelete() throws Exception {
  List<WALEntry> entries = new ArrayList<WALEntry>(BATCH_SIZE/2);
  List<Cell> cells = new ArrayList<Cell>();
  for(int i = 0; i < BATCH_SIZE/2; i++) {
    entries.add(createEntry(TABLE_NAME1, i, KeyValue.Type.Put, cells));
  }
  SINK.replicateEntries(entries, CellUtil.createCellScanner(cells));

  entries = new ArrayList<WALEntry>(BATCH_SIZE);
  cells = new ArrayList<Cell>();
  for(int i = 0; i < BATCH_SIZE; i++) {
    entries.add(createEntry(TABLE_NAME1, i,
        i % 2 != 0 ? KeyValue.Type.Put: KeyValue.Type.DeleteColumn, cells));
  }

  SINK.replicateEntries(entries, CellUtil.createCellScanner(cells.iterator()));
  Scan scan = new Scan();
  ResultScanner scanRes = table1.getScanner(scan);
  assertEquals(BATCH_SIZE/2, scanRes.next(BATCH_SIZE).length);
}
 
Example 10
Project: ditb   File: TestScannersWithLabels.java   View source code 6 votes vote down vote up
private static int insertData(TableName tableName, String column, double prob) throws IOException {
  byte[] k = new byte[3];
  byte[][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(column));

  List<Put> puts = new ArrayList<>();
  for (int i = 0; i < 9; i++) {
    Put put = new Put(Bytes.toBytes("row" + i));
    put.setDurability(Durability.SKIP_WAL);
    put.add(famAndQf[0], famAndQf[1], k);
    put.setCellVisibility(new CellVisibility("(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!"
        + TOPSECRET));
    puts.add(put);
  }
  try (Table table = new HTable(TEST_UTIL.getConfiguration(), tableName)) {
    table.put(puts);
  }
  return puts.size();
}
 
Example 11
Project: ditb   File: TestHFileDataBlockEncoder.java   View source code 6 votes vote down vote up
private HFileBlock createBlockOnDisk(List<KeyValue> kvs, HFileBlock block, boolean useTags)
    throws IOException {
  int size;
  HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext(
      blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER,
      block.getHFileContext());

  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  baos.write(block.getDummyHeaderForVersion());
  DataOutputStream dos = new DataOutputStream(baos);
  blockEncoder.startBlockEncoding(context, dos);
  for (KeyValue kv : kvs) {
    blockEncoder.encode(kv, context, dos);
  }
  BufferGrabbingByteArrayOutputStream stream = new BufferGrabbingByteArrayOutputStream();
  baos.writeTo(stream);
  blockEncoder.endBlockEncoding(context, dos, stream.getBuffer(), BlockType.DATA);
  byte[] encodedBytes = baos.toByteArray();
  size = encodedBytes.length - block.getDummyHeaderForVersion().length;
  return new HFileBlock(context.getBlockType(), size, size, -1, ByteBuffer.wrap(encodedBytes),
      HFileBlock.FILL_HEADER, 0, block.getOnDiskDataSizeWithHeader(), block.getHFileContext());
}
 
Example 12
Project: ditb   File: TestCompoundBloomFilter.java   View source code 6 votes vote down vote up
@Test
public void testCreateKey() {
  CompoundBloomFilterBase cbfb = new CompoundBloomFilterBase();
  byte[] row = "myRow".getBytes();
  byte[] qualifier = "myQualifier".getBytes();
  byte[] rowKey = cbfb.createBloomKey(row, 0, row.length,
      row, 0, 0);
  byte[] rowColKey = cbfb.createBloomKey(row, 0, row.length,
      qualifier, 0, qualifier.length);
  KeyValue rowKV = KeyValue.createKeyValueFromKey(rowKey);
  KeyValue rowColKV = KeyValue.createKeyValueFromKey(rowColKey);
  assertEquals(rowKV.getTimestamp(), rowColKV.getTimestamp());
  assertEquals(Bytes.toStringBinary(rowKV.getRow()),
      Bytes.toStringBinary(rowColKV.getRow()));
  assertEquals(0, rowKV.getQualifier().length);
}
 
Example 13
Project: ditb   File: TestStoreFileScannerWithTagCompression.java   View source code 6 votes vote down vote up
private void writeStoreFile(final StoreFile.Writer writer) throws IOException {
  byte[] fam = Bytes.toBytes("f");
  byte[] qualifier = Bytes.toBytes("q");
  long now = System.currentTimeMillis();
  byte[] b = Bytes.toBytes("k1");
  Tag t1 = new Tag((byte) 1, "tag1");
  Tag t2 = new Tag((byte) 2, "tag2");
  Tag t3 = new Tag((byte) 3, "tag3");
  try {
    writer.append(new KeyValue(b, fam, qualifier, now, b, new Tag[] { t1 }));
    b = Bytes.toBytes("k3");
    writer.append(new KeyValue(b, fam, qualifier, now, b, new Tag[] { t2, t1 }));
    b = Bytes.toBytes("k4");
    writer.append(new KeyValue(b, fam, qualifier, now, b, new Tag[] { t3 }));
    b = Bytes.toBytes("k5");
    writer.append(new KeyValue(b, fam, qualifier, now, b, new Tag[] { t3 }));
  } finally {
    writer.close();
  }
}
 
Example 14
Project: ditb   File: GetClosestRowBeforeTracker.java   View source code 6 votes vote down vote up
/**
 * @param c
 * @param kv Presume first on row: i.e. empty column, maximum timestamp and
 * a type of Type.Maximum
 * @param ttl Time to live in ms for this Store
 * @param metaregion True if this is hbase:meta or -ROOT- region.
 */
GetClosestRowBeforeTracker(final KVComparator c, final KeyValue kv,
    final long ttl, final boolean metaregion) {
  super();
  this.metaregion = metaregion;
  this.targetkey = kv;
  // If we are in a metaregion, then our table name is the prefix on the
  // targetkey.
  this.rowoffset = kv.getRowOffset();
  int l = -1;
  if (metaregion) {
    l = KeyValue.getDelimiter(kv.getRowArray(), rowoffset, kv.getRowLength(),
      HConstants.DELIMITER) - this.rowoffset;
  }
  this.tablenamePlusDelimiterLength = metaregion? l + 1: -1;
  this.now = System.currentTimeMillis();
  this.oldestUnexpiredTs = now - ttl;
  this.kvcomparator = c;
  KeyValue.RowOnlyComparator rc = new KeyValue.RowOnlyComparator(this.kvcomparator);
  this.deletes = new TreeMap<KeyValue, NavigableSet<KeyValue>>(rc);
}
 
Example 15
Project: ditb   File: WALEdit.java   View source code 6 votes vote down vote up
@Override
public void write(DataOutput out) throws IOException {
  LOG.warn("WALEdit is being serialized to writable - only expected in test code");
  out.writeInt(VERSION_2);
  out.writeInt(cells.size());
  // We interleave the two lists for code simplicity
  for (Cell cell : cells) {
    // This is not used in any of the core code flows so it is just fine to convert to KV
    KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
    if (compressionContext != null) {
      KeyValueCompression.writeKV(out, kv, compressionContext);
    } else{
      KeyValue.write(kv, out);
    }
  }
  if (scopes == null) {
    out.writeInt(0);
  } else {
    out.writeInt(scopes.size());
    for (byte[] key : scopes.keySet()) {
      Bytes.writeByteArray(out, key);
      out.writeInt(scopes.get(key));
    }
  }
}
 
Example 16
Project: ditb   File: TestHFileOutputFormat.java   View source code 6 votes vote down vote up
/**
 * Write random values to the writer assuming a table created using
 * {@link #FAMILIES} as column family descriptors
 */
private void writeRandomKeyValues(RecordWriter<ImmutableBytesWritable, KeyValue> writer,
    TaskAttemptContext context, Set<byte[]> families, int numRows)
    throws IOException, InterruptedException {
  byte keyBytes[] = new byte[Bytes.SIZEOF_INT];
  int valLength = 10;
  byte valBytes[] = new byte[valLength];

  int taskId = context.getTaskAttemptID().getTaskID().getId();
  assert taskId < Byte.MAX_VALUE : "Unit tests dont support > 127 tasks!";
  final byte [] qualifier = Bytes.toBytes("data");
  Random random = new Random();
  for (int i = 0; i < numRows; i++) {

    Bytes.putInt(keyBytes, 0, i);
    random.nextBytes(valBytes);
    ImmutableBytesWritable key = new ImmutableBytesWritable(keyBytes);

    for (byte[] family : families) {
      KeyValue kv = new KeyValue(keyBytes, family, qualifier, valBytes);
      writer.write(key, kv);
    }
  }
}
 
Example 17
Project: ditb   File: HRegion.java   View source code 6 votes vote down vote up
LCIndexQueryProcessor createLCIndexProcessor(IndexTableRelation indexTableRelation,
    Map<byte[], Store> storeMap, Scan scan, long readPt, KeyValue.KVComparator comparator) {
  try {
    ScanRange.ScanRangeList rangeList = ScanRange.ScanRangeList.getScanRangeList(scan);
    checkSingleScanFamily(scan, rangeList);
    // calculate the primary range for scan
    ScanRange primaryRange =
        selectTheBestRange(indexTableRelation, storeMap, rangeList, scan.getId());
    if (primaryRange != null) {
      printRanges(primaryRange, rangeList);
      return new LCIndexQueryProcessor(scan, indexTableRelation, primaryRange, rangeList,
          (HStore) storeMap.get(primaryRange.getFamily()),
          scan.getFamilyMap().get(primaryRange.getFamily()), readPt, comparator);
    }
  } catch (IOException e) {
    e.printStackTrace();
  }
  return null;
}
 
Example 18
Project: ditb   File: DefaultMemStore.java   View source code 6 votes vote down vote up
/**
 * Constructor.
 * @param c Comparator
 */
public DefaultMemStore(final Configuration conf,
                final KeyValue.KVComparator c) {
  this.conf = conf;
  this.comparator = c;
  this.cellSet = new CellSkipListSet(c);
  this.snapshot = new CellSkipListSet(c);
  timeRangeTracker = new TimeRangeTracker();
  snapshotTimeRangeTracker = new TimeRangeTracker();
  this.size = new AtomicLong(DEEP_OVERHEAD);
  this.snapshotSize = 0;
  if (conf.getBoolean(USEMSLAB_KEY, USEMSLAB_DEFAULT)) {
    String className = conf.get(MSLAB_CLASS_NAME, HeapMemStoreLAB.class.getName());
    this.allocator = ReflectionUtils.instantiateWithCustomCtor(className,
        new Class[] { Configuration.class }, new Object[] { conf });
  } else {
    this.allocator = null;
  }
}
 
Example 19
Project: ditb   File: TestStoreScanner.java   View source code 6 votes vote down vote up
public void testDeleteVersionMaskingMultiplePuts() throws IOException {
  long now = System.currentTimeMillis();
  KeyValue [] kvs1 = new KeyValue[] {
      KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care")
  };
  KeyValue [] kvs2 = new KeyValue[] {
      KeyValueTestUtil.create("R1", "cf", "a", now-500, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "a", now-100, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care")
  };
  List<KeyValueScanner> scanners = scanFixture(kvs1, kvs2);

  StoreScanner scan = new StoreScanner(new Scan(Bytes.toBytes("R1")),
      scanInfo, scanType, getCols("a"), scanners);
  List<Cell> results = new ArrayList<Cell>();
  // the two put at ts=now will be masked by the 1 delete, and
  // since the scan default returns 1 version we'll return the newest
  // key, which is kvs[2], now-100.
  assertEquals(true, scan.next(results));
  assertEquals(1, results.size());
  assertEquals(kvs2[1], results.get(0));
}
 
Example 20
Project: ditb   File: TestCellMessageCodec.java   View source code 6 votes vote down vote up
@Test
public void testOne() throws IOException {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  CountingOutputStream cos = new CountingOutputStream(baos);
  DataOutputStream dos = new DataOutputStream(cos);
  MessageCodec cmc = new MessageCodec();
  Codec.Encoder encoder = cmc.getEncoder(dos);
  final KeyValue kv =
    new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("q"), Bytes.toBytes("v"));
  encoder.write(kv);
  encoder.flush();
  dos.close();
  long offset = cos.getCount();
  CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray()));
  DataInputStream dis = new DataInputStream(cis);
  Codec.Decoder decoder = cmc.getDecoder(dis);
  assertTrue(decoder.advance()); // First read should pull in the KV
  assertFalse(decoder.advance()); // Second read should trip over the end-of-stream  marker and return false
  dis.close();
  assertEquals(offset, cis.getCount());
}
 
Example 21
Project: SparkDemo   File: HBaseTest.java   View source code 5 votes vote down vote up
/**
 * query record
 */
public static void getOneRecord(String tableName, String rowKey)
        throws IOException {
    Table table = connection.getTable(TableName.valueOf(tableName));
    Get get = new Get(rowKey.getBytes());
    Result rs = table.get(get);
    for (KeyValue kv : rs.raw()) {
        System.out.print(new String(kv.getRow()) + " ");
        System.out.print(new String(kv.getFamily()) + ":");
        System.out.print(new String(kv.getQualifier()) + " ");
        System.out.print(kv.getTimestamp() + " ");
        System.out.println(new String(kv.getValue()));
    }
}
 
Example 22
Project: ditb   File: HalfStoreFileReader.java   View source code 5 votes vote down vote up
/**
 * Creates a half file reader for a hfile referred to by an hfilelink.
 * @param fs fileystem to read from
 * @param p path to hfile
 * @param in {@link FSDataInputStreamWrapper}
 * @param size Full size of the hfile file
 * @param cacheConf
 * @param r original reference file (contains top or bottom)
 * @param conf Configuration
 * @throws IOException
 */
public HalfStoreFileReader(final FileSystem fs, final Path p, final FSDataInputStreamWrapper in,
    long size, final CacheConfig cacheConf,  final Reference r, final Configuration conf)
    throws IOException {
  super(fs, p, in, size, cacheConf, conf);
  // This is not actual midkey for this half-file; its just border
  // around which we split top and bottom.  Have to look in files to find
  // actual last and first keys for bottom and top halves.  Half-files don't
  // have an actual midkey themselves. No midkey is how we indicate file is
  // not splittable.
  this.splitkey = r.getSplitKey();
  this.splitCell = new KeyValue.KeyOnlyKeyValue(this.splitkey, 0, this.splitkey.length);
  // Is it top or bottom half?
  this.top = Reference.isTopFileRegion(r.getFileRegion());
}
 
Example 23
Project: ditb   File: TestGroupingTableMapper.java   View source code 5 votes vote down vote up
/**
 * Test GroupingTableMapper class
 */
@Test
public void testGroupingTableMapper() throws Exception {

  GroupingTableMapper mapper = new GroupingTableMapper();
  Configuration configuration = new Configuration();
  configuration.set(GroupingTableMapper.GROUP_COLUMNS, "family1:clm family2:clm");
  mapper.setConf(configuration);

  Result result = mock(Result.class);
  @SuppressWarnings("unchecked")
  Mapper<ImmutableBytesWritable, Result, ImmutableBytesWritable, Result>.Context context =
      mock(Mapper.Context.class);
  context.write(any(ImmutableBytesWritable.class), any(Result.class));
  List<Cell> keyValue = new ArrayList<Cell>();
  byte[] row = {};
  keyValue.add(new KeyValue(row, Bytes.toBytes("family2"), Bytes.toBytes("clm"), Bytes
      .toBytes("value1")));
  keyValue.add(new KeyValue(row, Bytes.toBytes("family1"), Bytes.toBytes("clm"), Bytes
      .toBytes("value2")));
  when(result.listCells()).thenReturn(keyValue);
  mapper.map(null, result, context);
  // template data
  byte[][] data = { Bytes.toBytes("value1"), Bytes.toBytes("value2") };
  ImmutableBytesWritable ibw = mapper.createGroupKey(data);
  verify(context).write(ibw, result);
}
 
Example 24
Project: ditb   File: TestRowEncoder.java   View source code 5 votes vote down vote up
@Test
public void testForwardScanner() {
  int counter = -1;
  while (searcher.advance()) {
    ++counter;
    KeyValue inputKv = rows.getInputs().get(counter);
    KeyValue outputKv = KeyValueUtil.copyToNewKeyValue(searcher.current());
    assertKeyAndValueEqual(inputKv, outputKv);
  }
  // assert same number of cells
  Assert.assertEquals(rows.getInputs().size(), counter + 1);
}
 
Example 25
Project: ditb   File: IncrementCoalescer.java   View source code 5 votes vote down vote up
private boolean internalQueueTincrement(TIncrement inc) throws TException {
  byte[][] famAndQf = KeyValue.parseColumn(inc.getColumn());
  if (famAndQf.length != 2) return false;

  return internalQueueIncrement(inc.getTable(), inc.getRow(), famAndQf[0], famAndQf[1],
    inc.getAmmount());
}
 
Example 26
Project: ditb   File: TsvImporterCustomTestMapper.java   View source code 5 votes vote down vote up
/**
 * Convert a line of TSV text into an HBase table row after transforming the
 * values by multiplying them by 3.
 */
@Override
public void map(LongWritable offset, Text value, Context context)
      throws IOException {
  byte[] family = Bytes.toBytes("FAM");
  final byte[][] qualifiers = { Bytes.toBytes("A"), Bytes.toBytes("B") };

  // do some basic line parsing
  byte[] lineBytes = value.getBytes();
  String[] valueTokens = new String(lineBytes, "UTF-8").split("\u001b");

  // create the rowKey and Put
  ImmutableBytesWritable rowKey =
    new ImmutableBytesWritable(Bytes.toBytes(valueTokens[0]));
  Put put = new Put(rowKey.copyBytes());
  put.setDurability(Durability.SKIP_WAL);

  //The value should look like this: VALUE1 or VALUE2. Let's multiply
  //the integer by 3
  for(int i = 1; i < valueTokens.length; i++) {
    String prefix = valueTokens[i].substring(0, "VALUE".length());
    String suffix = valueTokens[i].substring("VALUE".length());
    String newValue = prefix + Integer.parseInt(suffix) * 3;

    KeyValue kv = new KeyValue(rowKey.copyBytes(), family,
        qualifiers[i-1], Bytes.toBytes(newValue));
    put.add(kv);
  }

  try {
    context.write(rowKey, put);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }
}
 
Example 27
Project: ditb   File: TestInvocationRecordFilter.java   View source code 5 votes vote down vote up
public void verifyInvocationResults(Integer[] selectQualifiers,
    Integer[] expectedQualifiers) throws Exception {
  Get get = new Get(ROW_BYTES);
  for (int i = 0; i < selectQualifiers.length; i++) {
    get.addColumn(FAMILY_NAME_BYTES,
        Bytes.toBytes(QUALIFIER_PREFIX + selectQualifiers[i]));
  }

  get.setFilter(new InvocationRecordFilter());

  List<KeyValue> expectedValues = new ArrayList<KeyValue>();
  for (int i = 0; i < expectedQualifiers.length; i++) {
    expectedValues.add(new KeyValue(ROW_BYTES, FAMILY_NAME_BYTES, Bytes
        .toBytes(QUALIFIER_PREFIX + expectedQualifiers[i]),
        expectedQualifiers[i], Bytes.toBytes(VALUE_PREFIX
            + expectedQualifiers[i])));
  }

  Scan scan = new Scan(get);
  List<Cell> actualValues = new ArrayList<Cell>();
  List<Cell> temp = new ArrayList<Cell>();
  InternalScanner scanner = this.region.getScanner(scan);
  while (scanner.next(temp)) {
    actualValues.addAll(temp);
    temp.clear();
  }
  actualValues.addAll(temp);
  Assert.assertTrue("Actual values " + actualValues
      + " differ from the expected values:" + expectedValues,
      expectedValues.equals(actualValues));
}
 
Example 28
Project: ditb   File: TestResult.java   View source code 5 votes vote down vote up
public void testBasicGetValue() throws Exception {
  KeyValue [] kvs = genKVs(row, family, value, 1, 100);

  Arrays.sort(kvs, KeyValue.COMPARATOR);

  Result r = Result.create(kvs);

  for (int i = 0; i < 100; ++i) {
    final byte[] qf = Bytes.toBytes(i);

    assertByteEquals(Bytes.add(value, Bytes.toBytes(i)), r.getValue(family, qf));
    assertTrue(r.containsColumn(family, qf));
  }
}
 
Example 29
Project: ditb   File: FilterWrapper.java   View source code 5 votes vote down vote up
/**
 * This method is deprecated and you should override Cell getNextKeyHint(Cell) instead.
 */
@Override
@Deprecated
public KeyValue getNextKeyHint(KeyValue currentKV) throws IOException {
  // This will never get called.
  return KeyValueUtil.ensureKeyValue(this.filter.getNextCellHint((Cell)currentKV));
}
 
Example 30
Project: ditb   File: TestScannersFromClientSide.java   View source code 5 votes vote down vote up
@Test
public void testSmallScan() throws Exception {
  TableName TABLE = TableName.valueOf("testSmallScan");

  int numRows = 10;
  byte[][] ROWS = HTestConst.makeNAscii(ROW, numRows);

  int numQualifiers = 10;
  byte[][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, numQualifiers);

  Table ht = TEST_UTIL.createTable(TABLE, FAMILY);

  Put put;
  List<Put> puts = new ArrayList<Put>();
  for (int row = 0; row < ROWS.length; row++) {
    put = new Put(ROWS[row]);
    for (int qual = 0; qual < QUALIFIERS.length; qual++) {
      KeyValue kv = new KeyValue(ROWS[row], FAMILY, QUALIFIERS[qual], VALUE);
      put.add(kv);
    }
    puts.add(put);
  }
  ht.put(puts);

  int expectedRows = numRows;
  int expectedCols = numRows * numQualifiers;

  // Test normal and reversed
  testSmallScan(ht, true, expectedRows, expectedCols);
  testSmallScan(ht, false, expectedRows, expectedCols);
}
 
Example 31
Project: ditb   File: Put.java   View source code 5 votes vote down vote up
/**
 * Add the specified column and value, with the specified timestamp as
 * its version to this Put operation.
 * @param family family name
 * @param qualifier column qualifier
 * @param ts version timestamp
 * @param value column value
 * @return this
 */
public Put addColumn(byte [] family, byte [] qualifier, long ts, byte [] value) {
  if (ts < 0) {
    throw new IllegalArgumentException("Timestamp cannot be negative. ts=" + ts);
  }
  List<Cell> list = getCellList(family);
  KeyValue kv = createPutKeyValue(family, qualifier, ts, value);
  list.add(kv);
  familyMap.put(CellUtil.cloneFamily(kv), list);
  return this;
}
 
Example 32
Project: ditb   File: TestStoreScanner.java   View source code 5 votes vote down vote up
public void testDeleteFamily() throws IOException {
  KeyValue [] kvs = new KeyValue[] {
      KeyValueTestUtil.create("R1", "cf", "a", 100, KeyValue.Type.DeleteFamily, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "e", 11, KeyValue.Type.DeleteColumn, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "g", 11, KeyValue.Type.Delete, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
  };
  List<KeyValueScanner> scanners = scanFixture(kvs);
  StoreScanner scan = new StoreScanner(
      new Scan().setMaxVersions(Integer.MAX_VALUE), scanInfo, scanType, null,
      scanners);
  List<Cell> results = new ArrayList<Cell>();
  assertEquals(true, scan.next(results));
  assertEquals(0, results.size());
  assertEquals(true, scan.next(results));
  assertEquals(1, results.size());
  assertEquals(kvs[kvs.length-1], results.get(0));

  assertEquals(false, scan.next(results));
}
 
Example 33
Project: ditb   File: TestPrefetch.java   View source code 5 votes vote down vote up
public static KeyValue.Type generateKeyType(Random rand) {
  if (rand.nextBoolean()) {
    // Let's make half of KVs puts.
    return KeyValue.Type.Put;
  } else {
    KeyValue.Type keyType =
        KeyValue.Type.values()[1 + rand.nextInt(NUM_VALID_KEY_TYPES)];
    if (keyType == KeyValue.Type.Minimum || keyType == KeyValue.Type.Maximum)
    {
      throw new RuntimeException("Generated an invalid key type: " + keyType
          + ". " + "Probably the layout of KeyValue.Type has changed.");
    }
    return keyType;
  }
}
 
Example 34
Project: ditb   File: TestReplicationSource.java   View source code 5 votes vote down vote up
/**
 * Sanity check that we can move logs around while we are reading
 * from them. Should this test fail, ReplicationSource would have a hard
 * time reading logs that are being archived.
 * @throws Exception
 */
@Test
public void testLogMoving() throws Exception{
  Path logPath = new Path(logDir, "log");
  if (!FS.exists(logDir)) FS.mkdirs(logDir);
  if (!FS.exists(oldLogDir)) FS.mkdirs(oldLogDir);
  WALProvider.Writer writer = WALFactory.createWALWriter(FS, logPath,
      TEST_UTIL.getConfiguration());
  for(int i = 0; i < 3; i++) {
    byte[] b = Bytes.toBytes(Integer.toString(i));
    KeyValue kv = new KeyValue(b,b,b);
    WALEdit edit = new WALEdit();
    edit.add(kv);
    WALKey key = new WALKey(b, TableName.valueOf(b), 0, 0,
        HConstants.DEFAULT_CLUSTER_ID);
    writer.append(new WAL.Entry(key, edit));
    writer.sync();
  }
  writer.close();

  WAL.Reader reader = WALFactory.createReader(FS, logPath, TEST_UTIL.getConfiguration());
  WAL.Entry entry = reader.next();
  assertNotNull(entry);

  Path oldLogPath = new Path(oldLogDir, "log");
  FS.rename(logPath, oldLogPath);

  entry = reader.next();
  assertNotNull(entry);

  entry = reader.next();
  entry = reader.next();

  assertNull(entry);
  reader.close();
}
 
Example 35
Project: ditb   File: TestRowEncoder.java   View source code 5 votes vote down vote up
/**
 * probably not needed since testReverseScannerWithJitter() below is more thorough
 */
@Test
public void testReverseScanner() {
  searcher.positionAfterLastCell();
  int counter = -1;
  while (searcher.previous()) {
    ++counter;
    int oppositeIndex = rows.getInputs().size() - counter - 1;
    KeyValue inputKv = rows.getInputs().get(oppositeIndex);
    KeyValue outputKv = KeyValueUtil.copyToNewKeyValue(searcher.current());
    assertKeyAndValueEqual(inputKv, outputKv);
  }
  Assert.assertEquals(rows.getInputs().size(), counter + 1);
}
 
Example 36
Project: ditb   File: CollectionBackedScanner.java   View source code 5 votes vote down vote up
public CollectionBackedScanner(KeyValue.KVComparator comparator,
    Cell... array) {
  this.comparator = comparator;

  List<Cell> tmp = new ArrayList<Cell>(array.length);
  Collections.addAll(tmp, array);
  Collections.sort(tmp, comparator);
  data = tmp;
  init();
}
 
Example 37
Project: ditb   File: IndexSpecification.java   View source code 5 votes vote down vote up
/**
 * Set column for this index.
 *
 * @param indexColumn
 */
public void setIndexColumn(byte[] indexColumn) {
  this.indexColumn = indexColumn;
  byte[][] fq = KeyValue.parseColumn(this.indexColumn);
  this.family = fq[0];
  this.qualifier = fq[1];
  this.indexId = Bytes.add(this.family, Bytes.toBytes("_"), this.qualifier);
}
 
Example 38
Project: ditb   File: Delete.java   View source code 5 votes vote down vote up
/**
 * Delete the specified version of the specified column.
 * @param family family name
 * @param qualifier column qualifier
 * @param timestamp version timestamp
 * @return this for invocation chaining
 */
public Delete addColumn(byte [] family, byte [] qualifier, long timestamp) {
  if (timestamp < 0) {
    throw new IllegalArgumentException("Timestamp cannot be negative. ts=" + timestamp);
  }
  List<Cell> list = familyMap.get(family);
  if(list == null) {
    list = new ArrayList<Cell>();
  }
  KeyValue kv = new KeyValue(this.row, family, qualifier, timestamp, KeyValue.Type.Delete);
  list.add(kv);
  familyMap.put(family, list);
  return this;
}
 
Example 39
Project: ditb   File: HStore.java   View source code 5 votes vote down vote up
private boolean seekToScanner(final HFileScanner scanner, final KeyValue firstOnRow,
    final KeyValue firstKV) throws IOException {
  KeyValue kv = firstOnRow;
  // If firstOnRow < firstKV, set to firstKV
  if (this.comparator.compareRows(firstKV, firstOnRow) == 0) kv = firstKV;
  int result = scanner.seekTo(kv);
  return result != -1;
}
 
Example 40
Project: ditb   File: TestCellMessageCodec.java   View source code 5 votes vote down vote up
@Test
public void testThree() throws IOException {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  CountingOutputStream cos = new CountingOutputStream(baos);
  DataOutputStream dos = new DataOutputStream(cos);
  MessageCodec cmc = new MessageCodec();
  Codec.Encoder encoder = cmc.getEncoder(dos);
  final KeyValue kv1 = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("1"), Bytes.toBytes("1"));
  final KeyValue kv2 = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("2"), Bytes.toBytes("2"));
  final KeyValue kv3 = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("3"), Bytes.toBytes("3"));
  encoder.write(kv1);
  encoder.write(kv2);
  encoder.write(kv3);
  encoder.flush();
  dos.close();
  long offset = cos.getCount();
  CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray()));
  DataInputStream dis = new DataInputStream(cis);
  Codec.Decoder decoder = cmc.getDecoder(dis);
  assertTrue(decoder.advance());
  Cell c = decoder.current();
  assertTrue(CellComparator.equals(c, kv1));
  assertTrue(decoder.advance());
  c = decoder.current();
  assertTrue(CellComparator.equals(c, kv2));
  assertTrue(decoder.advance());
  c = decoder.current();
  assertTrue(CellComparator.equals(c, kv3));
  assertFalse(decoder.advance());
  dis.close();
  assertEquals(offset, cis.getCount());
}