Java Code Examples for org.apache.hadoop.hbase.KeyValue

The following are top voted examples for showing how to use org.apache.hadoop.hbase.KeyValue. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: ditb   File: TestSeekToBlockWithEncoders.java   Source Code and License 7 votes vote down vote up
/**
 * Test seeking while file is encoded.
 */
@Test
public void testSeekToBlockWithDecreasingCommonPrefix() throws IOException {
  List<KeyValue> sampleKv = new ArrayList<KeyValue>();
  KeyValue kv1 = new KeyValue(Bytes.toBytes("row10aaa"), Bytes.toBytes("f1"),
      Bytes.toBytes("q1"), Bytes.toBytes("val"));
  sampleKv.add(kv1);
  KeyValue kv2 = new KeyValue(Bytes.toBytes("row10aaa"), Bytes.toBytes("f1"),
      Bytes.toBytes("q2"), Bytes.toBytes("val"));
  sampleKv.add(kv2);
  KeyValue kv3 = new KeyValue(Bytes.toBytes("row10aaa"), Bytes.toBytes("f1"),
      Bytes.toBytes("q3"), Bytes.toBytes("val"));
  sampleKv.add(kv3);
  KeyValue kv4 = new KeyValue(Bytes.toBytes("row11baa"), Bytes.toBytes("f1"),
      Bytes.toBytes("q1"), Bytes.toBytes("val"));
  sampleKv.add(kv4);
  KeyValue toSeek = KeyValueUtil.createLastOnRow(kv3.getRowArray(), kv3.getRowOffset(),
      kv3.getRowLength(), null, 0, 0, null, 0, 0);
  seekToTheKey(kv3, sampleKv, toSeek);
}
 
Example 2
Project: ditb   File: ThriftServerRunner.java   Source Code and License 7 votes vote down vote up
@Override
public void deleteAllTs(ByteBuffer tableName,
                        ByteBuffer row,
                        ByteBuffer column,
    long timestamp, Map<ByteBuffer, ByteBuffer> attributes) throws IOError {
  Table table = null;
  try {
    table = getTable(tableName);
    Delete delete  = new Delete(getBytes(row));
    addAttributes(delete, attributes);
    byte [][] famAndQf = KeyValue.parseColumn(getBytes(column));
    if (famAndQf.length == 1) {
      delete.deleteFamily(famAndQf[0], timestamp);
    } else {
      delete.deleteColumns(famAndQf[0], famAndQf[1], timestamp);
    }
    table.delete(delete);

  } catch (IOException e) {
    LOG.warn(e.getMessage(), e);
    throw new IOError(Throwables.getStackTraceAsString(e));
  } finally {
    closeTable(table);
  }
}
 
Example 3
Project: SparkDemo   File: HBaseTest.java   Source Code and License 6 votes vote down vote up
/**
 * show data
 */
public static void getAllRecord(String tableName) {
    try {
        Table table = connection.getTable(TableName.valueOf(tableName));
        Scan s = new Scan();

        ResultScanner ss = table.getScanner(s);
        for (Result r : ss) {
            for (KeyValue kv : r.raw()) {
                System.out.print(new String(kv.getRow()) + " ");
                System.out.print(new String(kv.getFamily()) + ":");
                System.out.print(new String(kv.getQualifier()) + " ");
                System.out.print(kv.getTimestamp() + " ");
                System.out.println(new String(kv.getValue()));
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 
Example 4
Project: ditb   File: Range.java   Source Code and License 6 votes vote down vote up
public static Range[] fromFilter(SingleColumnValueFilter filter) {
  if (!(filter.getComparator() instanceof BinaryComparator)) {
    return new Range[0];
  }

  byte[] column = KeyValue.makeColumn(filter.getFamily(), filter.getQualifier());
  CompareOp compareOp = filter.getOperator();
  byte[] value = filter.getComparator().getValue();

  if (compareOp == CompareOp.NOT_EQUAL) {
    return new Range[] { new Range(column, null, CompareOp.NO_OP, value, CompareOp.LESS),
        new Range(column, value, CompareOp.GREATER, null, CompareOp.NO_OP) };
  } else {
    switch (compareOp) {
    case EQUAL:
    case GREATER_OR_EQUAL:
    case GREATER:
      return new Range[] { new Range(column, value, compareOp, null, CompareOp.NO_OP) };
    case LESS:
    case LESS_OR_EQUAL:
      return new Range[] { new Range(column, null, CompareOp.NO_OP, value, compareOp) };
    default:
      return new Range[0];
    }
  }
}
 
Example 5
Project: HBase-High-Performance-Cookbook   File: HBaseRegularClient.java   Source Code and License 6 votes vote down vote up
/**
 * Getting all records  a row from an existing SS tables 
 * @method getAllRecord
 * @inputParameters hbaseBtable Name used
 * @return type: no return type as its a void method 
 * 
 **/
@SuppressWarnings({ "deprecation", "resource" })
public static void getAllRecord(String myHbaseBtableName) {
  ResultScanner hbaseBSs = null;
  try {
    HTable hbaseBtable = new HTable(hbaseBconf, myHbaseBtableName);
    Scan hbaseBScan = new Scan();
    hbaseBSs = hbaseBtable.getScanner(hbaseBScan);
    for (Result r : hbaseBSs) {
      for (KeyValue hbaseBkv : r.raw()) {
        System.out.print(new String(hbaseBkv.getRow()) + " ");
        System.out.print(new String(hbaseBkv.getFamily()) + ":");
        System.out.print(new String(hbaseBkv.getQualifier()) + " ");
        System.out.print(hbaseBkv.getTimestamp() + " ");
        System.out.println(new String(hbaseBkv.getValue()));
      }
    }
  } catch (IOException eio) {
    eip.printStackTrace();
  } finally {
    if (hbaseBSs != null) hbaseBSs.close();
    // closing the ss hbaseBtable 
  }
}
 
Example 6
Project: ditb   File: TestSeekToBlockWithEncoders.java   Source Code and License 6 votes vote down vote up
@Test
public void testSeekToBlockWithDiffFamilyAndQualifer() throws IOException {
  List<KeyValue> sampleKv = new ArrayList<KeyValue>();
  KeyValue kv1 = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("fam1"), Bytes.toBytes("q1"),
      Bytes.toBytes("val"));
  sampleKv.add(kv1);
  KeyValue kv2 = new KeyValue(Bytes.toBytes("aab"), Bytes.toBytes("fam1"), Bytes.toBytes("q1"),
      Bytes.toBytes("val"));
  sampleKv.add(kv2);
  KeyValue kv4 = new KeyValue(Bytes.toBytes("aac"), Bytes.toBytes("fam1"), Bytes.toBytes("q1"),
      Bytes.toBytes("val"));
  sampleKv.add(kv4);
  KeyValue kv5 = new KeyValue(Bytes.toBytes("aac"), Bytes.toBytes("fam1"), Bytes.toBytes("q2"),
      Bytes.toBytes("val"));
  sampleKv.add(kv5);
  KeyValue toSeek = new KeyValue(Bytes.toBytes("aac"), Bytes.toBytes("fam2"),
      Bytes.toBytes("q2"), Bytes.toBytes("val"));
  seekToTheKey(kv5, sampleKv, toSeek);
}
 
Example 7
Project: ditb   File: TestSeekToBlockWithEncoders.java   Source Code and License 6 votes vote down vote up
@Test
public void testSeekToBlockWithDiffQualiferOnSameRowButDescendingInSize() throws IOException {
  List<KeyValue> sampleKv = new ArrayList<KeyValue>();
  KeyValue kv1 = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("qual1"),
      Bytes.toBytes("val"));
  sampleKv.add(kv1);
  KeyValue kv2 = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("qual2"),
      Bytes.toBytes("val"));
  sampleKv.add(kv2);
  KeyValue kv4 = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("qual3"),
      Bytes.toBytes("val"));
  sampleKv.add(kv4);
  KeyValue kv5 = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("qual4"),
      Bytes.toBytes("val"));
  sampleKv.add(kv5);
  KeyValue kv6 = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("qz"),
      Bytes.toBytes("val"));
  sampleKv.add(kv6);
  KeyValue toSeek = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("qz"),
      Bytes.toBytes("val"));
  seekToTheKey(kv6, sampleKv, toSeek);
}
 
Example 8
Project: ditb   File: BaseIndexScanner.java   Source Code and License 6 votes vote down vote up
public static List<Cell> recoverClusteringResult(List<Cell> cells, byte[] family,
    byte[] qualifier) {
  if (cells == null || cells.size() == 0) return cells;
  byte[][] indexColumn = IndexPutParser.parseIndexRowKey(cells.get(0).getRow());
  List<Cell> list = new ArrayList<>(cells.size() + 1);
  for (Cell cell : cells) {
    byte[] tag = cell.getTagsArray();
    if (tag != null && tag.length > KeyValue.MAX_TAGS_LENGTH) tag = null;
    KeyValue kv =
        new KeyValue(indexColumn[0], CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell),
            cell.getTimestamp(), KeyValue.Type.codeToType(cell.getTypeByte()),
            CellUtil.cloneValue(cell), tag);
    list.add(kv);
  }
  list.add(new KeyValue(indexColumn[0], family, qualifier, indexColumn[1]));
  Collections.sort(list, KeyValue.COMPARATOR);
  return list;
}
 
Example 9
Project: ditb   File: TestReplicationSink.java   Source Code and License 6 votes vote down vote up
/**
 * Insert a mix of puts and deletes
 * @throws Exception
 */
@Test
public void testMixedPutDelete() throws Exception {
  List<WALEntry> entries = new ArrayList<WALEntry>(BATCH_SIZE/2);
  List<Cell> cells = new ArrayList<Cell>();
  for(int i = 0; i < BATCH_SIZE/2; i++) {
    entries.add(createEntry(TABLE_NAME1, i, KeyValue.Type.Put, cells));
  }
  SINK.replicateEntries(entries, CellUtil.createCellScanner(cells));

  entries = new ArrayList<WALEntry>(BATCH_SIZE);
  cells = new ArrayList<Cell>();
  for(int i = 0; i < BATCH_SIZE; i++) {
    entries.add(createEntry(TABLE_NAME1, i,
        i % 2 != 0 ? KeyValue.Type.Put: KeyValue.Type.DeleteColumn, cells));
  }

  SINK.replicateEntries(entries, CellUtil.createCellScanner(cells.iterator()));
  Scan scan = new Scan();
  ResultScanner scanRes = table1.getScanner(scan);
  assertEquals(BATCH_SIZE/2, scanRes.next(BATCH_SIZE).length);
}
 
Example 10
Project: ditb   File: TestScannersWithLabels.java   Source Code and License 6 votes vote down vote up
private static int insertData(TableName tableName, String column, double prob) throws IOException {
  byte[] k = new byte[3];
  byte[][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(column));

  List<Put> puts = new ArrayList<>();
  for (int i = 0; i < 9; i++) {
    Put put = new Put(Bytes.toBytes("row" + i));
    put.setDurability(Durability.SKIP_WAL);
    put.add(famAndQf[0], famAndQf[1], k);
    put.setCellVisibility(new CellVisibility("(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!"
        + TOPSECRET));
    puts.add(put);
  }
  try (Table table = new HTable(TEST_UTIL.getConfiguration(), tableName)) {
    table.put(puts);
  }
  return puts.size();
}
 
Example 11
Project: ditb   File: TestHFileDataBlockEncoder.java   Source Code and License 6 votes vote down vote up
private HFileBlock createBlockOnDisk(List<KeyValue> kvs, HFileBlock block, boolean useTags)
    throws IOException {
  int size;
  HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext(
      blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER,
      block.getHFileContext());

  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  baos.write(block.getDummyHeaderForVersion());
  DataOutputStream dos = new DataOutputStream(baos);
  blockEncoder.startBlockEncoding(context, dos);
  for (KeyValue kv : kvs) {
    blockEncoder.encode(kv, context, dos);
  }
  BufferGrabbingByteArrayOutputStream stream = new BufferGrabbingByteArrayOutputStream();
  baos.writeTo(stream);
  blockEncoder.endBlockEncoding(context, dos, stream.getBuffer(), BlockType.DATA);
  byte[] encodedBytes = baos.toByteArray();
  size = encodedBytes.length - block.getDummyHeaderForVersion().length;
  return new HFileBlock(context.getBlockType(), size, size, -1, ByteBuffer.wrap(encodedBytes),
      HFileBlock.FILL_HEADER, 0, block.getOnDiskDataSizeWithHeader(), block.getHFileContext());
}
 
Example 12
Project: ditb   File: TestCompoundBloomFilter.java   Source Code and License 6 votes vote down vote up
@Test
public void testCreateKey() {
  CompoundBloomFilterBase cbfb = new CompoundBloomFilterBase();
  byte[] row = "myRow".getBytes();
  byte[] qualifier = "myQualifier".getBytes();
  byte[] rowKey = cbfb.createBloomKey(row, 0, row.length,
      row, 0, 0);
  byte[] rowColKey = cbfb.createBloomKey(row, 0, row.length,
      qualifier, 0, qualifier.length);
  KeyValue rowKV = KeyValue.createKeyValueFromKey(rowKey);
  KeyValue rowColKV = KeyValue.createKeyValueFromKey(rowColKey);
  assertEquals(rowKV.getTimestamp(), rowColKV.getTimestamp());
  assertEquals(Bytes.toStringBinary(rowKV.getRow()),
      Bytes.toStringBinary(rowColKV.getRow()));
  assertEquals(0, rowKV.getQualifier().length);
}
 
Example 13
Project: ditb   File: TestStoreFileScannerWithTagCompression.java   Source Code and License 6 votes vote down vote up
private void writeStoreFile(final StoreFile.Writer writer) throws IOException {
  byte[] fam = Bytes.toBytes("f");
  byte[] qualifier = Bytes.toBytes("q");
  long now = System.currentTimeMillis();
  byte[] b = Bytes.toBytes("k1");
  Tag t1 = new Tag((byte) 1, "tag1");
  Tag t2 = new Tag((byte) 2, "tag2");
  Tag t3 = new Tag((byte) 3, "tag3");
  try {
    writer.append(new KeyValue(b, fam, qualifier, now, b, new Tag[] { t1 }));
    b = Bytes.toBytes("k3");
    writer.append(new KeyValue(b, fam, qualifier, now, b, new Tag[] { t2, t1 }));
    b = Bytes.toBytes("k4");
    writer.append(new KeyValue(b, fam, qualifier, now, b, new Tag[] { t3 }));
    b = Bytes.toBytes("k5");
    writer.append(new KeyValue(b, fam, qualifier, now, b, new Tag[] { t3 }));
  } finally {
    writer.close();
  }
}
 
Example 14
Project: ditb   File: GetClosestRowBeforeTracker.java   Source Code and License 6 votes vote down vote up
/**
 * @param c
 * @param kv Presume first on row: i.e. empty column, maximum timestamp and
 * a type of Type.Maximum
 * @param ttl Time to live in ms for this Store
 * @param metaregion True if this is hbase:meta or -ROOT- region.
 */
GetClosestRowBeforeTracker(final KVComparator c, final KeyValue kv,
    final long ttl, final boolean metaregion) {
  super();
  this.metaregion = metaregion;
  this.targetkey = kv;
  // If we are in a metaregion, then our table name is the prefix on the
  // targetkey.
  this.rowoffset = kv.getRowOffset();
  int l = -1;
  if (metaregion) {
    l = KeyValue.getDelimiter(kv.getRowArray(), rowoffset, kv.getRowLength(),
      HConstants.DELIMITER) - this.rowoffset;
  }
  this.tablenamePlusDelimiterLength = metaregion? l + 1: -1;
  this.now = System.currentTimeMillis();
  this.oldestUnexpiredTs = now - ttl;
  this.kvcomparator = c;
  KeyValue.RowOnlyComparator rc = new KeyValue.RowOnlyComparator(this.kvcomparator);
  this.deletes = new TreeMap<KeyValue, NavigableSet<KeyValue>>(rc);
}
 
Example 15
Project: ditb   File: WALEdit.java   Source Code and License 6 votes vote down vote up
@Override
public void write(DataOutput out) throws IOException {
  LOG.warn("WALEdit is being serialized to writable - only expected in test code");
  out.writeInt(VERSION_2);
  out.writeInt(cells.size());
  // We interleave the two lists for code simplicity
  for (Cell cell : cells) {
    // This is not used in any of the core code flows so it is just fine to convert to KV
    KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
    if (compressionContext != null) {
      KeyValueCompression.writeKV(out, kv, compressionContext);
    } else{
      KeyValue.write(kv, out);
    }
  }
  if (scopes == null) {
    out.writeInt(0);
  } else {
    out.writeInt(scopes.size());
    for (byte[] key : scopes.keySet()) {
      Bytes.writeByteArray(out, key);
      out.writeInt(scopes.get(key));
    }
  }
}
 
Example 16
Project: ditb   File: TestHFileOutputFormat.java   Source Code and License 6 votes vote down vote up
/**
 * Write random values to the writer assuming a table created using
 * {@link #FAMILIES} as column family descriptors
 */
private void writeRandomKeyValues(RecordWriter<ImmutableBytesWritable, KeyValue> writer,
    TaskAttemptContext context, Set<byte[]> families, int numRows)
    throws IOException, InterruptedException {
  byte keyBytes[] = new byte[Bytes.SIZEOF_INT];
  int valLength = 10;
  byte valBytes[] = new byte[valLength];

  int taskId = context.getTaskAttemptID().getTaskID().getId();
  assert taskId < Byte.MAX_VALUE : "Unit tests dont support > 127 tasks!";
  final byte [] qualifier = Bytes.toBytes("data");
  Random random = new Random();
  for (int i = 0; i < numRows; i++) {

    Bytes.putInt(keyBytes, 0, i);
    random.nextBytes(valBytes);
    ImmutableBytesWritable key = new ImmutableBytesWritable(keyBytes);

    for (byte[] family : families) {
      KeyValue kv = new KeyValue(keyBytes, family, qualifier, valBytes);
      writer.write(key, kv);
    }
  }
}
 
Example 17
Project: ditb   File: HRegion.java   Source Code and License 6 votes vote down vote up
LCIndexQueryProcessor createLCIndexProcessor(IndexTableRelation indexTableRelation,
    Map<byte[], Store> storeMap, Scan scan, long readPt, KeyValue.KVComparator comparator) {
  try {
    ScanRange.ScanRangeList rangeList = ScanRange.ScanRangeList.getScanRangeList(scan);
    checkSingleScanFamily(scan, rangeList);
    // calculate the primary range for scan
    ScanRange primaryRange =
        selectTheBestRange(indexTableRelation, storeMap, rangeList, scan.getId());
    if (primaryRange != null) {
      printRanges(primaryRange, rangeList);
      return new LCIndexQueryProcessor(scan, indexTableRelation, primaryRange, rangeList,
          (HStore) storeMap.get(primaryRange.getFamily()),
          scan.getFamilyMap().get(primaryRange.getFamily()), readPt, comparator);
    }
  } catch (IOException e) {
    e.printStackTrace();
  }
  return null;
}
 
Example 18
Project: ditb   File: DefaultMemStore.java   Source Code and License 6 votes vote down vote up
/**
 * Constructor.
 * @param c Comparator
 */
public DefaultMemStore(final Configuration conf,
                final KeyValue.KVComparator c) {
  this.conf = conf;
  this.comparator = c;
  this.cellSet = new CellSkipListSet(c);
  this.snapshot = new CellSkipListSet(c);
  timeRangeTracker = new TimeRangeTracker();
  snapshotTimeRangeTracker = new TimeRangeTracker();
  this.size = new AtomicLong(DEEP_OVERHEAD);
  this.snapshotSize = 0;
  if (conf.getBoolean(USEMSLAB_KEY, USEMSLAB_DEFAULT)) {
    String className = conf.get(MSLAB_CLASS_NAME, HeapMemStoreLAB.class.getName());
    this.allocator = ReflectionUtils.instantiateWithCustomCtor(className,
        new Class[] { Configuration.class }, new Object[] { conf });
  } else {
    this.allocator = null;
  }
}
 
Example 19
Project: ditb   File: TestStoreScanner.java   Source Code and License 6 votes vote down vote up
public void testDeleteVersionMaskingMultiplePuts() throws IOException {
  long now = System.currentTimeMillis();
  KeyValue [] kvs1 = new KeyValue[] {
      KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care")
  };
  KeyValue [] kvs2 = new KeyValue[] {
      KeyValueTestUtil.create("R1", "cf", "a", now-500, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "a", now-100, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care")
  };
  List<KeyValueScanner> scanners = scanFixture(kvs1, kvs2);

  StoreScanner scan = new StoreScanner(new Scan(Bytes.toBytes("R1")),
      scanInfo, scanType, getCols("a"), scanners);
  List<Cell> results = new ArrayList<Cell>();
  // the two put at ts=now will be masked by the 1 delete, and
  // since the scan default returns 1 version we'll return the newest
  // key, which is kvs[2], now-100.
  assertEquals(true, scan.next(results));
  assertEquals(1, results.size());
  assertEquals(kvs2[1], results.get(0));
}
 
Example 20
Project: ditb   File: TestCellMessageCodec.java   Source Code and License 6 votes vote down vote up
@Test
public void testOne() throws IOException {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  CountingOutputStream cos = new CountingOutputStream(baos);
  DataOutputStream dos = new DataOutputStream(cos);
  MessageCodec cmc = new MessageCodec();
  Codec.Encoder encoder = cmc.getEncoder(dos);
  final KeyValue kv =
    new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("q"), Bytes.toBytes("v"));
  encoder.write(kv);
  encoder.flush();
  dos.close();
  long offset = cos.getCount();
  CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray()));
  DataInputStream dis = new DataInputStream(cis);
  Codec.Decoder decoder = cmc.getDecoder(dis);
  assertTrue(decoder.advance()); // First read should pull in the KV
  assertFalse(decoder.advance()); // Second read should trip over the end-of-stream  marker and return false
  dis.close();
  assertEquals(offset, cis.getCount());
}
 
Example 21
Project: SparkDemo   File: HBaseTest.java   Source Code and License 5 votes vote down vote up
/**
 * query record
 */
public static void getOneRecord(String tableName, String rowKey)
        throws IOException {
    Table table = connection.getTable(TableName.valueOf(tableName));
    Get get = new Get(rowKey.getBytes());
    Result rs = table.get(get);
    for (KeyValue kv : rs.raw()) {
        System.out.print(new String(kv.getRow()) + " ");
        System.out.print(new String(kv.getFamily()) + ":");
        System.out.print(new String(kv.getQualifier()) + " ");
        System.out.print(kv.getTimestamp() + " ");
        System.out.println(new String(kv.getValue()));
    }
}
 
Example 22
Project: ditb   File: HalfStoreFileReader.java   Source Code and License 5 votes vote down vote up
/**
 * Creates a half file reader for a hfile referred to by an hfilelink.
 * @param fs fileystem to read from
 * @param p path to hfile
 * @param in {@link FSDataInputStreamWrapper}
 * @param size Full size of the hfile file
 * @param cacheConf
 * @param r original reference file (contains top or bottom)
 * @param conf Configuration
 * @throws IOException
 */
public HalfStoreFileReader(final FileSystem fs, final Path p, final FSDataInputStreamWrapper in,
    long size, final CacheConfig cacheConf,  final Reference r, final Configuration conf)
    throws IOException {
  super(fs, p, in, size, cacheConf, conf);
  // This is not actual midkey for this half-file; its just border
  // around which we split top and bottom.  Have to look in files to find
  // actual last and first keys for bottom and top halves.  Half-files don't
  // have an actual midkey themselves. No midkey is how we indicate file is
  // not splittable.
  this.splitkey = r.getSplitKey();
  this.splitCell = new KeyValue.KeyOnlyKeyValue(this.splitkey, 0, this.splitkey.length);
  // Is it top or bottom half?
  this.top = Reference.isTopFileRegion(r.getFileRegion());
}
 
Example 23
Project: ditb   File: TestGroupingTableMapper.java   Source Code and License 5 votes vote down vote up
/**
 * Test GroupingTableMapper class
 */
@Test
public void testGroupingTableMapper() throws Exception {

  GroupingTableMapper mapper = new GroupingTableMapper();
  Configuration configuration = new Configuration();
  configuration.set(GroupingTableMapper.GROUP_COLUMNS, "family1:clm family2:clm");
  mapper.setConf(configuration);

  Result result = mock(Result.class);
  @SuppressWarnings("unchecked")
  Mapper<ImmutableBytesWritable, Result, ImmutableBytesWritable, Result>.Context context =
      mock(Mapper.Context.class);
  context.write(any(ImmutableBytesWritable.class), any(Result.class));
  List<Cell> keyValue = new ArrayList<Cell>();
  byte[] row = {};
  keyValue.add(new KeyValue(row, Bytes.toBytes("family2"), Bytes.toBytes("clm"), Bytes
      .toBytes("value1")));
  keyValue.add(new KeyValue(row, Bytes.toBytes("family1"), Bytes.toBytes("clm"), Bytes
      .toBytes("value2")));
  when(result.listCells()).thenReturn(keyValue);
  mapper.map(null, result, context);
  // template data
  byte[][] data = { Bytes.toBytes("value1"), Bytes.toBytes("value2") };
  ImmutableBytesWritable ibw = mapper.createGroupKey(data);
  verify(context).write(ibw, result);
}
 
Example 24
Project: ditb   File: TestRowEncoder.java   Source Code and License 5 votes vote down vote up
@Test
public void testForwardScanner() {
  int counter = -1;
  while (searcher.advance()) {
    ++counter;
    KeyValue inputKv = rows.getInputs().get(counter);
    KeyValue outputKv = KeyValueUtil.copyToNewKeyValue(searcher.current());
    assertKeyAndValueEqual(inputKv, outputKv);
  }
  // assert same number of cells
  Assert.assertEquals(rows.getInputs().size(), counter + 1);
}
 
Example 25
Project: ditb   File: IncrementCoalescer.java   Source Code and License 5 votes vote down vote up
private boolean internalQueueTincrement(TIncrement inc) throws TException {
  byte[][] famAndQf = KeyValue.parseColumn(inc.getColumn());
  if (famAndQf.length != 2) return false;

  return internalQueueIncrement(inc.getTable(), inc.getRow(), famAndQf[0], famAndQf[1],
    inc.getAmmount());
}
 
Example 26
Project: ditb   File: TsvImporterCustomTestMapper.java   Source Code and License 5 votes vote down vote up
/**
 * Convert a line of TSV text into an HBase table row after transforming the
 * values by multiplying them by 3.
 */
@Override
public void map(LongWritable offset, Text value, Context context)
      throws IOException {
  byte[] family = Bytes.toBytes("FAM");
  final byte[][] qualifiers = { Bytes.toBytes("A"), Bytes.toBytes("B") };

  // do some basic line parsing
  byte[] lineBytes = value.getBytes();
  String[] valueTokens = new String(lineBytes, "UTF-8").split("\u001b");

  // create the rowKey and Put
  ImmutableBytesWritable rowKey =
    new ImmutableBytesWritable(Bytes.toBytes(valueTokens[0]));
  Put put = new Put(rowKey.copyBytes());
  put.setDurability(Durability.SKIP_WAL);

  //The value should look like this: VALUE1 or VALUE2. Let's multiply
  //the integer by 3
  for(int i = 1; i < valueTokens.length; i++) {
    String prefix = valueTokens[i].substring(0, "VALUE".length());
    String suffix = valueTokens[i].substring("VALUE".length());
    String newValue = prefix + Integer.parseInt(suffix) * 3;

    KeyValue kv = new KeyValue(rowKey.copyBytes(), family,
        qualifiers[i-1], Bytes.toBytes(newValue));
    put.add(kv);
  }

  try {
    context.write(rowKey, put);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }
}
 
Example 27
Project: ditb   File: TestInvocationRecordFilter.java   Source Code and License 5 votes vote down vote up
public void verifyInvocationResults(Integer[] selectQualifiers,
    Integer[] expectedQualifiers) throws Exception {
  Get get = new Get(ROW_BYTES);
  for (int i = 0; i < selectQualifiers.length; i++) {
    get.addColumn(FAMILY_NAME_BYTES,
        Bytes.toBytes(QUALIFIER_PREFIX + selectQualifiers[i]));
  }

  get.setFilter(new InvocationRecordFilter());

  List<KeyValue> expectedValues = new ArrayList<KeyValue>();
  for (int i = 0; i < expectedQualifiers.length; i++) {
    expectedValues.add(new KeyValue(ROW_BYTES, FAMILY_NAME_BYTES, Bytes
        .toBytes(QUALIFIER_PREFIX + expectedQualifiers[i]),
        expectedQualifiers[i], Bytes.toBytes(VALUE_PREFIX
            + expectedQualifiers[i])));
  }

  Scan scan = new Scan(get);
  List<Cell> actualValues = new ArrayList<Cell>();
  List<Cell> temp = new ArrayList<Cell>();
  InternalScanner scanner = this.region.getScanner(scan);
  while (scanner.next(temp)) {
    actualValues.addAll(temp);
    temp.clear();
  }
  actualValues.addAll(temp);
  Assert.assertTrue("Actual values " + actualValues
      + " differ from the expected values:" + expectedValues,
      expectedValues.equals(actualValues));
}
 
Example 28
Project: ditb   File: TestResult.java   Source Code and License 5 votes vote down vote up
public void testBasicGetValue() throws Exception {
  KeyValue [] kvs = genKVs(row, family, value, 1, 100);

  Arrays.sort(kvs, KeyValue.COMPARATOR);

  Result r = Result.create(kvs);

  for (int i = 0; i < 100; ++i) {
    final byte[] qf = Bytes.toBytes(i);

    assertByteEquals(Bytes.add(value, Bytes.toBytes(i)), r.getValue(family, qf));
    assertTrue(r.containsColumn(family, qf));
  }
}
 
Example 29
Project: ditb   File: FilterWrapper.java   Source Code and License 5 votes vote down vote up
/**
 * This method is deprecated and you should override Cell getNextKeyHint(Cell) instead.
 */
@Override
@Deprecated
public KeyValue getNextKeyHint(KeyValue currentKV) throws IOException {
  // This will never get called.
  return KeyValueUtil.ensureKeyValue(this.filter.getNextCellHint((Cell)currentKV));
}
 
Example 30
Project: ditb   File: TestScannersFromClientSide.java   Source Code and License 5 votes vote down vote up
@Test
public void testSmallScan() throws Exception {
  TableName TABLE = TableName.valueOf("testSmallScan");

  int numRows = 10;
  byte[][] ROWS = HTestConst.makeNAscii(ROW, numRows);

  int numQualifiers = 10;
  byte[][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, numQualifiers);

  Table ht = TEST_UTIL.createTable(TABLE, FAMILY);

  Put put;
  List<Put> puts = new ArrayList<Put>();
  for (int row = 0; row < ROWS.length; row++) {
    put = new Put(ROWS[row]);
    for (int qual = 0; qual < QUALIFIERS.length; qual++) {
      KeyValue kv = new KeyValue(ROWS[row], FAMILY, QUALIFIERS[qual], VALUE);
      put.add(kv);
    }
    puts.add(put);
  }
  ht.put(puts);

  int expectedRows = numRows;
  int expectedCols = numRows * numQualifiers;

  // Test normal and reversed
  testSmallScan(ht, true, expectedRows, expectedCols);
  testSmallScan(ht, false, expectedRows, expectedCols);
}
 
Example 31
Project: ditb   File: Put.java   Source Code and License 5 votes vote down vote up
/**
 * Add the specified column and value, with the specified timestamp as
 * its version to this Put operation.
 * @param family family name
 * @param qualifier column qualifier
 * @param ts version timestamp
 * @param value column value
 * @return this
 */
public Put addColumn(byte [] family, byte [] qualifier, long ts, byte [] value) {
  if (ts < 0) {
    throw new IllegalArgumentException("Timestamp cannot be negative. ts=" + ts);
  }
  List<Cell> list = getCellList(family);
  KeyValue kv = createPutKeyValue(family, qualifier, ts, value);
  list.add(kv);
  familyMap.put(CellUtil.cloneFamily(kv), list);
  return this;
}
 
Example 32
Project: ditb   File: TestStoreScanner.java   Source Code and License 5 votes vote down vote up
public void testDeleteFamily() throws IOException {
  KeyValue [] kvs = new KeyValue[] {
      KeyValueTestUtil.create("R1", "cf", "a", 100, KeyValue.Type.DeleteFamily, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "e", 11, KeyValue.Type.DeleteColumn, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "g", 11, KeyValue.Type.Delete, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
  };
  List<KeyValueScanner> scanners = scanFixture(kvs);
  StoreScanner scan = new StoreScanner(
      new Scan().setMaxVersions(Integer.MAX_VALUE), scanInfo, scanType, null,
      scanners);
  List<Cell> results = new ArrayList<Cell>();
  assertEquals(true, scan.next(results));
  assertEquals(0, results.size());
  assertEquals(true, scan.next(results));
  assertEquals(1, results.size());
  assertEquals(kvs[kvs.length-1], results.get(0));

  assertEquals(false, scan.next(results));
}
 
Example 33
Project: ditb   File: TestPrefetch.java   Source Code and License 5 votes vote down vote up
public static KeyValue.Type generateKeyType(Random rand) {
  if (rand.nextBoolean()) {
    // Let's make half of KVs puts.
    return KeyValue.Type.Put;
  } else {
    KeyValue.Type keyType =
        KeyValue.Type.values()[1 + rand.nextInt(NUM_VALID_KEY_TYPES)];
    if (keyType == KeyValue.Type.Minimum || keyType == KeyValue.Type.Maximum)
    {
      throw new RuntimeException("Generated an invalid key type: " + keyType
          + ". " + "Probably the layout of KeyValue.Type has changed.");
    }
    return keyType;
  }
}
 
Example 34
Project: ditb   File: TestReplicationSource.java   Source Code and License 5 votes vote down vote up
/**
 * Sanity check that we can move logs around while we are reading
 * from them. Should this test fail, ReplicationSource would have a hard
 * time reading logs that are being archived.
 * @throws Exception
 */
@Test
public void testLogMoving() throws Exception{
  Path logPath = new Path(logDir, "log");
  if (!FS.exists(logDir)) FS.mkdirs(logDir);
  if (!FS.exists(oldLogDir)) FS.mkdirs(oldLogDir);
  WALProvider.Writer writer = WALFactory.createWALWriter(FS, logPath,
      TEST_UTIL.getConfiguration());
  for(int i = 0; i < 3; i++) {
    byte[] b = Bytes.toBytes(Integer.toString(i));
    KeyValue kv = new KeyValue(b,b,b);
    WALEdit edit = new WALEdit();
    edit.add(kv);
    WALKey key = new WALKey(b, TableName.valueOf(b), 0, 0,
        HConstants.DEFAULT_CLUSTER_ID);
    writer.append(new WAL.Entry(key, edit));
    writer.sync();
  }
  writer.close();

  WAL.Reader reader = WALFactory.createReader(FS, logPath, TEST_UTIL.getConfiguration());
  WAL.Entry entry = reader.next();
  assertNotNull(entry);

  Path oldLogPath = new Path(oldLogDir, "log");
  FS.rename(logPath, oldLogPath);

  entry = reader.next();
  assertNotNull(entry);

  entry = reader.next();
  entry = reader.next();

  assertNull(entry);
  reader.close();
}
 
Example 35
Project: ditb   File: TestRowEncoder.java   Source Code and License 5 votes vote down vote up
/**
 * probably not needed since testReverseScannerWithJitter() below is more thorough
 */
@Test
public void testReverseScanner() {
  searcher.positionAfterLastCell();
  int counter = -1;
  while (searcher.previous()) {
    ++counter;
    int oppositeIndex = rows.getInputs().size() - counter - 1;
    KeyValue inputKv = rows.getInputs().get(oppositeIndex);
    KeyValue outputKv = KeyValueUtil.copyToNewKeyValue(searcher.current());
    assertKeyAndValueEqual(inputKv, outputKv);
  }
  Assert.assertEquals(rows.getInputs().size(), counter + 1);
}
 
Example 36
Project: ditb   File: CollectionBackedScanner.java   Source Code and License 5 votes vote down vote up
public CollectionBackedScanner(KeyValue.KVComparator comparator,
    Cell... array) {
  this.comparator = comparator;

  List<Cell> tmp = new ArrayList<Cell>(array.length);
  Collections.addAll(tmp, array);
  Collections.sort(tmp, comparator);
  data = tmp;
  init();
}
 
Example 37
Project: ditb   File: IndexSpecification.java   Source Code and License 5 votes vote down vote up
/**
 * Set column for this index.
 *
 * @param indexColumn
 */
public void setIndexColumn(byte[] indexColumn) {
  this.indexColumn = indexColumn;
  byte[][] fq = KeyValue.parseColumn(this.indexColumn);
  this.family = fq[0];
  this.qualifier = fq[1];
  this.indexId = Bytes.add(this.family, Bytes.toBytes("_"), this.qualifier);
}
 
Example 38
Project: ditb   File: Delete.java   Source Code and License 5 votes vote down vote up
/**
 * Delete the specified version of the specified column.
 * @param family family name
 * @param qualifier column qualifier
 * @param timestamp version timestamp
 * @return this for invocation chaining
 */
public Delete addColumn(byte [] family, byte [] qualifier, long timestamp) {
  if (timestamp < 0) {
    throw new IllegalArgumentException("Timestamp cannot be negative. ts=" + timestamp);
  }
  List<Cell> list = familyMap.get(family);
  if(list == null) {
    list = new ArrayList<Cell>();
  }
  KeyValue kv = new KeyValue(this.row, family, qualifier, timestamp, KeyValue.Type.Delete);
  list.add(kv);
  familyMap.put(family, list);
  return this;
}
 
Example 39
Project: ditb   File: HStore.java   Source Code and License 5 votes vote down vote up
private boolean seekToScanner(final HFileScanner scanner, final KeyValue firstOnRow,
    final KeyValue firstKV) throws IOException {
  KeyValue kv = firstOnRow;
  // If firstOnRow < firstKV, set to firstKV
  if (this.comparator.compareRows(firstKV, firstOnRow) == 0) kv = firstKV;
  int result = scanner.seekTo(kv);
  return result != -1;
}
 
Example 40
Project: ditb   File: TestCellMessageCodec.java   Source Code and License 5 votes vote down vote up
@Test
public void testThree() throws IOException {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  CountingOutputStream cos = new CountingOutputStream(baos);
  DataOutputStream dos = new DataOutputStream(cos);
  MessageCodec cmc = new MessageCodec();
  Codec.Encoder encoder = cmc.getEncoder(dos);
  final KeyValue kv1 = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("1"), Bytes.toBytes("1"));
  final KeyValue kv2 = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("2"), Bytes.toBytes("2"));
  final KeyValue kv3 = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("3"), Bytes.toBytes("3"));
  encoder.write(kv1);
  encoder.write(kv2);
  encoder.write(kv3);
  encoder.flush();
  dos.close();
  long offset = cos.getCount();
  CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray()));
  DataInputStream dis = new DataInputStream(cis);
  Codec.Decoder decoder = cmc.getDecoder(dis);
  assertTrue(decoder.advance());
  Cell c = decoder.current();
  assertTrue(CellComparator.equals(c, kv1));
  assertTrue(decoder.advance());
  c = decoder.current();
  assertTrue(CellComparator.equals(c, kv2));
  assertTrue(decoder.advance());
  c = decoder.current();
  assertTrue(CellComparator.equals(c, kv3));
  assertFalse(decoder.advance());
  dis.close();
  assertEquals(offset, cis.getCount());
}
 
Example 41
Project: ditb   File: TestDefaultMemStore.java   Source Code and License 5 votes vote down vote up
public void testPutSameKey() {
  byte [] bytes = Bytes.toBytes(getName());
  KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes);
  this.memstore.add(kv);
  byte [] other = Bytes.toBytes("somethingelse");
  KeyValue samekey = new KeyValue(bytes, bytes, bytes, other);
  this.memstore.add(samekey);
  Cell found = this.memstore.cellSet.first();
  assertEquals(1, this.memstore.cellSet.size());
  assertTrue(Bytes.toString(found.getValue()), CellUtil.matchingValue(samekey, found));
}
 
Example 42
Project: ditb   File: TestScannerResource.java   Source Code and License 5 votes vote down vote up
static int insertData(Configuration conf, TableName tableName, String column, double prob)
    throws IOException {
  Random rng = new Random();
  byte[] k = new byte[3];
  byte [][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(column));
  List<Put> puts = new ArrayList<>();
  for (byte b1 = 'a'; b1 < 'z'; b1++) {
    for (byte b2 = 'a'; b2 < 'z'; b2++) {
      for (byte b3 = 'a'; b3 < 'z'; b3++) {
        if (rng.nextDouble() < prob) {
          k[0] = b1;
          k[1] = b2;
          k[2] = b3;
          Put put = new Put(k);
          put.setDurability(Durability.SKIP_WAL);
          put.add(famAndQf[0], famAndQf[1], k);
          puts.add(put);
        }
      }
    }
  }
  try (Connection conn = ConnectionFactory.createConnection(conf);
      Table table = conn.getTable(tableName)) {
    table.put(puts);
  }
  return puts.size();
}
 
Example 43
Project: ditb   File: TestDefaultMemStore.java   Source Code and License 5 votes vote down vote up
/**
 * Tests that the timeOfOldestEdit is updated correctly for the 
 * various edit operations in memstore.
 * @throws Exception
 */
public void testUpdateToTimeOfOldestEdit() throws Exception {
  try {
    EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
    EnvironmentEdgeManager.injectEdge(edge);
    DefaultMemStore memstore = new DefaultMemStore();
    long t = memstore.timeOfOldestEdit();
    assertEquals(t, Long.MAX_VALUE);

    // test the case that the timeOfOldestEdit is updated after a KV add
    memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
    t = memstore.timeOfOldestEdit();
    assertTrue(t == 1234);
    // snapshot() will reset timeOfOldestEdit. The method will also assert the 
    // value is reset to Long.MAX_VALUE
    t = runSnapshot(memstore);

    // test the case that the timeOfOldestEdit is updated after a KV delete
    memstore.delete(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
    t = memstore.timeOfOldestEdit();
    assertTrue(t == 1234);
    t = runSnapshot(memstore);

    // test the case that the timeOfOldestEdit is updated after a KV upsert
    List<Cell> l = new ArrayList<Cell>();
    KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
    kv1.setSequenceId(100);
    l.add(kv1);
    memstore.upsert(l, 1000);
    t = memstore.timeOfOldestEdit();
    assertTrue(t == 1234);
  } finally {
    EnvironmentEdgeManager.reset();
  }
}
 
Example 44
Project: ditb   File: WALEdit.java   Source Code and License 5 votes vote down vote up
/**
 * Create a bulk loader WALEdit
 *
 * @param hri                The HRegionInfo for the region in which we are bulk loading
 * @param bulkLoadDescriptor The descriptor for the Bulk Loader
 * @return The WALEdit for the BulkLoad
 */
public static WALEdit createBulkLoadEvent(HRegionInfo hri,
                                          WALProtos.BulkLoadDescriptor bulkLoadDescriptor) {
  KeyValue kv = new KeyValue(getRowForRegion(hri),
      METAFAMILY,
      BULK_LOAD,
      EnvironmentEdgeManager.currentTime(),
      bulkLoadDescriptor.toByteArray());
  return new WALEdit().add(kv);
}
 
Example 45
Project: ditb   File: KeyValueCompression.java   Source Code and License 5 votes vote down vote up
/**
 * Compresses and writes ourKV to out, a DataOutput.
 * 
 * @param out the DataOutput
 * @param keyVal the KV to compress and write
 * @param writeContext the compressionContext to use.
 * @throws IOException
 */
public static void writeKV(final DataOutput out, KeyValue keyVal,
    CompressionContext writeContext) throws IOException {
  byte[] backingArray = keyVal.getBuffer();
  int offset = keyVal.getOffset();

  // we first write the KeyValue infrastructure as VInts.
  WritableUtils.writeVInt(out, keyVal.getKeyLength());
  WritableUtils.writeVInt(out, keyVal.getValueLength());
  WritableUtils.writeVInt(out, keyVal.getTagsLength());

  // now we write the row key, as the row key is likely to be repeated
  // We save space only if we attempt to compress elements with duplicates
  Compressor.writeCompressed(keyVal.getBuffer(), keyVal.getRowOffset(),
      keyVal.getRowLength(), out, writeContext.rowDict);


  // now family, if it exists. if it doesn't, we write a 0 length array.
  Compressor.writeCompressed(keyVal.getBuffer(), keyVal.getFamilyOffset(),
      keyVal.getFamilyLength(), out, writeContext.familyDict);

  // qualifier next
  Compressor.writeCompressed(keyVal.getBuffer(), keyVal.getQualifierOffset(),
      keyVal.getQualifierLength(), out,
      writeContext.qualifierDict);

  // now we write the rest uncompressed
  int pos = keyVal.getTimestampOffset();
  int remainingLength = keyVal.getLength() + offset - (pos);
  out.write(backingArray, pos, remainingLength);
}
 
Example 46
Project: ditb   File: ReversedStoreScanner.java   Source Code and License 5 votes vote down vote up
@Override
protected void checkScanOrder(Cell prevKV, Cell kv,
    KeyValue.KVComparator comparator) throws IOException {
  // Check that the heap gives us KVs in an increasing order for same row and
  // decreasing order for different rows.
  assert prevKV == null || comparator == null || comparator.compareRows(kv, prevKV) < 0
      || (comparator.matchingRows(kv, prevKV) && comparator.compare(kv,
          prevKV) >= 0) : "Key " + prevKV
      + " followed by a " + "error order key " + kv + " in cf " + store
      + " in reversed scan";
}
 
Example 47
Project: ditb   File: TestReadOldRootAndMetaEdits.java   Source Code and License 5 votes vote down vote up
/**
 * Creates a WALEdit for the passed KeyValues and returns a WALProvider.Entry instance composed of
 * the WALEdit and passed WALKey.
 * @return WAL.Entry instance for the passed WALKey and KeyValues
 */
private WAL.Entry createAEntry(WALKey walKey, List<KeyValue> kvs) {
  WALEdit edit = new WALEdit();
  for (KeyValue kv : kvs )
  edit.add(kv);
  return new WAL.Entry(walKey, edit);
}
 
Example 48
Project: ditb   File: TestTags.java   Source Code and License 5 votes vote down vote up
private void updateMutationAddingTags(final Mutation m) {
  byte[] attribute = m.getAttribute("visibility");
  byte[] cf = null;
  List<Cell> updatedCells = new ArrayList<Cell>();
  if (attribute != null) {
    for (List<? extends Cell> edits : m.getFamilyCellMap().values()) {
      for (Cell cell : edits) {
        KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
        if (cf == null) {
          cf = kv.getFamily();
        }
        Tag tag = new Tag((byte) 1, attribute);
        List<Tag> tagList = new ArrayList<Tag>();
        tagList.add(tag);

        KeyValue newKV = new KeyValue(kv.getRow(), 0, kv.getRowLength(), kv.getFamily(), 0,
            kv.getFamilyLength(), kv.getQualifier(), 0, kv.getQualifierLength(),
            kv.getTimestamp(), KeyValue.Type.codeToType(kv.getType()), kv.getValue(), 0,
            kv.getValueLength(), tagList);
        ((List<Cell>) updatedCells).add(newKV);
      }
    }
    m.getFamilyCellMap().remove(cf);
    // Update the family map
    m.getFamilyCellMap().put(cf, updatedCells);
  }
}
 
Example 49
Project: ditb   File: TestFilter.java   Source Code and License 5 votes vote down vote up
@Test
public void testFirstKeyOnlyFilter() throws IOException {
  Scan s = new Scan();
  s.setFilter(new FirstKeyOnlyFilter());
  // Expected KVs, the first KV from each of the remaining 6 rows
  KeyValue [] kvs = {
      new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
      new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
      new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1])
  };
  verifyScanFull(s, kvs);
}
 
Example 50
Project: ditb   File: TestKeyValueScanFixture.java   Source Code and License 5 votes vote down vote up
public void testKeyValueScanFixture() throws IOException {
  KeyValue kvs[] = new KeyValue[]{
      KeyValueTestUtil.create("RowA", "family", "qf1",
          1, KeyValue.Type.Put, "value-1"),
      KeyValueTestUtil.create("RowA", "family", "qf2",
          1, KeyValue.Type.Put, "value-2"),
      KeyValueTestUtil.create("RowB", "family", "qf1",
          10, KeyValue.Type.Put, "value-10")
  };
  KeyValueScanner scan = new KeyValueScanFixture(
      KeyValue.COMPARATOR, kvs);

  KeyValue kv = KeyValueUtil.createFirstOnRow(Bytes.toBytes("RowA"));
  // should seek to this:
  assertTrue(scan.seek(kv));
  Cell res = scan.peek();
  assertEquals(kvs[0], res);

  kv = KeyValueUtil.createFirstOnRow(Bytes.toBytes("RowB"));
  assertTrue(scan.seek(kv));
  res = scan.peek();
  assertEquals(kvs[2], res);

  // ensure we pull things out properly:
  kv = KeyValueUtil.createFirstOnRow(Bytes.toBytes("RowA"));
  assertTrue(scan.seek(kv));
  assertEquals(kvs[0], scan.peek());
  assertEquals(kvs[0], scan.next());
  assertEquals(kvs[1], scan.peek());
  assertEquals(kvs[1], scan.next());
  assertEquals(kvs[2], scan.peek());
  assertEquals(kvs[2], scan.next());
  assertEquals(null, scan.peek());
  assertEquals(null, scan.next());
}
 
Example 51
Project: ditb   File: TestDefaultMemStore.java   Source Code and License 5 votes vote down vote up
/**
 * Add keyvalues with a fixed memstoreTs, and checks that memstore size is decreased
 * as older keyvalues are deleted from the memstore.
 * @throws Exception
 */
public void testUpsertMemstoreSize() throws Exception {
  Configuration conf = HBaseConfiguration.create();
  memstore = new DefaultMemStore(conf, KeyValue.COMPARATOR);
  long oldSize = memstore.size.get();

  List<Cell> l = new ArrayList<Cell>();
  KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
  KeyValue kv2 = KeyValueTestUtil.create("r", "f", "q", 101, "v");
  KeyValue kv3 = KeyValueTestUtil.create("r", "f", "q", 102, "v");

  kv1.setSequenceId(1); kv2.setSequenceId(1);kv3.setSequenceId(1);
  l.add(kv1); l.add(kv2); l.add(kv3);

  this.memstore.upsert(l, 2);// readpoint is 2
  long newSize = this.memstore.size.get();
  assert(newSize > oldSize);
  //The kv1 should be removed.
  assert(memstore.cellSet.size() == 2);
  
  KeyValue kv4 = KeyValueTestUtil.create("r", "f", "q", 104, "v");
  kv4.setSequenceId(1);
  l.clear(); l.add(kv4);
  this.memstore.upsert(l, 3);
  assertEquals(newSize, this.memstore.size.get());
  //The kv2 should be removed.
  assert(memstore.cellSet.size() == 2);
  //this.memstore = null;
}
 
Example 52
Project: ditb   File: Increment.java   Source Code and License 5 votes vote down vote up
/**
 * Increment the column from the specific family with the specified qualifier
 * by the specified amount.
 * <p>
 * Overrides previous calls to addColumn for this family and qualifier.
 * @param family family name
 * @param qualifier column qualifier
 * @param amount amount to increment by
 * @return the Increment object
 */
public Increment addColumn(byte [] family, byte [] qualifier, long amount) {
  if (family == null) {
    throw new IllegalArgumentException("family cannot be null");
  }
  if (qualifier == null) {
    throw new IllegalArgumentException("qualifier cannot be null");
  }
  List<Cell> list = getCellList(family);
  KeyValue kv = createPutKeyValue(family, qualifier, ts, Bytes.toBytes(amount));
  list.add(kv);
  familyMap.put(CellUtil.cloneFamily(kv), list);
  return this;
}
 
Example 53
Project: ditb   File: TestDependentColumnFilter.java   Source Code and License 5 votes vote down vote up
private List<KeyValue> makeTestVals() {
  List<KeyValue> testVals = new ArrayList<KeyValue>();
  testVals.add(new KeyValue(ROWS[0], FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]));
  testVals.add(new KeyValue(ROWS[0], FAMILIES[0], QUALIFIER, STAMPS[1], BAD_VALS[1]));
  testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[1], BAD_VALS[2]));
  testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[0], MATCH_VAL));
  testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]));

  return testVals;
}
 
Example 54
Project: ditb   File: TestDefaultMemStore.java   Source Code and License 5 votes vote down vote up
public void testGetWithDelete() throws IOException {
  byte [] row = Bytes.toBytes("testrow");
  byte [] fam = Bytes.toBytes("testfamily");
  byte [] qf1 = Bytes.toBytes("testqualifier");
  byte [] val = Bytes.toBytes("testval");

  long ts1 = System.nanoTime();
  KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
  long ts2 = ts1 + 1;
  KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
  long ts3 = ts2 +1;
  KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
  memstore.add(put1);
  memstore.add(put2);
  memstore.add(put3);

  assertEquals(3, memstore.cellSet.size());

  KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.Delete, val);
  memstore.delete(del2);

  List<Cell> expected = new ArrayList<Cell>();
  expected.add(put3);
  expected.add(del2);
  expected.add(put2);
  expected.add(put1);

  assertEquals(4, memstore.cellSet.size());
  int i = 0;
  for(Cell cell : memstore.cellSet) {
    assertEquals(expected.get(i++), cell);
  }
}
 
Example 55
Project: ditb   File: Put.java   Source Code and License 5 votes vote down vote up
/**
 * See {@link #add(byte[], byte[], long, byte[])}. This version expects
 * that the underlying arrays won't change. It's intended
 * for usage internal HBase to and for advanced client applications.
 */
public Put addImmutable(byte [] family, byte [] qualifier, long ts, byte [] value) {
  if (ts < 0) {
    throw new IllegalArgumentException("Timestamp cannot be negative. ts=" + ts);
  }
  List<Cell> list = getCellList(family);
  KeyValue kv = createPutKeyValue(family, qualifier, ts, value);
  list.add(kv);
  familyMap.put(family, list);
  return this;
}
 
Example 56
Project: ditb   File: TestWALReaderOnSecureWAL.java   Source Code and License 5 votes vote down vote up
private Path writeWAL(final WALFactory wals, final String tblName) throws IOException {
  Configuration conf = TEST_UTIL.getConfiguration();
  String clsName = conf.get(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName());
  conf.setClass(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, SecureWALCellCodec.class,
    WALCellCodec.class);
  try {
    TableName tableName = TableName.valueOf(tblName);
    HTableDescriptor htd = new HTableDescriptor(tableName);
    htd.addFamily(new HColumnDescriptor(tableName.getName()));
    HRegionInfo regioninfo = new HRegionInfo(tableName,
      HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false);
    final int total = 10;
    final byte[] row = Bytes.toBytes("row");
    final byte[] family = Bytes.toBytes("family");
    final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);

    // Write the WAL
    WAL wal = wals.getWAL(regioninfo.getEncodedNameAsBytes());
    for (int i = 0; i < total; i++) {
      WALEdit kvs = new WALEdit();
      kvs.add(new KeyValue(row, family, Bytes.toBytes(i), value));
      wal.append(htd, regioninfo, new WALKey(regioninfo.getEncodedNameAsBytes(), tableName,
          System.currentTimeMillis(), mvcc), kvs, true);
    }
    wal.sync();
    final Path walPath = DefaultWALProvider.getCurrentFileName(wal);
    wal.shutdown();
    
    return walPath;
  } finally {
    // restore the cell codec class
    conf.set(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, clsName);
  }
}
 
Example 57
Project: ditb   File: LMDIndexDirectStoreFileScanner.java   Source Code and License 5 votes vote down vote up
private List<byte[]> getRawRowkeyList(StoreFileScanner secondaryScanner,
    List<LMDBucket> bucketList, MDRange[] ranges) throws IOException {
  List<byte[]> rowkeyList = new ArrayList<>();
  for (LMDBucket bucket : bucketList) {
    Cell peekCell = secondaryScanner.peek();
    if (peekCell != null && Bytes.compareTo(bucket.getStartKey(), peekCell.getRow()) == 0) {
    } else {
      secondaryScanner.reseek(new KeyValue(bucket.getStartKey(), LMDIndexConstants.FAMILY,
          LMDIndexConstants.QUALIFIER));
    }
    Cell cell;
    while ((cell = secondaryScanner.peek()) != null) {
      if (Bytes.compareTo(bucket.getStopKey(), cell.getRow()) < 0) {
        break;
      }
      boolean included = true;
      int[] values = MDUtils.bitwiseUnzip(cell.getRow(), ranges.length);
      for (int i = 0; i < ranges.length; i++) {
        if (!ranges[i].include(values[i])) {
          included = false;
          break;
        }
      }
      if (included) {
        //          System.out.println("adding key: " + Bytes.toInt(cell.getQualifier()));
        rowkeyList.add(cell.getQualifier());
        secondaryScanner.next();
      } else {
        //          System.out.println("skipped key: " + Bytes.toInt(cell.getQualifier()));
        secondaryScanner.reseek(
            new KeyValue(cell.getRow(), LMDIndexConstants.FAMILY, LMDIndexConstants.QUALIFIER));
      }
    }
  }
  return rowkeyList;
}
 
Example 58
Project: ditb   File: LMDIndexDirectStoreFileScanner.java   Source Code and License 5 votes vote down vote up
@Override public boolean seek(Cell key) throws IOException {
  for (int i = rawRowkeyListIndex; i < rawRowkeyList.size(); i++) {
    int cmp = KeyValue.COMPARATOR.compare(key, new KeyValue(rawRowkeyList.get(i), null, null));
    if (cmp <= 0) {
      rawRowkeyListIndex = i;
      rawDataScanner.seek(new KeyValue(rawRowkeyList.get(rawRowkeyListIndex++), null, null));
      setCurrentCell(rawDataScanner.peek());
      return true;
    }
  }
  System.out.println("return false at rawRowkeyListIndex=" + rawRowkeyListIndex);
  setCurrentCell(null);
  return false;
}
 
Example 59
Project: ditb   File: LMDIndexDirectStoreFileScanner.java   Source Code and License 5 votes vote down vote up
public void dowhat() throws IOException {
  if (rawRowkeyListIndex < rawRowkeyList.size()) {
    rawDataScanner.seek(new KeyValue(rawRowkeyList.get(rawRowkeyListIndex), null, null));
    setCurrentCell(rawDataScanner.peek());
  } else {
    setCurrentCell(null);
  }
}
 
Example 60
Project: ditb   File: IntegrationTestBulkLoad.java   Source Code and License 5 votes vote down vote up
@Override
protected void map(LongWritable key, LongWritable value, Context context)
    throws IOException, InterruptedException {
  long chainId = value.get();
  LOG.info("Starting mapper with chainId:" + chainId);

  byte[] chainIdArray = Bytes.toBytes(chainId);
  long currentRow = 0;

  long chainLength = context.getConfiguration().getLong(CHAIN_LENGTH_KEY, CHAIN_LENGTH);
  long nextRow = getNextRow(0, chainLength);

  for (long i = 0; i < chainLength; i++) {
    byte[] rk = Bytes.toBytes(currentRow);

    // Next link in the chain.
    KeyValue linkKv = new KeyValue(rk, CHAIN_FAM, chainIdArray, Bytes.toBytes(nextRow));
    // What link in the chain this is.
    KeyValue sortKv = new KeyValue(rk, SORT_FAM, chainIdArray, Bytes.toBytes(i));
    // Added data so that large stores are created.
    KeyValue dataKv = new KeyValue(rk, DATA_FAM, chainIdArray,
      Bytes.toBytes(RandomStringUtils.randomAlphabetic(50))
    );

    // Emit the key values.
    context.write(new ImmutableBytesWritable(rk), linkKv);
    context.write(new ImmutableBytesWritable(rk), sortKv);
    context.write(new ImmutableBytesWritable(rk), dataKv);
    // Move to the next row.
    currentRow = nextRow;
    nextRow = getNextRow(i+1, chainLength);
  }
}