org.apache.hadoop.hbase.KeyValue Java Examples

The following examples show how to use org.apache.hadoop.hbase.KeyValue. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TransactionVisibilityFilter.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
@Override
public Cell transformCell(Cell cell) throws IOException {
  // Convert Tephra deletes back into HBase deletes
  if (tx.getVisibilityLevel() == Transaction.VisibilityLevel.SNAPSHOT_ALL) {
    if (DeleteTracker.isFamilyDelete(cell)) {
      return new KeyValue(CellUtil.cloneRow(cell), CellUtil.cloneFamily(cell), null, cell.getTimestamp(),
                          KeyValue.Type.DeleteFamily);
    } else if (isColumnDelete(cell)) {
      // Note: in some cases KeyValue.Type.Delete is used in Delete object,
      // and in some other cases KeyValue.Type.DeleteColumn is used.
      // Since Tephra cannot distinguish between the two, we return KeyValue.Type.DeleteColumn.
      // KeyValue.Type.DeleteColumn makes both CellUtil.isDelete and CellUtil.isDeleteColumns return true, and will
      // work in both cases.
      return new KeyValue(CellUtil.cloneRow(cell), CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell),
                          cell.getTimestamp(), KeyValue.Type.DeleteColumn);
    }
  }
  return cell;
}
 
Example #2
Source File: TestStoreScanner.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Ensure that expired delete family markers don't override valid puts
 */
@Test
public void testExpiredDeleteFamily() throws Exception {
  long now = System.currentTimeMillis();
  KeyValue[] kvs = new KeyValue[] {
    new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now-1000,
      KeyValue.Type.DeleteFamily),
    create("R1", "cf", "a", now-10, KeyValue.Type.Put,
      "dont-care"),
  };
  List<KeyValueScanner> scanners = scanFixture(kvs);
  Scan scan = new Scan();
  scan.readVersions(1);
  // scanner with ttl equal to 500
  ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
  try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) {
    List<Cell> results = new ArrayList<>();
    assertEquals(true, scanner.next(results));
    assertEquals(1, results.size());
    assertEquals(kvs[1], results.get(0));
    results.clear();

    assertEquals(false, scanner.next(results));
  }
}
 
Example #3
Source File: TestCompactingToCellFlatMapMemStore.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testCountOfCellsAfterFlatteningByScan() throws IOException {
  String[] keys1 = { "A", "B", "C" }; // A, B, C
  addRowsByKeysWith50Cols(memstore, keys1);
  // this should only flatten as there are no duplicates
  ((CompactingMemStore) memstore).flushInMemory();
  while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) {
    Threads.sleep(10);
  }
  List<KeyValueScanner> scanners = memstore.getScanners(Long.MAX_VALUE);
  // seek
  int count = 0;
  for(int i = 0; i < scanners.size(); i++) {
    scanners.get(i).seek(KeyValue.LOWESTKEY);
    while (scanners.get(i).next() != null) {
      count++;
    }
  }
  assertEquals("the count should be ", 150, count);
  for(int i = 0; i < scanners.size(); i++) {
    scanners.get(i).close();
  }
}
 
Example #4
Source File: JavaHBaseContextSuite.java    From learning-hadoop with Apache License 2.0 6 votes vote down vote up
public String call(Result result) throws Exception {
  Iterator<KeyValue> it = result.list().iterator();
  StringBuilder b = new StringBuilder();

  b.append(Bytes.toString(result.getRow()) + ":");

  while (it.hasNext()) {
    KeyValue kv = it.next();
    String q = Bytes.toString(kv.getQualifier());
    if (q.equals("counter")) {
      b.append("(" + Bytes.toString(kv.getQualifier()) + ","
          + Bytes.toLong(kv.getValue()) + ")");
    } else {
      b.append("(" + Bytes.toString(kv.getQualifier()) + ","
          + Bytes.toString(kv.getValue()) + ")");
    }
  }
  return b.toString();
}
 
Example #5
Source File: IndexHalfStoreFileReader.java    From phoenix with Apache License 2.0 6 votes vote down vote up
/**
 * @param fs
 * @param p
 * @param cacheConf
 * @param r
 * @param conf
 * @param indexMaintainers
 * @param viewConstants
 * @param regionInfo
 * @param regionStartKeyInHFile
 * @param splitKey
 * @throws IOException
 */
public IndexHalfStoreFileReader(final FileSystem fs, final Path p, final CacheConfig cacheConf,
        final Reference r, final Configuration conf,
        final Map<ImmutableBytesWritable, IndexMaintainer> indexMaintainers,
        final byte[][] viewConstants, final HRegionInfo regionInfo,
        final byte[] regionStartKeyInHFile, byte[] splitKey) throws IOException {
    super(fs, p, cacheConf, conf);
    this.splitkey = splitKey == null ? r.getSplitKey() : splitKey;
    // Is it top or bottom half?
    this.top = Reference.isTopFileRegion(r.getFileRegion());
    this.splitRow = CellUtil.cloneRow(KeyValue.createKeyValueFromKey(splitkey));
    this.indexMaintainers = indexMaintainers;
    this.viewConstants = viewConstants;
    this.regionInfo = regionInfo;
    this.regionStartKeyInHFile = regionStartKeyInHFile;
    this.offset = regionStartKeyInHFile.length;
}
 
Example #6
Source File: TestHFile.java    From hbase with Apache License 2.0 6 votes vote down vote up
private Path writeStoreFile() throws IOException {
  Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), "TestHFile");
  HFileContext meta = new HFileContextBuilder().withBlockSize(64 * 1024).build();
  StoreFileWriter sfw =
      new StoreFileWriter.Builder(conf, fs).withOutputDir(storeFileParentDir)
        .withFileContext(meta).build();

  final int rowLen = 32;
  Random RNG = new Random();
  for (int i = 0; i < 1000; ++i) {
    byte[] k = RandomKeyValueUtil.randomOrderedKey(RNG, i);
    byte[] v = RandomKeyValueUtil.randomValue(RNG);
    int cfLen = RNG.nextInt(k.length - rowLen + 1);
    KeyValue kv =
        new KeyValue(k, 0, rowLen, k, rowLen, cfLen, k, rowLen + cfLen,
            k.length - rowLen - cfLen, RNG.nextLong(), generateKeyType(RNG), v, 0, v.length);
    sfw.append(kv);
  }

  sfw.close();
  return sfw.getPath();
}
 
Example #7
Source File: SlicedRowFilter.java    From warp10-platform with Apache License 2.0 6 votes vote down vote up
@Override
public KeyValue getNextKeyHint(KeyValue currentKV) {
  //hintCount++;
  KeyValue hint = null;
      
  if (this.hintOffset >= 0 && this.hintOffset <= this.rangekeys.length - slicesLength) {
    hint = KeyValueUtil.createFirstOnRow(this.rangekeys, this.hintOffset, (short) (this.bounds[1] + 1));
    minRange = (hintOffset / this.slicesLength) / 2;
  } else {
    done = true;
  }

  /*
  byte[] row = currentKV.getRowArray();
  System.out.println("getNextKeyHint " + encodeHex(row, currentKV.getRowOffset(), currentKV.getRowLength()) + " nvalues = " + this.nvalues + " count = " + this.count + " hintOffset = " + hintOffset);
  if (null != hint) {
    row = hint.getRowArray();
    System.out.println("  hint = " + encodeHex(row, hint.getRowOffset(), hint.getRowLength())); 
  } else {
    System.out.println(" hint = null");
  }
  */
  
  return hint;
}
 
Example #8
Source File: MorphlineResultToSolrMapperTest.java    From hbase-indexer with Apache License 2.0 6 votes vote down vote up
@Test
public void testMap() throws Exception {
    MorphlineResultToSolrMapper resultMapper = new MorphlineResultToSolrMapper();
    resultMapper.configure(ImmutableMap.of(
        MorphlineResultToSolrMapper.MORPHLINE_FILE_PARAM, "src/test/resources/test-morphlines/extractHBaseCells.conf")
        );

    Cell kvA = new KeyValue(ROW, COLUMN_FAMILY_A, QUALIFIER_A, Bytes.toBytes(42));
    Cell kvB = new KeyValue(ROW, COLUMN_FAMILY_B, QUALIFIER_B, "dummy value".getBytes("UTF-8"));
    Result result = Result.create(Lists.newArrayList(kvA, kvB));

    Multimap expectedMap = ImmutableMultimap.of("fieldA", 42, "fieldB", "dummy value");

    resultMapper.map(result, updateWriter);
    verify(updateWriter).add(solrInputDocCaptor.capture());
    
    SolrInputDocument solrDocument = solrInputDocCaptor.getValue();
    assertEquals(expectedMap, toRecord(solrDocument).getFields());
}
 
Example #9
Source File: TestDefaultMemStore.java    From hbase with Apache License 2.0 6 votes vote down vote up
protected void verifyScanAcrossSnapshot2(KeyValue kv1, KeyValue kv2)
    throws IOException {
  List<KeyValueScanner> memstorescanners = this.memstore.getScanners(mvcc.getReadPoint());
  assertEquals(2, memstorescanners.size());
  final KeyValueScanner scanner0 = memstorescanners.get(0);
  final KeyValueScanner scanner1 = memstorescanners.get(1);
  scanner0.seek(KeyValueUtil.createFirstOnRow(HConstants.EMPTY_START_ROW));
  scanner1.seek(KeyValueUtil.createFirstOnRow(HConstants.EMPTY_START_ROW));
  Cell n0 = scanner0.next();
  Cell n1 = scanner1.next();
  assertTrue(kv1.equals(n0) || kv1.equals(n1));
  assertTrue(kv2.equals(n0)
      || kv2.equals(n1)
      || kv2.equals(scanner0.next())
      || kv2.equals(scanner1.next()));
  assertNull(scanner0.next());
  assertNull(scanner1.next());
}
 
Example #10
Source File: IndexerDemo.java    From hbase-secondary-index with GNU General Public License v3.0 6 votes vote down vote up
@Override
@SuppressWarnings("rawtypes")
public void postPut(final ObserverContext e, final Put put,
		final WALEdit edit, final boolean writeToWAL) throws IOException {

	byte[][] colkey = KeyValue.parseColumn(Bytes.toBytes(inputColumn));
	if (colkey.length > 1) {
		List kvList = put.get(colkey[0], colkey[1]);
		Iterator kvl = kvList.iterator();

		while (kvl.hasNext()) {
			KeyValue kv = (KeyValue) kvl.next();
			Put indexPut = new Put(kv.getValue());
			colkey = KeyValue.parseColumn(Bytes.toBytes(indexColumn));
			indexPut.add(colkey[0], colkey[1], kv.getRow());
			table.put(indexPut);
		}
	}
}
 
Example #11
Source File: IndexMemStore.java    From phoenix with Apache License 2.0 6 votes vote down vote up
/**
 * Move forward on the sub-lists set previously by seek.
 * @param key seek value (should be non-null)
 * @return true if there is at least one KV to read, false otherwise
 */
@Override
public synchronized boolean reseek(Cell key) {
  /*
   * See HBASE-4195 & HBASE-3855 & HBASE-6591 for the background on this implementation. This
   * code is executed concurrently with flush and puts, without locks. Two points must be known
   * when working on this code: 1) It's not possible to use the 'kvTail' and 'snapshot'
   * variables, as they are modified during a flush. 2) The ideal implementation for performance
   * would use the sub skip list implicitly pointed by the iterators 'kvsetIt' and 'snapshotIt'.
   * Unfortunately the Java API does not offer a method to get it. So we remember the last keys
   * we iterated to and restore the reseeked set to at least that point.
   */

  KeyValue kv = KeyValueUtil.ensureKeyValue(key);
  kvsetIt = kvsetAtCreation.tailSet(getHighest(kv, kvsetItRow)).iterator();
  return seekInSubLists();
}
 
Example #12
Source File: TTable.java    From phoenix-omid with Apache License 2.0 6 votes vote down vote up
/**
 * @param put an instance of Put
 * @param timestamp  timestamp to be used as cells version
 * @param commitTimestamp  timestamp to be used as commit timestamp
 * @throws IOException if a remote or network exception occurs.
 */
static public Put markPutAsCommitted(Put put, long timestamp, long commitTimestamp) {
    final Put tsput = new Put(put.getRow(), timestamp);
    propagateAttributes(put, tsput);

    Map<byte[], List<Cell>> kvs = put.getFamilyCellMap();
    for (List<Cell> kvl : kvs.values()) {
        for (Cell c : kvl) {
            KeyValue kv = KeyValueUtil.ensureKeyValue(c);
            Bytes.putLong(kv.getValueArray(), kv.getTimestampOffset(), timestamp);
            try {
                tsput.add(kv);
            } catch (IOException e) {
                // The existing Put has this Cell, so the cloned one
                // will never throw an IOException when it's added.
                throw new RuntimeException(e);
            }
            tsput.addColumn(CellUtil.cloneFamily(kv),
                    CellUtils.addShadowCellSuffixPrefix(CellUtil.cloneQualifier(kv), 0, CellUtil.cloneQualifier(kv).length),
                    kv.getTimestamp(),
                    Bytes.toBytes(commitTimestamp));
        }
    }

    return tsput;
}
 
Example #13
Source File: IndexManagementUtil.java    From phoenix with Apache License 2.0 6 votes vote down vote up
/**
 * check to see if the kvs in the update match any of the passed columns. Generally, this is useful to for an index
 * codec to determine if a given update should even be indexed. This assumes that for any index, there are going to
 * small number of columns, versus the number of kvs in any one batch.
 */
public static boolean updateMatchesColumns(Collection<KeyValue> update, List<ColumnReference> columns) {
    // check to see if the kvs in the new update even match any of the columns requested
    // assuming that for any index, there are going to small number of columns, versus the number of
    // kvs in any one batch.
    boolean matches = false;
    outer: for (KeyValue kv : update) {
        for (ColumnReference ref : columns) {
            if (ref.matchesFamily(kv.getFamilyArray(), kv.getFamilyOffset(),
                kv.getFamilyLength())
                    && ref.matchesQualifier(kv.getQualifierArray(), kv.getQualifierOffset(),
                        kv.getQualifierLength())) {
                matches = true;
                // if a single column matches a single kv, we need to build a whole scanner
                break outer;
            }
        }
    }
    return matches;
}
 
Example #14
Source File: ConnectionlessTest.java    From phoenix with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("deprecation")
private static void assertRow1(Iterator<KeyValue> iterator, byte[] expectedRowKey1) {
    KeyValue kv;
    assertTrue(iterator.hasNext());
    kv = iterator.next();
    assertArrayEquals(expectedRowKey1, kv.getRow());        
    assertEquals(name1, PVarchar.INSTANCE.toObject(kv.getValue()));
    assertTrue(iterator.hasNext());
    kv = iterator.next();
    assertArrayEquals(expectedRowKey1, kv.getRow());        
    assertEquals(now, PDate.INSTANCE.toObject(kv.getValue()));
    assertTrue(iterator.hasNext());
    kv = iterator.next();
    assertArrayEquals(expectedRowKey1, kv.getRow());        
    assertNull(PVarchar.INSTANCE.toObject(kv.getValue()));
}
 
Example #15
Source File: CellSkipFilter.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
@Override
public ReturnCode filterKeyValue(Cell cell) throws IOException {
  if (skipCellVersion(cell)) {
    return ReturnCode.NEXT_COL;
  }

  ReturnCode code = filter.filterKeyValue(cell);
  if (code == ReturnCode.NEXT_COL || code == ReturnCode.INCLUDE_AND_NEXT_COL) {
    // only store the reference to the keyvalue if we are returning NEXT_COL or INCLUDE_AND_NEXT_COL
    skipColumn = KeyValue.createFirstOnRow(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
                                             cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
                                             cell.getQualifierArray(), cell.getQualifierOffset(),
                                             cell.getQualifierLength());
  } else {
    skipColumn = null;
  }
  return code;
}
 
Example #16
Source File: Sequence.java    From phoenix with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
public long dropSequence(Result result) throws SQLException {
    KeyValue statusKV = result.raw()[0];
    long timestamp = statusKV.getTimestamp();
    int statusCode = PDataType.INTEGER.getCodec().decodeInt(statusKV.getBuffer(), statusKV.getValueOffset(), null);
    SQLExceptionCode code = statusCode == 0 ? null : SQLExceptionCode.fromErrorCode(statusCode);
    if (code == null) {
        // Insert delete marker so that point-in-time sequences work
        insertSequenceValue(new SequenceValue(timestamp, true));
        return timestamp;
    }
    // TODO: We could have the server return the timestamps of the
    // delete markers and we could insert them here, but this seems
    // like overkill.
    // if (code == SQLExceptionCode.SEQUENCE_UNDEFINED) {
    // }
    throw new SQLExceptionInfo.Builder(code)
        .setSchemaName(key.getSchemaName())
        .setTableName(key.getSequenceName())
        .build().buildException();
}
 
Example #17
Source File: TestCompoundBloomFilter.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateKey() {
  byte[] row = Bytes.toBytes("myRow");
  byte[] qualifier = Bytes.toBytes("myQualifier");
  // Mimic what Storefile.createBloomKeyValue() does
  byte[] rowKey = KeyValueUtil.createFirstOnRow(row, 0, row.length, new byte[0], 0, 0, row, 0, 0).getKey();
  byte[] rowColKey = KeyValueUtil.createFirstOnRow(row, 0, row.length,
      new byte[0], 0, 0, qualifier, 0, qualifier.length).getKey();
  KeyValue rowKV = KeyValueUtil.createKeyValueFromKey(rowKey);
  KeyValue rowColKV = KeyValueUtil.createKeyValueFromKey(rowColKey);
  assertEquals(rowKV.getTimestamp(), rowColKV.getTimestamp());
  assertEquals(Bytes.toStringBinary(rowKV.getRowArray(), rowKV.getRowOffset(),
    rowKV.getRowLength()), Bytes.toStringBinary(rowColKV.getRowArray(), rowColKV.getRowOffset(),
    rowColKV.getRowLength()));
  assertEquals(0, rowKV.getQualifierLength());
}
 
Example #18
Source File: TestStoreScanner.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testScanSameTimestamp() throws IOException {
  // returns only 1 of these 2 even though same timestamp
  KeyValue [] kvs = new KeyValue[] {
      create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
      create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
  };
  List<KeyValueScanner> scanners = Arrays.asList(
    new KeyValueScanner[] {new KeyValueScanFixture(CellComparator.getInstance(), kvs)});

  Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
  // this only uses maxVersions (default=1) and TimeRange (default=all)
  try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, null, scanners)) {
    List<Cell> results = new ArrayList<>();
    assertEquals(true, scan.next(results));
    assertEquals(1, results.size());
    assertEquals(1, scan.memstoreOnlyReads);
    assertEquals(kvs[0], results.get(0));
  }
}
 
Example #19
Source File: KeyValueUtil.java    From phoenix with Apache License 2.0 5 votes vote down vote up
public static KeyValue newKeyValue(byte[] key, byte[] cf, byte[] cq, long ts, byte[] value, int valueOffset, int valueLength) {
    return new KeyValue(key, 0, key.length,
            cf, 0, cf.length,
            cq, 0, cq.length,
            ts, Type.Put,
            value, valueOffset, valueLength);
}
 
Example #20
Source File: Indexer.java    From hbase-indexer with Apache License 2.0 5 votes vote down vote up
@Override
protected void calculateIndexUpdates(List<RowData> rowDataList, SolrUpdateCollector updateCollector) throws IOException {
    Map<String, KeyValue> idToKeyValue = calculateUniqueEvents(rowDataList);
    for (Entry<String, KeyValue> idToKvEntry : idToKeyValue.entrySet()) {
        String documentId = idToKvEntry.getKey();

        KeyValue keyValue = idToKvEntry.getValue();
        if (CellUtil.isDelete(keyValue)) {
            handleDelete(documentId, keyValue, updateCollector, uniqueKeyFormatter);
        } else {
            Result result = Result.create(Collections.<Cell>singletonList(keyValue));
            SolrUpdateWriter updateWriter = new RowAndFamilyAddingSolrUpdateWriter(
                    conf.getRowField(),
                    conf.getColumnFamilyField(),
                    uniqueKeyFormatter,
                    keyValue,
                    new IdAddingSolrUpdateWriter(
                            conf.getUniqueKeyField(),
                            documentId,
                            conf.getTableNameField(),
                            tableName,
                            updateCollector));

            mapper.map(result, updateWriter);

        }
    }
}
 
Example #21
Source File: TestFSHLogProvider.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void addEdits(WAL log, RegionInfo hri, TableDescriptor htd, int times,
    NavigableMap<byte[], Integer> scopes) throws IOException {
  final byte[] row = Bytes.toBytes("row");
  for (int i = 0; i < times; i++) {
    long timestamp = System.currentTimeMillis();
    WALEdit cols = new WALEdit();
    cols.add(new KeyValue(row, row, row, timestamp, row));
    log.appendData(hri,
      getWalKey(hri.getEncodedNameAsBytes(), htd.getTableName(), timestamp, scopes), cols);
  }
  log.sync();
}
 
Example #22
Source File: IndexedTableAdmin.java    From hbase-secondary-index with GNU General Public License v3.0 5 votes vote down vote up
private HTableDescriptor createIndexTableDesc(final byte[] baseTableName, final IndexSpecification indexSpec) {
    HTableDescriptor indexTableDesc = new HTableDescriptor(indexSpec.getIndexedTableName(baseTableName));
    Set<byte[]> families = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
    families.add(IndexedTable.INDEX_COL_FAMILY);
    for (byte[] column : indexSpec.getAllColumns()) {
        families.add(KeyValue.parseColumn(column)[0]);
    }

    for (byte[] colFamily : families) {
        indexTableDesc.addFamily(new HColumnDescriptor(colFamily));
    }

    return indexTableDesc;
}
 
Example #23
Source File: Indexer.java    From phoenix with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
/**
 * Extract the index updates from the WAL Edit
 * @param edit to search for index updates
 * @return the mutations to apply to the index tables
 */
private Collection<Pair<Mutation, byte[]>> extractIndexUpdate(WALEdit edit) {
  Collection<Pair<Mutation, byte[]>> indexUpdates = new ArrayList<Pair<Mutation, byte[]>>();
  for (KeyValue kv : edit.getKeyValues()) {
    if (kv instanceof IndexedKeyValue) {
      IndexedKeyValue ikv = (IndexedKeyValue) kv;
      indexUpdates.add(new Pair<Mutation, byte[]>(ikv.getMutation(), ikv.getIndexTable()));
    }
  }

  return indexUpdates;
}
 
Example #24
Source File: TestLocalTableState.java    From phoenix with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Test
public void testCorrectOrderingWithLazyLoadingColumns() throws Exception {
  Put m = new Put(row);
  m.add(fam, qual, ts, val);
  // setup mocks
  Configuration conf = new Configuration(false);
  RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);
  Mockito.when(env.getConfiguration()).thenReturn(conf);

  HRegion region = Mockito.mock(HRegion.class);
  Mockito.when(env.getRegion()).thenReturn(region);
  RegionScanner scanner = Mockito.mock(RegionScanner.class);
  Mockito.when(region.getScanner(Mockito.any(Scan.class))).thenReturn(scanner);
  final byte[] stored = Bytes.toBytes("stored-value");
  Mockito.when(scanner.next(Mockito.any(List.class))).thenAnswer(new Answer<Boolean>() {
    @Override
    public Boolean answer(InvocationOnMock invocation) throws Throwable {
      List<KeyValue> list = (List<KeyValue>) invocation.getArguments()[0];
      KeyValue kv = new KeyValue(row, fam, qual, ts, Type.Put, stored);
      kv.setMemstoreTS(0);
      list.add(kv);
      return false;
    }
  });


  LocalHBaseState state = new LocalTable(env);
  LocalTableState table = new LocalTableState(env, state, m);
  //add the kvs from the mutation
  table.addPendingUpdates(m.get(fam, qual));

  // setup the lookup
  ColumnReference col = new ColumnReference(fam, qual);
  table.setCurrentTimestamp(ts);
  //check that our value still shows up first on scan, even though this is a lazy load
  Pair<Scanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col));
  Scanner s = p.getFirst();
  assertEquals("Didn't get the pending mutation's value first", m.get(fam, qual).get(0), s.next());
}
 
Example #25
Source File: HBaseIndexBulkMapperBase.java    From hgraphdb with Apache License 2.0 5 votes vote down vote up
@Override
protected void map(ImmutableBytesWritable key, Result result, Context context)
        throws IOException, InterruptedException {

    final ImmutableBytesWritable outputKey = new ImmutableBytesWritable();
    final Iterator<? extends Mutation> mutations = constructMutations(result);
    List<KeyValue> keyValueList = toKeyValues(IteratorUtils.list(mutations));
    for (KeyValue kv : keyValueList) {
        outputKey.set(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength());
        context.write(outputKey, kv);
    }
}
 
Example #26
Source File: ColumnBasedIndexerTest.java    From hbase-indexer with Apache License 2.0 5 votes vote down vote up
@Test
public void testCalculateIndexUpdates_DeleteRow_NoRowFieldDefinedForIndexer() throws IOException {
    KeyValue toDelete = new KeyValue(Bytes.toBytes("_row_"), Bytes.toBytes("_cf_"),
                                     Bytes.toBytes("_qual_"),
                                     0L, Type.Delete);
    RowData eventRowData = createEventRowData("_row_", toDelete);
    
    indexer.calculateIndexUpdates(ImmutableList.of(eventRowData), updateCollector);

    assertTrue(updateCollector.getDeleteQueries().isEmpty());
    assertTrue(updateCollector.getIdsToDelete().isEmpty());
    assertTrue(updateCollector.getDocumentsToAdd().isEmpty());
}
 
Example #27
Source File: TestStoreScanner.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test @Ignore("this fails, since we don't handle deletions, etc, in peek")
public void testPeek() throws Exception {
  KeyValue[] kvs = new KeyValue [] {
      create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
      create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"),
  };
  List<KeyValueScanner> scanners = scanFixture(kvs);
  Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
  try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
    assertNull(scan.peek());
  }
}
 
Example #28
Source File: HBaseSail.java    From Halyard with Apache License 2.0 5 votes vote down vote up
protected void addStatementInternal(Resource subj, IRI pred, Value obj, Resource context, long timestamp) throws SailException {
    if (!isWritable()) throw new SailException(tableName + " is read only");
    try {
        for (KeyValue kv : HalyardTableUtils.toKeyValues(subj, pred, obj, context, false, timestamp)) { //serialize the key value pairs relating to the statement in HBase
            put(kv);
        }
    } catch (IOException e) {
        throw new SailException(e);
    }
}
 
Example #29
Source File: BufferedSortedQueue.java    From phoenix with Apache License 2.0 5 votes vote down vote up
private List<KeyValue> toKeyValues(ResultEntry entry) {
    Tuple result = entry.getResult();
    int size = result.size();
    List<KeyValue> kvs = new ArrayList<KeyValue>(size);
    for (int i = 0; i < size; i++) {
        kvs.add(PhoenixKeyValueUtil.maybeCopyCell(result.getValue(i)));
    }
    return kvs;
}
 
Example #30
Source File: TestIndexMemStore.java    From phoenix with Apache License 2.0 5 votes vote down vote up
/**
 * We don't expect custom KeyValue creation, so we can't get into weird situations, where a
 * {@link Type#DeleteFamily} has a column qualifier specified.
 * @throws Exception
 */
@Test
public void testExpectedOrdering() throws Exception {
  IndexMemStore store = new IndexMemStore();
  KeyValue kv = new KeyValue(row, family, qual, 12, Type.Put, val);
  store.add(kv, true);
  KeyValue kv2 = new KeyValue(row, family, qual, 10, Type.Put, val2);
  store.add(kv2, true);
  KeyValue df = new KeyValue(row, family, null, 11, Type.DeleteFamily, null);
  store.add(df, true);
  KeyValue dc = new KeyValue(row, family, qual, 11, Type.DeleteColumn, null);
  store.add(dc, true);
  KeyValue d = new KeyValue(row, family, qual, 12, Type.Delete, null);
  store.add(d, true);

  // null qualifiers should always sort before the non-null cases
  ReseekableScanner scanner = store.getScanner();
  KeyValue first = KeyValueUtil.createFirstOnRow(row);
  assertTrue("Didn't have any data in the scanner", scanner.seek(first));
  assertTrue("Didn't get delete family first (no qualifier == sort first)", df == scanner.next());
  assertTrue("Didn't get point delete before corresponding put", d == scanner.next());
  assertTrue("Didn't get larger ts Put", kv == scanner.next());
  assertTrue("Didn't get delete column before corresponding put(delete sorts first)",
    dc == scanner.next());
  assertTrue("Didn't get smaller ts Put", kv2 == scanner.next());
  assertNull("Have more data in the scanner", scanner.next());
}