Java Code Examples for org.apache.hadoop.hbase.KeyValue

The following examples show how to use org.apache.hadoop.hbase.KeyValue. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
Source Project: phoenix   Source File: IndexHalfStoreFileReader.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * @param fs
 * @param p
 * @param cacheConf
 * @param r
 * @param conf
 * @param indexMaintainers
 * @param viewConstants
 * @param regionInfo
 * @param regionStartKeyInHFile
 * @param splitKey
 * @throws IOException
 */
public IndexHalfStoreFileReader(final FileSystem fs, final Path p, final CacheConfig cacheConf,
        final Reference r, final Configuration conf,
        final Map<ImmutableBytesWritable, IndexMaintainer> indexMaintainers,
        final byte[][] viewConstants, final HRegionInfo regionInfo,
        final byte[] regionStartKeyInHFile, byte[] splitKey) throws IOException {
    super(fs, p, cacheConf, conf);
    this.splitkey = splitKey == null ? r.getSplitKey() : splitKey;
    // Is it top or bottom half?
    this.top = Reference.isTopFileRegion(r.getFileRegion());
    this.splitRow = CellUtil.cloneRow(KeyValue.createKeyValueFromKey(splitkey));
    this.indexMaintainers = indexMaintainers;
    this.viewConstants = viewConstants;
    this.regionInfo = regionInfo;
    this.regionStartKeyInHFile = regionStartKeyInHFile;
    this.offset = regionStartKeyInHFile.length;
}
 
Example 2
@Test
public void testMap() throws Exception {
    MorphlineResultToSolrMapper resultMapper = new MorphlineResultToSolrMapper();
    resultMapper.configure(ImmutableMap.of(
        MorphlineResultToSolrMapper.MORPHLINE_FILE_PARAM, "src/test/resources/test-morphlines/extractHBaseCells.conf")
        );

    Cell kvA = new KeyValue(ROW, COLUMN_FAMILY_A, QUALIFIER_A, Bytes.toBytes(42));
    Cell kvB = new KeyValue(ROW, COLUMN_FAMILY_B, QUALIFIER_B, "dummy value".getBytes("UTF-8"));
    Result result = Result.create(Lists.newArrayList(kvA, kvB));

    Multimap expectedMap = ImmutableMultimap.of("fieldA", 42, "fieldB", "dummy value");

    resultMapper.map(result, updateWriter);
    verify(updateWriter).add(solrInputDocCaptor.capture());
    
    SolrInputDocument solrDocument = solrInputDocCaptor.getValue();
    assertEquals(expectedMap, toRecord(solrDocument).getFields());
}
 
Example 3
@Override
@SuppressWarnings("rawtypes")
public void postPut(final ObserverContext e, final Put put,
		final WALEdit edit, final boolean writeToWAL) throws IOException {

	byte[][] colkey = KeyValue.parseColumn(Bytes.toBytes(inputColumn));
	if (colkey.length > 1) {
		List kvList = put.get(colkey[0], colkey[1]);
		Iterator kvl = kvList.iterator();

		while (kvl.hasNext()) {
			KeyValue kv = (KeyValue) kvl.next();
			Put indexPut = new Put(kv.getValue());
			colkey = KeyValue.parseColumn(Bytes.toBytes(indexColumn));
			indexPut.add(colkey[0], colkey[1], kv.getRow());
			table.put(indexPut);
		}
	}
}
 
Example 4
Source Project: phoenix   Source File: IndexMemStore.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Move forward on the sub-lists set previously by seek.
 * @param key seek value (should be non-null)
 * @return true if there is at least one KV to read, false otherwise
 */
@Override
public synchronized boolean reseek(Cell key) {
  /*
   * See HBASE-4195 & HBASE-3855 & HBASE-6591 for the background on this implementation. This
   * code is executed concurrently with flush and puts, without locks. Two points must be known
   * when working on this code: 1) It's not possible to use the 'kvTail' and 'snapshot'
   * variables, as they are modified during a flush. 2) The ideal implementation for performance
   * would use the sub skip list implicitly pointed by the iterators 'kvsetIt' and 'snapshotIt'.
   * Unfortunately the Java API does not offer a method to get it. So we remember the last keys
   * we iterated to and restore the reseeked set to at least that point.
   */

  KeyValue kv = KeyValueUtil.ensureKeyValue(key);
  kvsetIt = kvsetAtCreation.tailSet(getHighest(kv, kvsetItRow)).iterator();
  return seekInSubLists();
}
 
Example 5
Source Project: phoenix-omid   Source File: TTable.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * @param put an instance of Put
 * @param timestamp  timestamp to be used as cells version
 * @param commitTimestamp  timestamp to be used as commit timestamp
 * @throws IOException if a remote or network exception occurs.
 */
static public Put markPutAsCommitted(Put put, long timestamp, long commitTimestamp) {
    final Put tsput = new Put(put.getRow(), timestamp);
    propagateAttributes(put, tsput);

    Map<byte[], List<Cell>> kvs = put.getFamilyCellMap();
    for (List<Cell> kvl : kvs.values()) {
        for (Cell c : kvl) {
            KeyValue kv = KeyValueUtil.ensureKeyValue(c);
            Bytes.putLong(kv.getValueArray(), kv.getTimestampOffset(), timestamp);
            try {
                tsput.add(kv);
            } catch (IOException e) {
                // The existing Put has this Cell, so the cloned one
                // will never throw an IOException when it's added.
                throw new RuntimeException(e);
            }
            tsput.addColumn(CellUtil.cloneFamily(kv),
                    CellUtils.addShadowCellSuffixPrefix(CellUtil.cloneQualifier(kv), 0, CellUtil.cloneQualifier(kv).length),
                    kv.getTimestamp(),
                    Bytes.toBytes(commitTimestamp));
        }
    }

    return tsput;
}
 
Example 6
Source Project: hbase   Source File: TestCompoundBloomFilter.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateKey() {
  byte[] row = Bytes.toBytes("myRow");
  byte[] qualifier = Bytes.toBytes("myQualifier");
  // Mimic what Storefile.createBloomKeyValue() does
  byte[] rowKey = KeyValueUtil.createFirstOnRow(row, 0, row.length, new byte[0], 0, 0, row, 0, 0).getKey();
  byte[] rowColKey = KeyValueUtil.createFirstOnRow(row, 0, row.length,
      new byte[0], 0, 0, qualifier, 0, qualifier.length).getKey();
  KeyValue rowKV = KeyValueUtil.createKeyValueFromKey(rowKey);
  KeyValue rowColKV = KeyValueUtil.createKeyValueFromKey(rowColKey);
  assertEquals(rowKV.getTimestamp(), rowColKV.getTimestamp());
  assertEquals(Bytes.toStringBinary(rowKV.getRowArray(), rowKV.getRowOffset(),
    rowKV.getRowLength()), Bytes.toStringBinary(rowColKV.getRowArray(), rowColKV.getRowOffset(),
    rowColKV.getRowLength()));
  assertEquals(0, rowKV.getQualifierLength());
}
 
Example 7
Source Project: hbase   Source File: TestStoreScanner.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testScanSameTimestamp() throws IOException {
  // returns only 1 of these 2 even though same timestamp
  KeyValue [] kvs = new KeyValue[] {
      create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
      create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
  };
  List<KeyValueScanner> scanners = Arrays.asList(
    new KeyValueScanner[] {new KeyValueScanFixture(CellComparator.getInstance(), kvs)});

  Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
  // this only uses maxVersions (default=1) and TimeRange (default=all)
  try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, null, scanners)) {
    List<Cell> results = new ArrayList<>();
    assertEquals(true, scan.next(results));
    assertEquals(1, results.size());
    assertEquals(1, scan.memstoreOnlyReads);
    assertEquals(kvs[0], results.get(0));
  }
}
 
Example 8
Source Project: phoenix   Source File: Sequence.java    License: BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
public long dropSequence(Result result) throws SQLException {
    KeyValue statusKV = result.raw()[0];
    long timestamp = statusKV.getTimestamp();
    int statusCode = PDataType.INTEGER.getCodec().decodeInt(statusKV.getBuffer(), statusKV.getValueOffset(), null);
    SQLExceptionCode code = statusCode == 0 ? null : SQLExceptionCode.fromErrorCode(statusCode);
    if (code == null) {
        // Insert delete marker so that point-in-time sequences work
        insertSequenceValue(new SequenceValue(timestamp, true));
        return timestamp;
    }
    // TODO: We could have the server return the timestamps of the
    // delete markers and we could insert them here, but this seems
    // like overkill.
    // if (code == SQLExceptionCode.SEQUENCE_UNDEFINED) {
    // }
    throw new SQLExceptionInfo.Builder(code)
        .setSchemaName(key.getSchemaName())
        .setTableName(key.getSequenceName())
        .build().buildException();
}
 
Example 9
Source Project: phoenix-tephra   Source File: CellSkipFilter.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ReturnCode filterKeyValue(Cell cell) throws IOException {
  if (skipCellVersion(cell)) {
    return ReturnCode.NEXT_COL;
  }

  ReturnCode code = filter.filterKeyValue(cell);
  if (code == ReturnCode.NEXT_COL || code == ReturnCode.INCLUDE_AND_NEXT_COL) {
    // only store the reference to the keyvalue if we are returning NEXT_COL or INCLUDE_AND_NEXT_COL
    skipColumn = KeyValue.createFirstOnRow(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
                                             cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
                                             cell.getQualifierArray(), cell.getQualifierOffset(),
                                             cell.getQualifierLength());
  } else {
    skipColumn = null;
  }
  return code;
}
 
Example 10
Source Project: phoenix   Source File: ConnectionlessTest.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("deprecation")
private static void assertRow1(Iterator<KeyValue> iterator, byte[] expectedRowKey1) {
    KeyValue kv;
    assertTrue(iterator.hasNext());
    kv = iterator.next();
    assertArrayEquals(expectedRowKey1, kv.getRow());        
    assertEquals(name1, PVarchar.INSTANCE.toObject(kv.getValue()));
    assertTrue(iterator.hasNext());
    kv = iterator.next();
    assertArrayEquals(expectedRowKey1, kv.getRow());        
    assertEquals(now, PDate.INSTANCE.toObject(kv.getValue()));
    assertTrue(iterator.hasNext());
    kv = iterator.next();
    assertArrayEquals(expectedRowKey1, kv.getRow());        
    assertNull(PVarchar.INSTANCE.toObject(kv.getValue()));
}
 
Example 11
Source Project: phoenix   Source File: IndexManagementUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * check to see if the kvs in the update match any of the passed columns. Generally, this is useful to for an index
 * codec to determine if a given update should even be indexed. This assumes that for any index, there are going to
 * small number of columns, versus the number of kvs in any one batch.
 */
public static boolean updateMatchesColumns(Collection<KeyValue> update, List<ColumnReference> columns) {
    // check to see if the kvs in the new update even match any of the columns requested
    // assuming that for any index, there are going to small number of columns, versus the number of
    // kvs in any one batch.
    boolean matches = false;
    outer: for (KeyValue kv : update) {
        for (ColumnReference ref : columns) {
            if (ref.matchesFamily(kv.getFamilyArray(), kv.getFamilyOffset(),
                kv.getFamilyLength())
                    && ref.matchesQualifier(kv.getQualifierArray(), kv.getQualifierOffset(),
                        kv.getQualifierLength())) {
                matches = true;
                // if a single column matches a single kv, we need to build a whole scanner
                break outer;
            }
        }
    }
    return matches;
}
 
Example 12
Source Project: hbase   Source File: TestDefaultMemStore.java    License: Apache License 2.0 6 votes vote down vote up
protected void verifyScanAcrossSnapshot2(KeyValue kv1, KeyValue kv2)
    throws IOException {
  List<KeyValueScanner> memstorescanners = this.memstore.getScanners(mvcc.getReadPoint());
  assertEquals(2, memstorescanners.size());
  final KeyValueScanner scanner0 = memstorescanners.get(0);
  final KeyValueScanner scanner1 = memstorescanners.get(1);
  scanner0.seek(KeyValueUtil.createFirstOnRow(HConstants.EMPTY_START_ROW));
  scanner1.seek(KeyValueUtil.createFirstOnRow(HConstants.EMPTY_START_ROW));
  Cell n0 = scanner0.next();
  Cell n1 = scanner1.next();
  assertTrue(kv1.equals(n0) || kv1.equals(n1));
  assertTrue(kv2.equals(n0)
      || kv2.equals(n1)
      || kv2.equals(scanner0.next())
      || kv2.equals(scanner1.next()));
  assertNull(scanner0.next());
  assertNull(scanner1.next());
}
 
Example 13
@Override
public Cell transformCell(Cell cell) throws IOException {
  // Convert Tephra deletes back into HBase deletes
  if (tx.getVisibilityLevel() == Transaction.VisibilityLevel.SNAPSHOT_ALL) {
    if (DeleteTracker.isFamilyDelete(cell)) {
      return new KeyValue(CellUtil.cloneRow(cell), CellUtil.cloneFamily(cell), null, cell.getTimestamp(),
                          KeyValue.Type.DeleteFamily);
    } else if (isColumnDelete(cell)) {
      // Note: in some cases KeyValue.Type.Delete is used in Delete object,
      // and in some other cases KeyValue.Type.DeleteColumn is used.
      // Since Tephra cannot distinguish between the two, we return KeyValue.Type.DeleteColumn.
      // KeyValue.Type.DeleteColumn makes both CellUtil.isDelete and CellUtil.isDeleteColumns return true, and will
      // work in both cases.
      return new KeyValue(CellUtil.cloneRow(cell), CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell),
                          cell.getTimestamp(), KeyValue.Type.DeleteColumn);
    }
  }
  return cell;
}
 
Example 14
Source Project: hbase   Source File: TestHFile.java    License: Apache License 2.0 6 votes vote down vote up
private Path writeStoreFile() throws IOException {
  Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), "TestHFile");
  HFileContext meta = new HFileContextBuilder().withBlockSize(64 * 1024).build();
  StoreFileWriter sfw =
      new StoreFileWriter.Builder(conf, fs).withOutputDir(storeFileParentDir)
        .withFileContext(meta).build();

  final int rowLen = 32;
  Random RNG = new Random();
  for (int i = 0; i < 1000; ++i) {
    byte[] k = RandomKeyValueUtil.randomOrderedKey(RNG, i);
    byte[] v = RandomKeyValueUtil.randomValue(RNG);
    int cfLen = RNG.nextInt(k.length - rowLen + 1);
    KeyValue kv =
        new KeyValue(k, 0, rowLen, k, rowLen, cfLen, k, rowLen + cfLen,
            k.length - rowLen - cfLen, RNG.nextLong(), generateKeyType(RNG), v, 0, v.length);
    sfw.append(kv);
  }

  sfw.close();
  return sfw.getPath();
}
 
Example 15
Source Project: hbase   Source File: TestStoreScanner.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Ensure that expired delete family markers don't override valid puts
 */
@Test
public void testExpiredDeleteFamily() throws Exception {
  long now = System.currentTimeMillis();
  KeyValue[] kvs = new KeyValue[] {
    new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now-1000,
      KeyValue.Type.DeleteFamily),
    create("R1", "cf", "a", now-10, KeyValue.Type.Put,
      "dont-care"),
  };
  List<KeyValueScanner> scanners = scanFixture(kvs);
  Scan scan = new Scan();
  scan.readVersions(1);
  // scanner with ttl equal to 500
  ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
  try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) {
    List<Cell> results = new ArrayList<>();
    assertEquals(true, scanner.next(results));
    assertEquals(1, results.size());
    assertEquals(kvs[1], results.get(0));
    results.clear();

    assertEquals(false, scanner.next(results));
  }
}
 
Example 16
Source Project: hbase   Source File: TestCompactingToCellFlatMapMemStore.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCountOfCellsAfterFlatteningByScan() throws IOException {
  String[] keys1 = { "A", "B", "C" }; // A, B, C
  addRowsByKeysWith50Cols(memstore, keys1);
  // this should only flatten as there are no duplicates
  ((CompactingMemStore) memstore).flushInMemory();
  while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) {
    Threads.sleep(10);
  }
  List<KeyValueScanner> scanners = memstore.getScanners(Long.MAX_VALUE);
  // seek
  int count = 0;
  for(int i = 0; i < scanners.size(); i++) {
    scanners.get(i).seek(KeyValue.LOWESTKEY);
    while (scanners.get(i).next() != null) {
      count++;
    }
  }
  assertEquals("the count should be ", 150, count);
  for(int i = 0; i < scanners.size(); i++) {
    scanners.get(i).close();
  }
}
 
Example 17
Source Project: learning-hadoop   Source File: JavaHBaseContextSuite.java    License: Apache License 2.0 6 votes vote down vote up
public String call(Result result) throws Exception {
  Iterator<KeyValue> it = result.list().iterator();
  StringBuilder b = new StringBuilder();

  b.append(Bytes.toString(result.getRow()) + ":");

  while (it.hasNext()) {
    KeyValue kv = it.next();
    String q = Bytes.toString(kv.getQualifier());
    if (q.equals("counter")) {
      b.append("(" + Bytes.toString(kv.getQualifier()) + ","
          + Bytes.toLong(kv.getValue()) + ")");
    } else {
      b.append("(" + Bytes.toString(kv.getQualifier()) + ","
          + Bytes.toString(kv.getValue()) + ")");
    }
  }
  return b.toString();
}
 
Example 18
Source Project: warp10-platform   Source File: SlicedRowFilter.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public KeyValue getNextKeyHint(KeyValue currentKV) {
  //hintCount++;
  KeyValue hint = null;
      
  if (this.hintOffset >= 0 && this.hintOffset <= this.rangekeys.length - slicesLength) {
    hint = KeyValueUtil.createFirstOnRow(this.rangekeys, this.hintOffset, (short) (this.bounds[1] + 1));
    minRange = (hintOffset / this.slicesLength) / 2;
  } else {
    done = true;
  }

  /*
  byte[] row = currentKV.getRowArray();
  System.out.println("getNextKeyHint " + encodeHex(row, currentKV.getRowOffset(), currentKV.getRowLength()) + " nvalues = " + this.nvalues + " count = " + this.count + " hintOffset = " + hintOffset);
  if (null != hint) {
    row = hint.getRowArray();
    System.out.println("  hint = " + encodeHex(row, hint.getRowOffset(), hint.getRowLength())); 
  } else {
    System.out.println(" hint = null");
  }
  */
  
  return hint;
}
 
Example 19
Source Project: phoenix   Source File: IndexMemStore.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void rollback(KeyValue kv) {
  if (LOG.isDebugEnabled()) {
    LOG.debug("Rolling back: " + toString(kv));
  }
  // If the key is in the store, delete it
  this.kvset.remove(kv);
  if (LOG.isTraceEnabled()) {
    dump();
  }
}
 
Example 20
Source Project: Halyard   Source File: HBaseSail.java    License: Apache License 2.0 5 votes vote down vote up
protected void addStatementInternal(Resource subj, IRI pred, Value obj, Resource context, long timestamp) throws SailException {
    if (!isWritable()) throw new SailException(tableName + " is read only");
    try {
        for (KeyValue kv : HalyardTableUtils.toKeyValues(subj, pred, obj, context, false, timestamp)) { //serialize the key value pairs relating to the statement in HBase
            put(kv);
        }
    } catch (IOException e) {
        throw new SailException(e);
    }
}
 
Example 21
Source Project: hbase   Source File: TestHFileWriterV3WithDataEncoders.java    License: Apache License 2.0 5 votes vote down vote up
private long scanBlocks(int entryCount, HFileContext context, List<KeyValue> keyValues,
    FSDataInputStream fsdis, FixedFileTrailer trailer, HFileContext meta,
    HFileBlock.FSReader blockReader, int entriesRead, int blocksRead,
    DataBlockEncoder encoder) throws IOException {
  // Scan blocks the way the reader would scan them
  fsdis.seek(0);
  long curBlockPos = 0;
  while (curBlockPos <= trailer.getLastDataBlockOffset()) {
    HFileBlockDecodingContext ctx = blockReader.getBlockDecodingContext();
    HFileBlock block = blockReader.readBlockData(curBlockPos, -1, false, false, true)
      .unpack(context, blockReader);
    Assert.assertEquals(BlockType.ENCODED_DATA, block.getBlockType());
    ByteBuff origBlock = block.getBufferReadOnly();
    int pos = block.headerSize() + DataBlockEncoding.ID_SIZE;
    origBlock.position(pos);
    origBlock.limit(pos + block.getUncompressedSizeWithoutHeader() - DataBlockEncoding.ID_SIZE);
    ByteBuff buf =  origBlock.slice();
    DataBlockEncoder.EncodedSeeker seeker =
      encoder.createSeeker(encoder.newDataBlockDecodingContext(meta));
    seeker.setCurrentBuffer(buf);
    Cell res = seeker.getCell();
    KeyValue kv = keyValues.get(entriesRead);
    Assert.assertEquals(0, CellComparatorImpl.COMPARATOR.compare(res, kv));
    ++entriesRead;
    while(seeker.next()) {
      res = seeker.getCell();
      kv = keyValues.get(entriesRead);
      Assert.assertEquals(0, CellComparatorImpl.COMPARATOR.compare(res, kv));
      ++entriesRead;
    }
    ++blocksRead;
    curBlockPos += block.getOnDiskSizeWithHeader();
  }
  LOG.info("Finished reading: entries={}, blocksRead = {}", entriesRead, blocksRead);
  Assert.assertEquals(entryCount, entriesRead);
  return curBlockPos;
}
 
Example 22
Source Project: kylin-on-parquet-v2   Source File: KeyValueCreator.java    License: Apache License 2.0 5 votes vote down vote up
public KeyValue create(byte[] keyBytes, int keyOffset, int keyLength, byte[] value, int voffset, int vlen) {
    return new KeyValue(keyBytes, keyOffset, keyLength, //
            cfBytes, 0, cfBytes.length, //
            qBytes, 0, qBytes.length, //
            timestamp, KeyValue.Type.Put, //
            value, voffset, vlen);
}
 
Example 23
Source Project: hbase   Source File: TestReplicationSource.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Sanity check that we can move logs around while we are reading
 * from them. Should this test fail, ReplicationSource would have a hard
 * time reading logs that are being archived.
 */
@Test
public void testLogMoving() throws Exception{
  Path logPath = new Path(logDir, "log");
  if (!FS.exists(logDir)) FS.mkdirs(logDir);
  if (!FS.exists(oldLogDir)) FS.mkdirs(oldLogDir);
  WALProvider.Writer writer = WALFactory.createWALWriter(FS, logPath,
      TEST_UTIL.getConfiguration());
  for(int i = 0; i < 3; i++) {
    byte[] b = Bytes.toBytes(Integer.toString(i));
    KeyValue kv = new KeyValue(b,b,b);
    WALEdit edit = new WALEdit();
    edit.add(kv);
    WALKeyImpl key = new WALKeyImpl(b, TableName.valueOf(b), 0, 0,
        HConstants.DEFAULT_CLUSTER_ID);
    writer.append(new WAL.Entry(key, edit));
    writer.sync(false);
  }
  writer.close();

  WAL.Reader reader = WALFactory.createReader(FS, logPath, TEST_UTIL.getConfiguration());
  WAL.Entry entry = reader.next();
  assertNotNull(entry);

  Path oldLogPath = new Path(oldLogDir, "log");
  FS.rename(logPath, oldLogPath);

  entry = reader.next();
  assertNotNull(entry);

  entry = reader.next();
  entry = reader.next();

  assertNull(entry);
  reader.close();
}
 
Example 24
Source Project: kylin-on-parquet-v2   Source File: CubeHFileJob.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void doReduce(RowKeyWritable row, Iterable<KeyValue> kvs, Context context) throws java.io.IOException, InterruptedException {
    for (KeyValue kv : kvs) {
        immutableBytesWritable.set(kv.getKey());
        context.write(immutableBytesWritable, kv);
    }
}
 
Example 25
Source Project: hbase   Source File: StripeStoreFileManager.java    License: Apache License 2.0 5 votes vote down vote up
/** See {@link StoreFileManager#getCandidateFilesForRowKeyBefore(KeyValue)}
 * for details on this methods. */
@Override
public Iterator<HStoreFile> getCandidateFilesForRowKeyBefore(final KeyValue targetKey) {
  KeyBeforeConcatenatedLists result = new KeyBeforeConcatenatedLists();
  // Order matters for this call.
  result.addSublist(state.level0Files);
  if (!state.stripeFiles.isEmpty()) {
    int lastStripeIndex = findStripeForRow(CellUtil.cloneRow(targetKey), false);
    for (int stripeIndex = lastStripeIndex; stripeIndex >= 0; --stripeIndex) {
      result.addSublist(state.stripeFiles.get(stripeIndex));
    }
  }
  return result.iterator();
}
 
Example 26
@Override
public void doMap(KEYIN key, Object value, Context context) throws IOException, InterruptedException {
    Collection<String[]> rowCollection = lookupTableInputFormat.parseMapperInput(value);
    for (String[] row : rowCollection) {
        HBaseRow hBaseRow = encoder.encode(row);

        byte[] rowKey = hBaseRow.getRowKey();
        Map<byte[], byte[]> qualifierValMap = hBaseRow.getQualifierValMap();
        outputKey.set(rowKey);
        for (Entry<byte[], byte[]> qualifierValEntry : qualifierValMap.entrySet()) {
            KeyValue outputValue = createKeyValue(rowKey, qualifierValEntry.getKey(), qualifierValEntry.getValue());
            context.write(outputKey, outputValue);
        }
    }
}
 
Example 27
Source Project: hbase   Source File: IntegrationTestImportTsv.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Verify the data described by <code>simple_tsv</code> matches
 * <code>simple_expected</code>.
 */
protected void doLoadIncrementalHFiles(Path hfiles, TableName tableName)
    throws Exception {

  String[] args = { hfiles.toString(), tableName.getNameAsString() };
  LOG.info(format("Running LoadIncrememntalHFiles with args: %s", Arrays.asList(args)));
  assertEquals("Loading HFiles failed.", 0,
    ToolRunner.run(new BulkLoadHFilesTool(getConf()), args));

  Table table = null;
  Scan scan = new Scan() {{
    setCacheBlocks(false);
    setCaching(1000);
  }};
  try {
    table = util.getConnection().getTable(tableName);
    Iterator<Result> resultsIt = table.getScanner(scan).iterator();
    Iterator<KeyValue> expectedIt = simple_expected.iterator();
    while (resultsIt.hasNext() && expectedIt.hasNext()) {
      Result r = resultsIt.next();
      for (Cell actual : r.rawCells()) {
        assertTrue(
          "Ran out of expected values prematurely!",
          expectedIt.hasNext());
        KeyValue expected = expectedIt.next();
        assertEquals("Scan produced surprising result", 0,
          CellComparator.getInstance().compare(expected, actual));
      }
    }
    assertFalse("Did not consume all expected values.", expectedIt.hasNext());
    assertFalse("Did not consume all scan results.", resultsIt.hasNext());
  } finally {
    if (null != table) table.close();
  }
}
 
Example 28
Source Project: hbase   Source File: TestSingleColumnValueFilter.java    License: Apache License 2.0 5 votes vote down vote up
private void regexPatternFilterTests(Filter filter)
    throws Exception {
  KeyValue cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER,
    FULLSTRING_1);
  assertTrue("regexTrue",
    filter.filterCell(cell) == Filter.ReturnCode.INCLUDE);
  byte[] buffer = cell.getBuffer();
  Cell c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length);
  assertTrue("regexTrue", filter.filterCell(c) == Filter.ReturnCode.INCLUDE);
  assertFalse("regexFilterAllRemaining", filter.filterAllRemaining());
  assertFalse("regexFilterNotNull", filter.filterRow());
}
 
Example 29
Source Project: hbase   Source File: TestHStoreFile.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Test for HBASE-8012
 */
@Test
public void testReseek() throws Exception {
  // write the file
  Path f = new Path(ROOT_DIR, name.getMethodName());
  HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
  // Make a store file and write data to it.
  StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs).withFilePath(f)
    .withFileContext(meta).build();

  writeStoreFile(writer);
  writer.close();

  ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, f).build();
  HFileInfo fileInfo = new HFileInfo(context, conf);
  StoreFileReader reader =
    new StoreFileReader(context, fileInfo, cacheConf, new AtomicInteger(0), conf);
  fileInfo.initMetaAndIndex(reader.getHFileReader());

  // Now do reseek with empty KV to position to the beginning of the file

  KeyValue k = KeyValueUtil.createFirstOnRow(HConstants.EMPTY_BYTE_ARRAY);
  StoreFileScanner s = getStoreFileScanner(reader, false, false);
  s.reseek(k);

  assertNotNull("Intial reseek should position at the beginning of the file", s.peek());
}
 
Example 30
Source Project: hbase   Source File: WALEdit.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @return A Marker WALEdit that has <code>c</code> serialized as its value
 */
public static WALEdit createCompaction(final RegionInfo hri, final CompactionDescriptor c) {
  byte [] pbbytes = c.toByteArray();
  KeyValue kv = new KeyValue(getRowForRegion(hri), METAFAMILY, COMPACTION,
    EnvironmentEdgeManager.currentTime(), pbbytes);
  return new WALEdit().add(kv, METAFAMILY); //replication scope null so this won't be replicated
}