org.apache.hadoop.hbase.regionserver.wal.WALEdit Java Examples

The following examples show how to use org.apache.hadoop.hbase.regionserver.wal.WALEdit. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: IndexerDemo.java    From hbase-secondary-index with GNU General Public License v3.0 6 votes vote down vote up
@Override
@SuppressWarnings("rawtypes")
public void postPut(final ObserverContext e, final Put put,
		final WALEdit edit, final boolean writeToWAL) throws IOException {

	byte[][] colkey = KeyValue.parseColumn(Bytes.toBytes(inputColumn));
	if (colkey.length > 1) {
		List kvList = put.get(colkey[0], colkey[1]);
		Iterator kvl = kvList.iterator();

		while (kvl.hasNext()) {
			KeyValue kv = (KeyValue) kvl.next();
			Put indexPut = new Put(kv.getValue());
			colkey = KeyValue.parseColumn(Bytes.toBytes(indexColumn));
			indexPut.add(colkey[0], colkey[1], kv.getRow());
			table.put(indexPut);
		}
	}
}
 
Example #2
Source File: Indexer.java    From phoenix with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
public void preDeleteWithExceptions(ObserverContext<RegionCoprocessorEnvironment> e,
    Delete delete, WALEdit edit, boolean writeToWAL) throws Exception {
  // if we are making the update as part of a batch, we need to add in a batch marker so the WAL
  // is retained
  if (this.builder.getBatchId(delete) != null) {
    edit.add(BATCH_MARKER);
    return;
  }

  // get the mapping for index column -> target index table
  Collection<Pair<Mutation, byte[]>> indexUpdates = this.builder.getIndexUpdate(delete);

  if (doPre(indexUpdates, edit, writeToWAL)) {
    takeUpdateLock("delete");
  }
}
 
Example #3
Source File: Indexer.java    From phoenix with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
@Override
public void preDelete(ObserverContext<RegionCoprocessorEnvironment> e, Delete delete,
    WALEdit edit, boolean writeToWAL) throws IOException {
    if (this.disabled) {
        super.preDelete(e, delete, edit, writeToWAL);
        return;
      }
  try {
    preDeleteWithExceptions(e, delete, edit, writeToWAL);
    return;
  } catch (Throwable t) {
    rethrowIndexingException(t);
  }
  throw new RuntimeException(
      "Somehow didn't return an index update but also didn't propagate the failure to the client!");
}
 
Example #4
Source File: Indexer.java    From phoenix with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
@Override
public void preWALRestore(ObserverContext<RegionCoprocessorEnvironment> env, HRegionInfo info,
    HLogKey logKey, WALEdit logEdit) throws IOException {
    if (this.disabled) {
        super.preWALRestore(env, info, logKey, logEdit);
        return;
      }
  // TODO check the regions in transition. If the server on which the region lives is this one,
  // then we should rety that write later in postOpen.
  // we might be able to get even smarter here and pre-split the edits that are server-local
  // into their own recovered.edits file. This then lets us do a straightforward recovery of each
  // region (and more efficiently as we aren't writing quite as hectically from this one place).

  /*
   * Basically, we let the index regions recover for a little while long before retrying in the
   * hopes they come up before the primary table finishes.
   */
  Collection<Pair<Mutation, byte[]>> indexUpdates = extractIndexUpdate(logEdit);
  recoveryWriter.writeAndKillYourselfOnFailure(indexUpdates);
}
 
Example #5
Source File: EnrichmentCoprocessor.java    From metron with Apache License 2.0 6 votes vote down vote up
@Override
public void postPut(ObserverContext<RegionCoprocessorEnvironment> e, Put put, WALEdit edit,
    Durability durability) throws IOException {
  LOG.trace("enrichment coprocessor postPut call begin");
  try {
    LOG.trace("Extracting enrichment type from rowkey");
    String type = getEnrichmentType(put);
    // Make the value json so we can add metadata at a later time, if desired.
    final String metadata = "{}";
    LOG.trace("Enrichment type '{}' extracted from rowkey", type);
    addToCache(type, metadata);
  } catch (Throwable t) {
    LOG.warn("Exception occurred while processing Put operation in coprocessor", t);
    // Anything other than an IOException will cause the coprocessor to be disabled.
    throw new IOException("Error occurred while processing enrichment Put.", t);
  }
  LOG.trace("enrichment coprocessor postPut call complete");
}
 
Example #6
Source File: THLog.java    From hbase-secondary-index with GNU General Public License v3.0 6 votes vote down vote up
/**
 * Write a transactional state to the log for a commit request.
 * 
 * @param regionInfo
 * @param update
 * @param transactionId
 * @throws IOException
 */
private void appendCommitRequest(final HRegionInfo regionInfo,
		final long now, final TransactionState transactionState)
		throws IOException {

	@SuppressWarnings("deprecation")
	THLogKey key = new THLogKey(regionInfo.getRegionName(), regionInfo
			.getTableDesc().getName(), -1, now,
			THLogKey.TrxOp.COMMIT_REQUEST,
			transactionState.getTransactionId());

	WALEdit e = new WALEdit();

	for (WriteAction write : transactionState.getWriteOrdering()) {
		for (KeyValue value : write.getKeyValues()) {
			e.add(value);
		}
	}

	super.append(regionInfo, key, e, null, false);

}
 
Example #7
Source File: Indexer.java    From phoenix with Apache License 2.0 6 votes vote down vote up
@Override
public void preWALRestore(ObserverContext<RegionCoprocessorEnvironment> env, HRegionInfo info,
    HLogKey logKey, WALEdit logEdit) throws IOException {
    if (this.disabled) {
        super.preWALRestore(env, info, logKey, logEdit);
        return;
      }
  // TODO check the regions in transition. If the server on which the region lives is this one,
  // then we should rety that write later in postOpen.
  // we might be able to get even smarter here and pre-split the edits that are server-local
  // into their own recovered.edits file. This then lets us do a straightforward recovery of each
  // region (and more efficiently as we aren't writing quite as hectically from this one place).

  /*
   * Basically, we let the index regions recover for a little while long before retrying in the
   * hopes they come up before the primary table finishes.
   */
  Collection<Pair<Mutation, byte[]>> indexUpdates = extractIndexUpdate(logEdit);
  recoveryWriter.write(indexUpdates);
}
 
Example #8
Source File: HbaseSolrIndexCoprocesser.java    From hbase-increment-index with MIT License 6 votes vote down vote up
@Override
public void postPut(ObserverContext<RegionCoprocessorEnvironment> e, Put put, WALEdit edit, Durability durability) throws IOException {
    String rowkey = Bytes.toString(put.getRow());//得到rowkey
    SolrInputDocument doc =new SolrInputDocument();//实例化索引Doc
    doc.addField(config.getString("solr_hbase_rowkey_name"),rowkey);//添加主键
    for(String cf:config.getString("hbase_column_family").split(",")) {//遍历所有的列簇
        List<Cell> cells = put.getFamilyCellMap().get(Bytes.toBytes(cf));
        if(cells==null||cells.isEmpty()) continue; // 跳过取值为空或null的数据
        for (Cell kv : cells ) {
            String name=Bytes.toString(CellUtil.cloneQualifier(kv));//获取列名
            String value=Bytes.toString(kv.getValueArray());//获取列值 or CellUtil.cloneValue(kv)
            doc.addField(name,value);//添加到索引doc里面
        }
    }
    //发送数据到本地缓存
    SolrIndexTools.addDoc(doc);
}
 
Example #9
Source File: Indexer.java    From phoenix with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
/**
 * Search the {@link WALEdit} for the first {@link IndexedKeyValue} present
 * @param edit {@link WALEdit}
 * @return the first {@link IndexedKeyValue} in the {@link WALEdit} or <tt>null</tt> if not
 *         present
 */
private IndexedKeyValue getFirstIndexedKeyValue(WALEdit edit) {
  for (KeyValue kv : edit.getKeyValues()) {
    if (kv instanceof IndexedKeyValue) {
      return (IndexedKeyValue) kv;
    }
  }
  return null;
}
 
Example #10
Source File: TransactionAwareHTableTest.java    From phoenix-tephra with Apache License 2.0 5 votes vote down vote up
@Override
public void preDelete(final ObserverContext<RegionCoprocessorEnvironment> c,
                      final Delete delete, final WALEdit edit,
                      final Durability durability) throws IOException {
  if (delete.getAttribute(TEST_ATTRIBUTE) == null) {
    throw new DoNotRetryIOException("Delete should preserve attributes");
  }
  if (delete.getDurability() != Durability.USE_DEFAULT) {
    throw new DoNotRetryIOException("Durability is not propagated correctly");
  }
}
 
Example #11
Source File: Indexer.java    From phoenix with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
/**
 * Extract the index updates from the WAL Edit
 * @param edit to search for index updates
 * @return the mutations to apply to the index tables
 */
private Collection<Pair<Mutation, byte[]>> extractIndexUpdate(WALEdit edit) {
  Collection<Pair<Mutation, byte[]>> indexUpdates = new ArrayList<Pair<Mutation, byte[]>>();
  for (KeyValue kv : edit.getKeyValues()) {
    if (kv instanceof IndexedKeyValue) {
      IndexedKeyValue ikv = (IndexedKeyValue) kv;
      indexUpdates.add(new Pair<Mutation, byte[]>(ikv.getMutation(), ikv.getIndexTable()));
    }
  }

  return indexUpdates;
}
 
Example #12
Source File: RegionObserverTest.java    From BigData-In-Practice with Apache License 2.0 5 votes vote down vote up
/**
 * 2. 不能直接删除unDeleteCol    删除countCol的时候将unDeleteCol一同删除
 */
@Override
public void preDelete(ObserverContext<RegionCoprocessorEnvironment> e, Delete delete,
                      WALEdit edit,
                      Durability durability) throws IOException {
    //判断是否操作cf列族
    List<Cell> cells = delete.getFamilyCellMap().get(columnFamily);
    if (cells == null || cells.size() == 0) {
        return;
    }

    boolean deleteFlag = false;
    for (Cell cell : cells) {
        byte[] qualifier = CellUtil.cloneQualifier(cell);

        if (Arrays.equals(qualifier, unDeleteCol)) {
            throw new IOException("can not delete unDel column");
        }

        if (Arrays.equals(qualifier, countCol)) {
            deleteFlag = true;
        }
    }

    if (deleteFlag) {
        delete.addColumn(columnFamily, unDeleteCol);
    }
}
 
Example #13
Source File: HbaseSolrIndexCoprocesser.java    From hbase-increment-index with MIT License 5 votes vote down vote up
@Override
public void postDelete(ObserverContext<RegionCoprocessorEnvironment> e, Delete delete, WALEdit edit, Durability durability) throws IOException {
    //得到rowkey
    String rowkey = Bytes.toString(delete.getRow());
    //发送数据本地缓存
    SolrIndexTools.delDoc(rowkey);
}
 
Example #14
Source File: TransactionAwareHTableTest.java    From phoenix-tephra with Apache License 2.0 5 votes vote down vote up
@Override
public void preDelete(final ObserverContext<RegionCoprocessorEnvironment> c,
                      final Delete delete, final WALEdit edit,
                      final Durability durability) throws IOException {
  if (delete.getAttribute(TEST_ATTRIBUTE) == null) {
    throw new DoNotRetryIOException("Delete should preserve attributes");
  }
  if (delete.getDurability() != Durability.USE_DEFAULT) {
    throw new DoNotRetryIOException("Durability is not propagated correctly");
  }
}
 
Example #15
Source File: TransactionAwareHTableTest.java    From phoenix-tephra with Apache License 2.0 5 votes vote down vote up
@Override
public void prePut(final ObserverContext<RegionCoprocessorEnvironment> c,
                   final Put put, final WALEdit edit,
                   final Durability durability) throws IOException {
  if (put.getAttribute(TEST_ATTRIBUTE) == null) {
    throw new DoNotRetryIOException("Put should preserve attributes");
  }
  if (put.getDurability() != Durability.USE_DEFAULT) {
    throw new DoNotRetryIOException("Durability is not propagated correctly");
  }
}
 
Example #16
Source File: TransactionAwareHTableTest.java    From phoenix-tephra with Apache License 2.0 5 votes vote down vote up
@Override
public void prePut(final ObserverContext<RegionCoprocessorEnvironment> c,
                   final Put put, final WALEdit edit,
                   final Durability durability) throws IOException {
  if (put.getAttribute(TEST_ATTRIBUTE) == null) {
    throw new DoNotRetryIOException("Put should preserve attributes");
  }
  if (put.getDurability() != Durability.USE_DEFAULT) {
    throw new DoNotRetryIOException("Durability is not propagated correctly");
  }
}
 
Example #17
Source File: TransactionAwareHTableTest.java    From phoenix-tephra with Apache License 2.0 5 votes vote down vote up
@Override
public void prePut(final ObserverContext<RegionCoprocessorEnvironment> c,
                   final Put put, final WALEdit edit,
                   final Durability durability) throws IOException {
  if (put.getAttribute(TEST_ATTRIBUTE) == null) {
    throw new DoNotRetryIOException("Put should preserve attributes");
  }
  if (put.getDurability() != Durability.USE_DEFAULT) {
    throw new DoNotRetryIOException("Durability is not propagated correctly");
  }
}
 
Example #18
Source File: Indexer.java    From phoenix with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
private void doPost(WALEdit edit, Mutation m, boolean writeToWAL) throws IOException {
  try {
    doPostWithExceptions(edit, m, writeToWAL);
    return;
  } catch (Throwable e) {
    rethrowIndexingException(e);
  }
  throw new RuntimeException(
      "Somehow didn't complete the index update, but didn't return succesfully either!");
}
 
Example #19
Source File: Indexer.java    From phoenix with Apache License 2.0 5 votes vote down vote up
/**
 * Add the index updates to the WAL, or write to the index table, if the WAL has been disabled
 * @return <tt>true</tt> if the WAL has been updated.
 * @throws IOException
 */
private boolean doPre(Collection<Pair<Mutation, byte[]>> indexUpdates, final WALEdit edit,
    final Durability durability) throws IOException {
  // no index updates, so we are done
  if (indexUpdates == null || indexUpdates.size() == 0) {
    return false;
  }

  // if writing to wal is disabled, we never see the WALEdit updates down the way, so do the index
  // update right away
  if (durability == Durability.SKIP_WAL) {
    try {
      this.writer.write(indexUpdates);
      return false;
    } catch (Throwable e) {
      LOG.error("Failed to update index with entries:" + indexUpdates, e);
      IndexManagementUtil.rethrowIndexingException(e);
    }
  }

  // we have all the WAL durability, so we just update the WAL entry and move on
  for (Pair<Mutation, byte[]> entry : indexUpdates) {
    edit.add(new IndexedKeyValue(entry.getSecond(), entry.getFirst()));
  }

  return true;
}
 
Example #20
Source File: Indexer.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@Override
public void postPut(ObserverContext<RegionCoprocessorEnvironment> e, Put put, WALEdit edit,
    final Durability durability) throws IOException {
    if (this.disabled) {
    super.postPut(e, put, edit, durability);
        return;
      }
  doPost(edit, put, durability);
}
 
Example #21
Source File: Indexer.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@Override
public void postDelete(ObserverContext<RegionCoprocessorEnvironment> e, Delete delete,
    WALEdit edit, final Durability durability) throws IOException {
    if (this.disabled) {
    super.postDelete(e, delete, edit, durability);
        return;
      }
  doPost(edit, delete, durability);
}
 
Example #22
Source File: Indexer.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@Override
public void postBatchMutate(ObserverContext<RegionCoprocessorEnvironment> c,
    MiniBatchOperationInProgress<Mutation> miniBatchOp) throws IOException {
    if (this.disabled) {
        super.postBatchMutate(c, miniBatchOp);
        return;
      }
  this.builder.batchCompleted(miniBatchOp);

  //each batch operation, only the first one will have anything useful, so we can just grab that
  Mutation mutation = miniBatchOp.getOperation(0);
  WALEdit edit = miniBatchOp.getWalEdit(0);
  doPost(edit, mutation, mutation.getDurability());
}
 
Example #23
Source File: Indexer.java    From phoenix with Apache License 2.0 5 votes vote down vote up
private void doPost(WALEdit edit, Mutation m, final Durability durability) throws IOException {
  try {
    doPostWithExceptions(edit, m, durability);
    return;
  } catch (Throwable e) {
    rethrowIndexingException(e);
  }
  throw new RuntimeException(
      "Somehow didn't complete the index update, but didn't return succesfully either!");
}
 
Example #24
Source File: Indexer.java    From phoenix with Apache License 2.0 5 votes vote down vote up
/**
 * Search the {@link WALEdit} for the first {@link IndexedKeyValue} present
 * @param edit {@link WALEdit}
 * @return the first {@link IndexedKeyValue} in the {@link WALEdit} or <tt>null</tt> if not
 *         present
 */
private IndexedKeyValue getFirstIndexedKeyValue(WALEdit edit) {
  for (Cell kv : edit.getCells()) {
    if (kv instanceof IndexedKeyValue) {
      return (IndexedKeyValue) kv;
    }
  }
  return null;
}
 
Example #25
Source File: Indexer.java    From phoenix with Apache License 2.0 5 votes vote down vote up
/**
 * Extract the index updates from the WAL Edit
 * @param edit to search for index updates
 * @return the mutations to apply to the index tables
 */
private Collection<Pair<Mutation, byte[]>> extractIndexUpdate(WALEdit edit) {
  Collection<Pair<Mutation, byte[]>> indexUpdates = new ArrayList<Pair<Mutation, byte[]>>();
  for (Cell kv : edit.getCells()) {
    if (kv instanceof IndexedKeyValue) {
      IndexedKeyValue ikv = (IndexedKeyValue) kv;
      indexUpdates.add(new Pair<Mutation, byte[]>(ikv.getMutation(), ikv.getIndexTable()));
    }
  }

  return indexUpdates;
}
 
Example #26
Source File: TestWALRecoveryCaching.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@Override
public void preWALRestore(ObserverContext<RegionCoprocessorEnvironment> env, HRegionInfo info,
    HLogKey logKey, WALEdit logEdit) throws IOException {
  try {
    LOG.debug("Restoring logs for index table");
    if (allowIndexTableToRecover != null) {
      allowIndexTableToRecover.await();
      LOG.debug("Completed index table recovery wait latch");
    }
  } catch (InterruptedException e) {
    Assert.fail("Should not be interrupted while waiting to allow the index to restore WALs.");
  }
}
 
Example #27
Source File: Indexer.java    From phoenix with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
@Override
public void prePut(final ObserverContext<RegionCoprocessorEnvironment> c, final Put put,
    final WALEdit edit, final boolean writeToWAL) throws IOException {
    if (this.disabled) {
        super.prePut(c, put, edit, writeToWAL);
        return;
      }
  // just have to add a batch marker to the WALEdit so we get the edit again in the batch
  // processing step. We let it throw an exception here because something terrible has happened.
  edit.add(BATCH_MARKER);
}
 
Example #28
Source File: Indexer.java    From phoenix with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
/**
 * Add the index updates to the WAL, or write to the index table, if the WAL has been disabled
 * @return <tt>true</tt> if the WAL has been updated.
 * @throws IOException
 */
private boolean doPre(Collection<Pair<Mutation, byte[]>> indexUpdates, final WALEdit edit,
    final boolean writeToWAL) throws IOException {
  // no index updates, so we are done
  if (indexUpdates == null || indexUpdates.size() == 0) {
    return false;
  }

  // if writing to wal is disabled, we never see the WALEdit updates down the way, so do the index
  // update right away
  if (!writeToWAL) {
    try {
      this.writer.write(indexUpdates);
      return false;
    } catch (Throwable e) {
      LOG.error("Failed to update index with entries:" + indexUpdates, e);
      IndexManagementUtil.rethrowIndexingException(e);
    }
  }

  // we have all the WAL durability, so we just update the WAL entry and move on
  for (Pair<Mutation, byte[]> entry : indexUpdates) {
    edit.add(new IndexedKeyValue(entry.getSecond(), entry.getFirst()));
  }

  return true;
}
 
Example #29
Source File: Indexer.java    From phoenix with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
@Override
public void postPut(ObserverContext<RegionCoprocessorEnvironment> e, Put put, WALEdit edit,
    boolean writeToWAL) throws IOException {
    if (this.disabled) {
        super.postPut(e, put, edit, writeToWAL);
        return;
      }
  doPost(edit, put, writeToWAL);
}
 
Example #30
Source File: Indexer.java    From phoenix with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
@Override
public void postDelete(ObserverContext<RegionCoprocessorEnvironment> e, Delete delete,
    WALEdit edit, boolean writeToWAL) throws IOException {
    if (this.disabled) {
        super.postDelete(e, delete, edit, writeToWAL);
        return;
      }
  doPost(edit,delete, writeToWAL);
}