org.apache.hadoop.hbase.regionserver.wal.WALEdit Java Examples
The following examples show how to use
org.apache.hadoop.hbase.regionserver.wal.WALEdit.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: IndexerDemo.java From hbase-secondary-index with GNU General Public License v3.0 | 6 votes |
@Override @SuppressWarnings("rawtypes") public void postPut(final ObserverContext e, final Put put, final WALEdit edit, final boolean writeToWAL) throws IOException { byte[][] colkey = KeyValue.parseColumn(Bytes.toBytes(inputColumn)); if (colkey.length > 1) { List kvList = put.get(colkey[0], colkey[1]); Iterator kvl = kvList.iterator(); while (kvl.hasNext()) { KeyValue kv = (KeyValue) kvl.next(); Put indexPut = new Put(kv.getValue()); colkey = KeyValue.parseColumn(Bytes.toBytes(indexColumn)); indexPut.add(colkey[0], colkey[1], kv.getRow()); table.put(indexPut); } } }
Example #2
Source File: Indexer.java From phoenix with BSD 3-Clause "New" or "Revised" License | 6 votes |
public void preDeleteWithExceptions(ObserverContext<RegionCoprocessorEnvironment> e, Delete delete, WALEdit edit, boolean writeToWAL) throws Exception { // if we are making the update as part of a batch, we need to add in a batch marker so the WAL // is retained if (this.builder.getBatchId(delete) != null) { edit.add(BATCH_MARKER); return; } // get the mapping for index column -> target index table Collection<Pair<Mutation, byte[]>> indexUpdates = this.builder.getIndexUpdate(delete); if (doPre(indexUpdates, edit, writeToWAL)) { takeUpdateLock("delete"); } }
Example #3
Source File: Indexer.java From phoenix with BSD 3-Clause "New" or "Revised" License | 6 votes |
@Override public void preDelete(ObserverContext<RegionCoprocessorEnvironment> e, Delete delete, WALEdit edit, boolean writeToWAL) throws IOException { if (this.disabled) { super.preDelete(e, delete, edit, writeToWAL); return; } try { preDeleteWithExceptions(e, delete, edit, writeToWAL); return; } catch (Throwable t) { rethrowIndexingException(t); } throw new RuntimeException( "Somehow didn't return an index update but also didn't propagate the failure to the client!"); }
Example #4
Source File: Indexer.java From phoenix with BSD 3-Clause "New" or "Revised" License | 6 votes |
@Override public void preWALRestore(ObserverContext<RegionCoprocessorEnvironment> env, HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException { if (this.disabled) { super.preWALRestore(env, info, logKey, logEdit); return; } // TODO check the regions in transition. If the server on which the region lives is this one, // then we should rety that write later in postOpen. // we might be able to get even smarter here and pre-split the edits that are server-local // into their own recovered.edits file. This then lets us do a straightforward recovery of each // region (and more efficiently as we aren't writing quite as hectically from this one place). /* * Basically, we let the index regions recover for a little while long before retrying in the * hopes they come up before the primary table finishes. */ Collection<Pair<Mutation, byte[]>> indexUpdates = extractIndexUpdate(logEdit); recoveryWriter.writeAndKillYourselfOnFailure(indexUpdates); }
Example #5
Source File: EnrichmentCoprocessor.java From metron with Apache License 2.0 | 6 votes |
@Override public void postPut(ObserverContext<RegionCoprocessorEnvironment> e, Put put, WALEdit edit, Durability durability) throws IOException { LOG.trace("enrichment coprocessor postPut call begin"); try { LOG.trace("Extracting enrichment type from rowkey"); String type = getEnrichmentType(put); // Make the value json so we can add metadata at a later time, if desired. final String metadata = "{}"; LOG.trace("Enrichment type '{}' extracted from rowkey", type); addToCache(type, metadata); } catch (Throwable t) { LOG.warn("Exception occurred while processing Put operation in coprocessor", t); // Anything other than an IOException will cause the coprocessor to be disabled. throw new IOException("Error occurred while processing enrichment Put.", t); } LOG.trace("enrichment coprocessor postPut call complete"); }
Example #6
Source File: THLog.java From hbase-secondary-index with GNU General Public License v3.0 | 6 votes |
/** * Write a transactional state to the log for a commit request. * * @param regionInfo * @param update * @param transactionId * @throws IOException */ private void appendCommitRequest(final HRegionInfo regionInfo, final long now, final TransactionState transactionState) throws IOException { @SuppressWarnings("deprecation") THLogKey key = new THLogKey(regionInfo.getRegionName(), regionInfo .getTableDesc().getName(), -1, now, THLogKey.TrxOp.COMMIT_REQUEST, transactionState.getTransactionId()); WALEdit e = new WALEdit(); for (WriteAction write : transactionState.getWriteOrdering()) { for (KeyValue value : write.getKeyValues()) { e.add(value); } } super.append(regionInfo, key, e, null, false); }
Example #7
Source File: Indexer.java From phoenix with Apache License 2.0 | 6 votes |
@Override public void preWALRestore(ObserverContext<RegionCoprocessorEnvironment> env, HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException { if (this.disabled) { super.preWALRestore(env, info, logKey, logEdit); return; } // TODO check the regions in transition. If the server on which the region lives is this one, // then we should rety that write later in postOpen. // we might be able to get even smarter here and pre-split the edits that are server-local // into their own recovered.edits file. This then lets us do a straightforward recovery of each // region (and more efficiently as we aren't writing quite as hectically from this one place). /* * Basically, we let the index regions recover for a little while long before retrying in the * hopes they come up before the primary table finishes. */ Collection<Pair<Mutation, byte[]>> indexUpdates = extractIndexUpdate(logEdit); recoveryWriter.write(indexUpdates); }
Example #8
Source File: HbaseSolrIndexCoprocesser.java From hbase-increment-index with MIT License | 6 votes |
@Override public void postPut(ObserverContext<RegionCoprocessorEnvironment> e, Put put, WALEdit edit, Durability durability) throws IOException { String rowkey = Bytes.toString(put.getRow());//得到rowkey SolrInputDocument doc =new SolrInputDocument();//实例化索引Doc doc.addField(config.getString("solr_hbase_rowkey_name"),rowkey);//添加主键 for(String cf:config.getString("hbase_column_family").split(",")) {//遍历所有的列簇 List<Cell> cells = put.getFamilyCellMap().get(Bytes.toBytes(cf)); if(cells==null||cells.isEmpty()) continue; // 跳过取值为空或null的数据 for (Cell kv : cells ) { String name=Bytes.toString(CellUtil.cloneQualifier(kv));//获取列名 String value=Bytes.toString(kv.getValueArray());//获取列值 or CellUtil.cloneValue(kv) doc.addField(name,value);//添加到索引doc里面 } } //发送数据到本地缓存 SolrIndexTools.addDoc(doc); }
Example #9
Source File: Indexer.java From phoenix with BSD 3-Clause "New" or "Revised" License | 5 votes |
/** * Search the {@link WALEdit} for the first {@link IndexedKeyValue} present * @param edit {@link WALEdit} * @return the first {@link IndexedKeyValue} in the {@link WALEdit} or <tt>null</tt> if not * present */ private IndexedKeyValue getFirstIndexedKeyValue(WALEdit edit) { for (KeyValue kv : edit.getKeyValues()) { if (kv instanceof IndexedKeyValue) { return (IndexedKeyValue) kv; } } return null; }
Example #10
Source File: TransactionAwareHTableTest.java From phoenix-tephra with Apache License 2.0 | 5 votes |
@Override public void preDelete(final ObserverContext<RegionCoprocessorEnvironment> c, final Delete delete, final WALEdit edit, final Durability durability) throws IOException { if (delete.getAttribute(TEST_ATTRIBUTE) == null) { throw new DoNotRetryIOException("Delete should preserve attributes"); } if (delete.getDurability() != Durability.USE_DEFAULT) { throw new DoNotRetryIOException("Durability is not propagated correctly"); } }
Example #11
Source File: Indexer.java From phoenix with BSD 3-Clause "New" or "Revised" License | 5 votes |
/** * Extract the index updates from the WAL Edit * @param edit to search for index updates * @return the mutations to apply to the index tables */ private Collection<Pair<Mutation, byte[]>> extractIndexUpdate(WALEdit edit) { Collection<Pair<Mutation, byte[]>> indexUpdates = new ArrayList<Pair<Mutation, byte[]>>(); for (KeyValue kv : edit.getKeyValues()) { if (kv instanceof IndexedKeyValue) { IndexedKeyValue ikv = (IndexedKeyValue) kv; indexUpdates.add(new Pair<Mutation, byte[]>(ikv.getMutation(), ikv.getIndexTable())); } } return indexUpdates; }
Example #12
Source File: RegionObserverTest.java From BigData-In-Practice with Apache License 2.0 | 5 votes |
/** * 2. 不能直接删除unDeleteCol 删除countCol的时候将unDeleteCol一同删除 */ @Override public void preDelete(ObserverContext<RegionCoprocessorEnvironment> e, Delete delete, WALEdit edit, Durability durability) throws IOException { //判断是否操作cf列族 List<Cell> cells = delete.getFamilyCellMap().get(columnFamily); if (cells == null || cells.size() == 0) { return; } boolean deleteFlag = false; for (Cell cell : cells) { byte[] qualifier = CellUtil.cloneQualifier(cell); if (Arrays.equals(qualifier, unDeleteCol)) { throw new IOException("can not delete unDel column"); } if (Arrays.equals(qualifier, countCol)) { deleteFlag = true; } } if (deleteFlag) { delete.addColumn(columnFamily, unDeleteCol); } }
Example #13
Source File: HbaseSolrIndexCoprocesser.java From hbase-increment-index with MIT License | 5 votes |
@Override public void postDelete(ObserverContext<RegionCoprocessorEnvironment> e, Delete delete, WALEdit edit, Durability durability) throws IOException { //得到rowkey String rowkey = Bytes.toString(delete.getRow()); //发送数据本地缓存 SolrIndexTools.delDoc(rowkey); }
Example #14
Source File: TransactionAwareHTableTest.java From phoenix-tephra with Apache License 2.0 | 5 votes |
@Override public void preDelete(final ObserverContext<RegionCoprocessorEnvironment> c, final Delete delete, final WALEdit edit, final Durability durability) throws IOException { if (delete.getAttribute(TEST_ATTRIBUTE) == null) { throw new DoNotRetryIOException("Delete should preserve attributes"); } if (delete.getDurability() != Durability.USE_DEFAULT) { throw new DoNotRetryIOException("Durability is not propagated correctly"); } }
Example #15
Source File: TransactionAwareHTableTest.java From phoenix-tephra with Apache License 2.0 | 5 votes |
@Override public void prePut(final ObserverContext<RegionCoprocessorEnvironment> c, final Put put, final WALEdit edit, final Durability durability) throws IOException { if (put.getAttribute(TEST_ATTRIBUTE) == null) { throw new DoNotRetryIOException("Put should preserve attributes"); } if (put.getDurability() != Durability.USE_DEFAULT) { throw new DoNotRetryIOException("Durability is not propagated correctly"); } }
Example #16
Source File: TransactionAwareHTableTest.java From phoenix-tephra with Apache License 2.0 | 5 votes |
@Override public void prePut(final ObserverContext<RegionCoprocessorEnvironment> c, final Put put, final WALEdit edit, final Durability durability) throws IOException { if (put.getAttribute(TEST_ATTRIBUTE) == null) { throw new DoNotRetryIOException("Put should preserve attributes"); } if (put.getDurability() != Durability.USE_DEFAULT) { throw new DoNotRetryIOException("Durability is not propagated correctly"); } }
Example #17
Source File: TransactionAwareHTableTest.java From phoenix-tephra with Apache License 2.0 | 5 votes |
@Override public void prePut(final ObserverContext<RegionCoprocessorEnvironment> c, final Put put, final WALEdit edit, final Durability durability) throws IOException { if (put.getAttribute(TEST_ATTRIBUTE) == null) { throw new DoNotRetryIOException("Put should preserve attributes"); } if (put.getDurability() != Durability.USE_DEFAULT) { throw new DoNotRetryIOException("Durability is not propagated correctly"); } }
Example #18
Source File: Indexer.java From phoenix with BSD 3-Clause "New" or "Revised" License | 5 votes |
private void doPost(WALEdit edit, Mutation m, boolean writeToWAL) throws IOException { try { doPostWithExceptions(edit, m, writeToWAL); return; } catch (Throwable e) { rethrowIndexingException(e); } throw new RuntimeException( "Somehow didn't complete the index update, but didn't return succesfully either!"); }
Example #19
Source File: Indexer.java From phoenix with Apache License 2.0 | 5 votes |
/** * Add the index updates to the WAL, or write to the index table, if the WAL has been disabled * @return <tt>true</tt> if the WAL has been updated. * @throws IOException */ private boolean doPre(Collection<Pair<Mutation, byte[]>> indexUpdates, final WALEdit edit, final Durability durability) throws IOException { // no index updates, so we are done if (indexUpdates == null || indexUpdates.size() == 0) { return false; } // if writing to wal is disabled, we never see the WALEdit updates down the way, so do the index // update right away if (durability == Durability.SKIP_WAL) { try { this.writer.write(indexUpdates); return false; } catch (Throwable e) { LOG.error("Failed to update index with entries:" + indexUpdates, e); IndexManagementUtil.rethrowIndexingException(e); } } // we have all the WAL durability, so we just update the WAL entry and move on for (Pair<Mutation, byte[]> entry : indexUpdates) { edit.add(new IndexedKeyValue(entry.getSecond(), entry.getFirst())); } return true; }
Example #20
Source File: Indexer.java From phoenix with Apache License 2.0 | 5 votes |
@Override public void postPut(ObserverContext<RegionCoprocessorEnvironment> e, Put put, WALEdit edit, final Durability durability) throws IOException { if (this.disabled) { super.postPut(e, put, edit, durability); return; } doPost(edit, put, durability); }
Example #21
Source File: Indexer.java From phoenix with Apache License 2.0 | 5 votes |
@Override public void postDelete(ObserverContext<RegionCoprocessorEnvironment> e, Delete delete, WALEdit edit, final Durability durability) throws IOException { if (this.disabled) { super.postDelete(e, delete, edit, durability); return; } doPost(edit, delete, durability); }
Example #22
Source File: Indexer.java From phoenix with Apache License 2.0 | 5 votes |
@Override public void postBatchMutate(ObserverContext<RegionCoprocessorEnvironment> c, MiniBatchOperationInProgress<Mutation> miniBatchOp) throws IOException { if (this.disabled) { super.postBatchMutate(c, miniBatchOp); return; } this.builder.batchCompleted(miniBatchOp); //each batch operation, only the first one will have anything useful, so we can just grab that Mutation mutation = miniBatchOp.getOperation(0); WALEdit edit = miniBatchOp.getWalEdit(0); doPost(edit, mutation, mutation.getDurability()); }
Example #23
Source File: Indexer.java From phoenix with Apache License 2.0 | 5 votes |
private void doPost(WALEdit edit, Mutation m, final Durability durability) throws IOException { try { doPostWithExceptions(edit, m, durability); return; } catch (Throwable e) { rethrowIndexingException(e); } throw new RuntimeException( "Somehow didn't complete the index update, but didn't return succesfully either!"); }
Example #24
Source File: Indexer.java From phoenix with Apache License 2.0 | 5 votes |
/** * Search the {@link WALEdit} for the first {@link IndexedKeyValue} present * @param edit {@link WALEdit} * @return the first {@link IndexedKeyValue} in the {@link WALEdit} or <tt>null</tt> if not * present */ private IndexedKeyValue getFirstIndexedKeyValue(WALEdit edit) { for (Cell kv : edit.getCells()) { if (kv instanceof IndexedKeyValue) { return (IndexedKeyValue) kv; } } return null; }
Example #25
Source File: Indexer.java From phoenix with Apache License 2.0 | 5 votes |
/** * Extract the index updates from the WAL Edit * @param edit to search for index updates * @return the mutations to apply to the index tables */ private Collection<Pair<Mutation, byte[]>> extractIndexUpdate(WALEdit edit) { Collection<Pair<Mutation, byte[]>> indexUpdates = new ArrayList<Pair<Mutation, byte[]>>(); for (Cell kv : edit.getCells()) { if (kv instanceof IndexedKeyValue) { IndexedKeyValue ikv = (IndexedKeyValue) kv; indexUpdates.add(new Pair<Mutation, byte[]>(ikv.getMutation(), ikv.getIndexTable())); } } return indexUpdates; }
Example #26
Source File: TestWALRecoveryCaching.java From phoenix with Apache License 2.0 | 5 votes |
@Override public void preWALRestore(ObserverContext<RegionCoprocessorEnvironment> env, HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException { try { LOG.debug("Restoring logs for index table"); if (allowIndexTableToRecover != null) { allowIndexTableToRecover.await(); LOG.debug("Completed index table recovery wait latch"); } } catch (InterruptedException e) { Assert.fail("Should not be interrupted while waiting to allow the index to restore WALs."); } }
Example #27
Source File: Indexer.java From phoenix with BSD 3-Clause "New" or "Revised" License | 5 votes |
@Override public void prePut(final ObserverContext<RegionCoprocessorEnvironment> c, final Put put, final WALEdit edit, final boolean writeToWAL) throws IOException { if (this.disabled) { super.prePut(c, put, edit, writeToWAL); return; } // just have to add a batch marker to the WALEdit so we get the edit again in the batch // processing step. We let it throw an exception here because something terrible has happened. edit.add(BATCH_MARKER); }
Example #28
Source File: Indexer.java From phoenix with BSD 3-Clause "New" or "Revised" License | 5 votes |
/** * Add the index updates to the WAL, or write to the index table, if the WAL has been disabled * @return <tt>true</tt> if the WAL has been updated. * @throws IOException */ private boolean doPre(Collection<Pair<Mutation, byte[]>> indexUpdates, final WALEdit edit, final boolean writeToWAL) throws IOException { // no index updates, so we are done if (indexUpdates == null || indexUpdates.size() == 0) { return false; } // if writing to wal is disabled, we never see the WALEdit updates down the way, so do the index // update right away if (!writeToWAL) { try { this.writer.write(indexUpdates); return false; } catch (Throwable e) { LOG.error("Failed to update index with entries:" + indexUpdates, e); IndexManagementUtil.rethrowIndexingException(e); } } // we have all the WAL durability, so we just update the WAL entry and move on for (Pair<Mutation, byte[]> entry : indexUpdates) { edit.add(new IndexedKeyValue(entry.getSecond(), entry.getFirst())); } return true; }
Example #29
Source File: Indexer.java From phoenix with BSD 3-Clause "New" or "Revised" License | 5 votes |
@Override public void postPut(ObserverContext<RegionCoprocessorEnvironment> e, Put put, WALEdit edit, boolean writeToWAL) throws IOException { if (this.disabled) { super.postPut(e, put, edit, writeToWAL); return; } doPost(edit, put, writeToWAL); }
Example #30
Source File: Indexer.java From phoenix with BSD 3-Clause "New" or "Revised" License | 5 votes |
@Override public void postDelete(ObserverContext<RegionCoprocessorEnvironment> e, Delete delete, WALEdit edit, boolean writeToWAL) throws IOException { if (this.disabled) { super.postDelete(e, delete, edit, writeToWAL); return; } doPost(edit,delete, writeToWAL); }