com.carrotsearch.hppc.LongHashSet Java Examples

@Override
protected boolean matchesSafely(Translog.Snapshot snapshot) {
    try {
        final LongSet seqNoList = new LongHashSet();
        Translog.Operation op;
        while ((op = snapshot.next()) != null) {
            seqNoList.add(op.seqNo());
        }
        for (long i = minSeqNo; i <= maxSeqNo; i++) {
            if (seqNoList.contains(i) == false) {
                notFoundSeqNo.add(i);
            }
        }
        return notFoundSeqNo.isEmpty();
    } catch (IOException ex) {
        throw new ElasticsearchException("failed to read snapshot content", ex);
    }
}
 

@Override
public void rollbackSubtransactions(long txnId, LongHashSet subtransactions) throws IOException {
    long beginTS = txnId & SIConstants.TRANSANCTION_ID_MASK;

    ReadWriteLock readWriteLock=lockStriper.get(beginTS);
    Lock wl=readWriteLock.writeLock();
    wl.lock();
    try{
        TxnHolder txnHolder=txnMap.get(beginTS);
        if(txnHolder==null) return; //no transaction exists

        Txn txn=txnHolder.txn;

        Txn.State state=txn.getState();
        if(state!=Txn.State.ACTIVE) return; //nothing to do if we aren't active
        txnHolder.txn=getRolledbackSubtxns(beginTS,txn,subtransactions);
    }finally{
        wl.unlock();
    }
}
 

@Test
public void testSimpleIDAcquisition() throws BackendException {
    final IDBlockSizer blockSizer = new InnerIDBlockSizer();
    idAuthorities[0].setIDBlockSizer(blockSizer);
    int numTrials = 100;
    LongSet ids = new LongHashSet((int)blockSize*numTrials);
    long previous = 0;
    for (int i=0;i<numTrials;i++) {
        IDBlock block = idAuthorities[0].getIDBlock(0, 0, GET_ID_BLOCK_TIMEOUT);
        checkBlock(block,ids);
        if (hasEmptyUid) {
            if (previous!=0)
                assertEquals(previous+1, block.getId(0));
            previous=block.getId(block.numIds()-1);
        }
    }
}
 

/**
 * Initializes storage for the specified {@link HLLType} and changes the
 * instance's {@link #type}.
 *
 * @param type the {@link HLLType} to initialize storage for. This cannot be
 *        <code>null</code> and must be an instantiable type.
 */
private void initializeStorage(final HLLType type) {
    this.type = type;
    switch(type) {
        case EMPTY:
            // nothing to be done
            break;
        case EXPLICIT:
            this.explicitStorage = new LongHashSet();
            break;
        case SPARSE:
            this.sparseProbabilisticStorage = new IntByteHashMap();
            break;
        case FULL:
            this.probabilisticStorage = new BitVector(regwidth, m);
            break;
        default:
            throw new RuntimeException("Unsupported HLL type " + type);
    }
}
 

private Query getGroupQuery(String fname,
                         FieldType ft,
                         int size,
                         LongHashSet groupSet) {

  BytesRef[] bytesRefs = new BytesRef[size];
  int index = -1;
  BytesRefBuilder term = new BytesRefBuilder();
  Iterator<LongCursor> it = groupSet.iterator();

  while (it.hasNext()) {
    LongCursor cursor = it.next();
    String stringVal = numericToString(ft, cursor.value);
    ft.readableToIndexed(stringVal, term);
    bytesRefs[++index] = term.toBytesRef();
  }

  return new TermInSetQuery(fname, bytesRefs);
}
 

private Query getPointGroupQuery(SchemaField sf,
                                 int size,
                                 LongHashSet groupSet) {

  Iterator<LongCursor> it = groupSet.iterator();
  List<String> values = new ArrayList<>(size);
  FieldType ft = sf.getType();
  while (it.hasNext()) {
    LongCursor cursor = it.next();
    values.add(numericToString(ft, cursor.value));
  }

  return sf.getType().getSetQuery(null, sf, values);
}
 

/**
 * Asserts that values in both sets are exactly equal.
 */
private static void assertElementsEqual(final HLL hllA, final HLL hllB) {
    final LongHashSet internalSetA = hllA.explicitStorage;
    final LongHashSet internalSetB = hllB.explicitStorage;

    assertTrue(internalSetA.equals(internalSetB));
}
 

private Txn getRolledbackSubtxns(long txnId, Txn txn, LongHashSet subtransactions) {
    final LongHashSet subs = subtransactions.clone();
    return new ForwardingTxnView(txn){
        @Override
        public LongHashSet getRolledback() {
            return subs;
        }
    };
}
 

public NumericGroupExpandCollector(String field, long nullValue, LongHashSet groupSet, IntHashSet collapsedSet, int limit, Sort sort) throws IOException {
  int numGroups = collapsedSet.size();
  this.nullValue = nullValue;
  groups = new LongObjectHashMap<>(numGroups);
  for (LongCursor cursor : groupSet) {
    groups.put(cursor.value, getCollector(limit, sort));
  }

  this.field = field;
  this.collapsedSet = collapsedSet;
}
 

@Override
public void rollbackSubtransactions(long txnId, LongHashSet subtransactions) throws IOException {
    byte[] rowKey=getTransactionRowKey(txnId);
    TxnMessage.TxnLifecycleMessage lifecycle=TxnMessage.TxnLifecycleMessage.newBuilder()
            .setTxnId(txnId).addAllRolledbackSubTxns(Longs.asList(subtransactions.toArray()))
            .setAction(TxnMessage.LifecycleAction.ROLLBACK_SUBTRANSACTIONS).build();
    try(TxnNetworkLayer table = tableFactory.accessTxnNetwork()){
        table.lifecycleAction(rowKey,lifecycle);
        rollbacks.incrementAndGet();
    }
}
 

@Override
public void rollbackSubtransactions(long txnId, LongHashSet subtransactions) throws IOException {
    long beginTS = txnId & SIConstants.TRANSANCTION_ID_MASK;

    TxnHolder txnHolder=txnMap.get(beginTS);
    if(txnHolder==null) return; //no transaction exists

    Txn txn=txnHolder.txn;

    Txn.State state=txn.getState();
    if(state!=Txn.State.ACTIVE) return; //nothing to do if we aren't active
    txnHolder.txn=getRolledbackSubtxns(beginTS,txn,subtransactions);
}
 

private Txn getRolledbackSubtxns(long txnId, Txn txn, LongHashSet subtransactions) {
    final LongHashSet subs = subtransactions.clone();
    return new ForwardingTxnView(txn){
        @Override
        public LongHashSet getRolledback() {
            return subs;
        }
    };
}
 

@Override
public void rollbackSubtransactions(long txnId, LongHashSet rolledback) throws IOException {
    if(restoreMode){
        return; // we are in restore mode, don't try to access the store
    }
    store.rollbackSubtransactions(txnId, rolledback);
}
 

private LongFilter(int numValids, int numInvalids) {
    if (numValids > 0) {
        valids = new LongHashSet(numValids);
    }
    if (numInvalids > 0) {
        invalids = new LongHashSet(numInvalids);
    }
}
 

private void resolveChildConflicts(Partition table,DataPut put,LongHashSet conflictingChildren) throws IOException{
    if(conflictingChildren!=null && !conflictingChildren.isEmpty()){
        DataDelete delete=opFactory.newDelete(put.key());
        Iterable<DataCell> cells=put.cells();
        for(LongCursor lc : conflictingChildren){
            for(DataCell dc : cells){
                delete.deleteColumn(dc.family(),dc.qualifier(),lc.value);
            }
            delete.deleteColumn(SIConstants.DEFAULT_FAMILY_BYTES,SIConstants.TOMBSTONE_COLUMN_BYTES,lc.value);
            delete.deleteColumn(SIConstants.DEFAULT_FAMILY_BYTES,SIConstants.COMMIT_TIMESTAMP_COLUMN_BYTES,lc.value);
        }
        delete.addAttribute(SIConstants.SUPPRESS_INDEXING_ATTRIBUTE_NAME,SIConstants.SUPPRESS_INDEXING_ATTRIBUTE_VALUE);
        table.delete(delete);
    }
}
 

@Override
public LongHashSet getRolledback() {
    if (getSubId() == 0) {
        return rolledback.clone();
    } else {
        return parentReference.getRolledback();
    }
}
 

@Override
public LongHashSet getRolledback() {
    return delegate.getRolledback();
}
 

@Override
public void rollbackSubtransactions(long txnId, LongHashSet rolledback) throws IOException {
	throw new UnsupportedOperationException("Cannot rollback subtransactions from the UnsupportedLifecycle Manager. Use a real Lifecycle manager instead");
}
 

@Override
public void rollbackSubtransactions(long txnId, LongHashSet rolledback) throws IOException {
    lifecycleManager.rollbackSubtransactions(txnId, rolledback);
}
 

private IntObjectHashMap<DataPut> checkConflictsForKvBatch(Partition table,
                                                           ConflictRollForward rollForwardQueue,
                                                           Pair<KVPair, Lock>[] dataAndLocks,
                                                           LongHashSet[] conflictingChildren,
                                                           TxnView transaction,
                                                           byte[] family, byte[] qualifier,
                                                           ConstraintChecker constraintChecker,
                                                           TxnFilter constraintStateFilter,
                                                           MutationStatus[] finalStatus, boolean skipConflictDetection,
                                                           boolean skipWAL, TxnSupplier supplier, boolean rollforward) throws IOException {
    IntObjectHashMap<DataPut> finalMutationsToWrite = new IntObjectHashMap(dataAndLocks.length, 0.9f);
    DataResult possibleConflicts = null;
    BitSet bloomInMemoryCheck  = skipConflictDetection ? null : table.getBloomInMemoryCheck(constraintChecker!=null,dataAndLocks);
    List<ByteSlice> toRollforward = null;
    if (rollforward) {
        toRollforward = new ArrayList<>(dataAndLocks.length);
    }
    for(int i = 0; i<dataAndLocks.length; i++) {
        Pair<KVPair, Lock> baseDataAndLock = dataAndLocks[i];
        if(baseDataAndLock==null) continue;

        ConflictResults conflictResults=ConflictResults.NO_CONFLICT;
        KVPair kvPair=baseDataAndLock.getFirst();
        KVPair.Type writeType=kvPair.getType();
        if(!skipConflictDetection && (constraintChecker!=null || !KVPair.Type.INSERT.equals(writeType))){
            /*
             *
             * If the table has no keys, then the hbase row key is a randomly generated UUID, so it's not
             * going to incur a write/write penalty, because there isn't any other row there (as long as we are inserting).
             * Therefore, we do not need to perform a write/write conflict check or a constraint check
             *
             * We know that this is the case because there is no constraint checker (constraint checkers are only
             * applied on key elements.
             */
            //todo -sf remove the Row key copy here
            possibleConflicts=bloomInMemoryCheck==null||bloomInMemoryCheck.get(i)?table.getLatest(kvPair.getRowKey(),possibleConflicts):null;
            if(possibleConflicts!=null){
                //we need to check for write conflicts
                try {
                    conflictResults = ensureNoWriteConflict(transaction, writeType, possibleConflicts, rollForwardQueue, supplier);
                } catch (IOException ioe) {
                    if (ioe instanceof WriteConflict) {
                        finalStatus[i] = operationStatusLib.failure(ioe);
                        continue;
                    } else throw ioe;
                }
                if(applyConstraint(constraintChecker,constraintStateFilter,i,kvPair,possibleConflicts,finalStatus,conflictResults.hasAdditiveConflicts())) //filter this row out, it fails the constraint
                    continue;
            }
            //TODO -sf- if type is an UPSERT, and conflict type is ADDITIVE_CONFLICT, then we
            //set the status on the row to ADDITIVE_CONFLICT_DURING_UPSERT
            if(KVPair.Type.UPSERT.equals(writeType)){
                /*
                 * If the type is an upsert, then we want to check for an ADDITIVE conflict. If so,
                 * we fail this row with an ADDITIVE_UPSERT_CONFLICT.
                 */
                if(conflictResults.hasAdditiveConflicts()){
                    finalStatus[i]=operationStatusLib.failure(exceptionLib.additiveWriteConflict());
                }
            }
        }

        conflictingChildren[i] = conflictResults.getChildConflicts();
        boolean addFirstOccurrenceToken = false;
        if (possibleConflicts == null) {
            // First write
            addFirstOccurrenceToken = true;
        } else if (KVPair.Type.DELETE.equals(writeType) && possibleConflicts.firstWriteToken() != null) {
            // Delete following first write
            assert possibleConflicts.userData() != null;
            addFirstOccurrenceToken = possibleConflicts.firstWriteToken().version() == possibleConflicts.userData().version();
        }
        DataPut mutationToRun = getMutationToRun(
                table, kvPair, family, qualifier, transaction, conflictResults, addFirstOccurrenceToken, skipWAL, toRollforward);
        finalMutationsToWrite.put(i,mutationToRun);
    }
    if (toRollforward != null && toRollforward.size() > 0) {
        SIDriver.driver().getRollForward().submitForResolution(table,transaction.getTxnId(),toRollforward);
    }

    return finalMutationsToWrite;
}
 

public LongHashSet getChildConflicts() {
    return childConflicts;
}
 

private ConflictResults(LongHashSet childConflicts, boolean hasTombstone) {
    this.childConflicts = childConflicts;
    this.hasTombstone = hasTombstone;
}
 

/** Set of subtransactions that have been rolledback */
LongHashSet getRolledback();
 

public static Collector<Long, ?, LongHashSet> toLongSet() {
    return new CollectorImpl<>(LongHashSet::new, LongHashSet::add, (left, right) -> {
        left.addAll(right);
        return left;
    }, CH_ID);
}
 

/**
 * Used by unit-tests
 */
public LongHashSet getLongHashSet() {
  return set;
}
 

@Test
public void testMultiIDAcquisition() throws Throwable {
    final int numPartitions = MAX_NUM_PARTITIONS;
    final int numAcquisitionsPerThreadPartition = 100;
    final IDBlockSizer blockSizer = new InnerIDBlockSizer();
    for (int i = 0; i < CONCURRENCY; i++) idAuthorities[i].setIDBlockSizer(blockSizer);
    final List<ConcurrentLinkedQueue<IDBlock>> ids = new ArrayList<ConcurrentLinkedQueue<IDBlock>>(numPartitions);
    for (int i = 0; i < numPartitions; i++) {
        ids.add(new ConcurrentLinkedQueue<IDBlock>());
    }

    final int maxIterations = numAcquisitionsPerThreadPartition * numPartitions * 2;
    final Collection<Future<?>> futures = new ArrayList<Future<?>>(CONCURRENCY);
    ExecutorService es = Executors.newFixedThreadPool(CONCURRENCY);

    Set<String> uids = new HashSet<String>(CONCURRENCY);
    for (int i = 0; i < CONCURRENCY; i++) {
        final IDAuthority idAuthority = idAuthorities[i];
        final IDStressor stressRunnable = new IDStressor(
                numAcquisitionsPerThreadPartition, numPartitions,
                maxIterations, idAuthority, ids);
        uids.add(idAuthority.getUniqueID());
        futures.add(es.submit(stressRunnable));
    }

    // If this fails, it's likely to be a bug in the test rather than the
    // IDAuthority (the latter is technically possible, just less likely)
    assertEquals(CONCURRENCY, uids.size());

    for (Future<?> f : futures) {
        try {
            f.get();
        } catch (ExecutionException e) {
            throw e.getCause();
        }
    }

    for (int i = 0; i < numPartitions; i++) {
        ConcurrentLinkedQueue<IDBlock> list = ids.get(i);
        assertEquals(numAcquisitionsPerThreadPartition * CONCURRENCY, list.size());
        LongSet idset = new LongHashSet((int)blockSize*list.size());
        for (IDBlock block : list) checkBlock(block,idset);
    }

    es.shutdownNow();
}
 

private void checkBlock(IDBlock block) {
    assertTrue(blockSize<10000);
    LongSet ids = new LongHashSet((int)blockSize);
    checkBlock(block,ids);
}
 

LongHashSet getTxnSet()
{
    return txnSet;
}
 

void rollbackSubtransactions(long txnId, LongHashSet rolledback) throws IOException; 

void rollbackSubtransactions(long txnId, LongHashSet subtransactions) throws IOException;