com.carrotsearch.hppc.IntSet Java Examples

@Test
public void testBuildReaderAllocations() {
    RoutingBuilder routingBuilder = new RoutingBuilder(clusterService.state(), routingProvider);
    routingBuilder.allocateRouting(tableInfo, WhereClause.MATCH_ALL, RoutingProvider.ShardSelection.ANY, null);

    ReaderAllocations readerAllocations = routingBuilder.buildReaderAllocations();

    assertThat(readerAllocations.indices().size(), is(1));
    assertThat(readerAllocations.indices().get(0), is(relationName.indexNameOrAlias()));
    assertThat(readerAllocations.nodeReaders().size(), is(2));

    IntSet n1 = readerAllocations.nodeReaders().get("n1");
    assertThat(n1.size(), is(2));
    assertThat(n1.contains(0), is(true));
    assertThat(n1.contains(2), is(true));

    IntSet n2 = readerAllocations.nodeReaders().get("n2");
    assertThat(n2.size(), is(2));
    assertThat(n2.contains(1), is(true));
    assertThat(n2.contains(3), is(true));

    assertThat(readerAllocations.bases().get(relationName.indexNameOrAlias()), is(0));

    ReaderAllocations readerAllocations2 = routingBuilder.buildReaderAllocations();
    assertThat(readerAllocations, CoreMatchers.not(is(readerAllocations2)));
}
 

@Override
public CompletableFuture<? extends Iterator<Row>> apply(ReaderBuckets readerBuckets, boolean isLastCall) {
    List<CompletableFuture<IntObjectMap<? extends Bucket>>> futures = new ArrayList<>();
    Iterator<Map.Entry<String, IntSet>> it = readerIdsByNode.entrySet().iterator();
    while (it.hasNext()) {
        Map.Entry<String, IntSet> entry = it.next();
        IntObjectHashMap<IntContainer> toFetch = readerBuckets.generateToFetch(entry.getValue());
        if (toFetch.isEmpty() && !isLastCall) {
            continue;
        }
        final String nodeId = entry.getKey();
        try {
            futures.add(fetchOperation.fetch(nodeId, toFetch, isLastCall));
        } catch (Throwable t) {
            futures.add(CompletableFuture.failedFuture(t));
        }
        if (isLastCall) {
            it.remove();
        }
    }
    return CompletableFutures.allAsList(futures).thenApply(readerBuckets::getOutputRows);
}
 

@SuppressWarnings({"rawtypes"})
public Map<String, ? extends IntObjectMap<Streamer[]>> generateStreamersGroupedByReaderAndNode() {
    HashMap<String, IntObjectHashMap<Streamer[]>> streamersByReaderByNode = new HashMap<>();
    for (Map.Entry<String, IntSet> entry : nodeReaders.entrySet()) {
        IntObjectHashMap<Streamer[]> streamersByReaderId = new IntObjectHashMap<>();
        String nodeId = entry.getKey();
        streamersByReaderByNode.put(nodeId, streamersByReaderId);
        for (IntCursor readerIdCursor : entry.getValue()) {
            int readerId = readerIdCursor.value;
            String index = readerIndices.floorEntry(readerId).getValue();
            RelationName relationName = indicesToIdents.get(index);
            FetchSource fetchSource = fetchSources.get(relationName);
            if (fetchSource == null) {
                continue;
            }
            streamersByReaderId.put(readerIdCursor.value, Symbols.streamerArray(fetchSource.references()));
        }
    }
    return streamersByReaderByNode;
}
 

public FetchProjection(int fetchPhaseId,
                       int suppliedFetchSize,
                       Map<RelationName, FetchSource> fetchSources,
                       List<Symbol> outputSymbols,
                       Map<String, IntSet> nodeReaders,
                       TreeMap<Integer, String> readerIndices,
                       Map<String, RelationName> indicesToIdents) {
    assert outputSymbols.stream().noneMatch(s ->
        SymbolVisitors.any(x -> x instanceof ScopedSymbol || x instanceof SelectSymbol, s))
        : "Cannot operate on Field or SelectSymbol symbols: " + outputSymbols;
    this.fetchPhaseId = fetchPhaseId;
    this.fetchSources = fetchSources;
    this.outputSymbols = outputSymbols;
    this.nodeReaders = nodeReaders;
    this.readerIndices = readerIndices;
    this.indicesToIdents = indicesToIdents;
    this.fetchSize = boundedFetchSize(suppliedFetchSize, MAX_FETCH_SIZE);
}
 

/**
 * Initializes an index, to be restored from snapshot
 */
private Builder initializeAsRestore(IndexMetaData indexMetaData, SnapshotRecoverySource recoverySource, IntSet ignoreShards, boolean asNew, UnassignedInfo unassignedInfo) {
    assert indexMetaData.getIndex().equals(index);
    if (!shards.isEmpty()) {
        throw new IllegalStateException("trying to initialize an index with fresh shards, but already has shards created");
    }
    for (int shardNumber = 0; shardNumber < indexMetaData.getNumberOfShards(); shardNumber++) {
        ShardId shardId = new ShardId(index, shardNumber);
        IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId);
        for (int i = 0; i <= indexMetaData.getNumberOfReplicas(); i++) {
            boolean primary = i == 0;
            if (asNew && ignoreShards.contains(shardNumber)) {
                // This shards wasn't completely snapshotted - restore it as new shard
                indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, primary,
                    primary ? EmptyStoreRecoverySource.INSTANCE : PeerRecoverySource.INSTANCE, unassignedInfo));
            } else {
                indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, primary,
                    primary ? recoverySource : PeerRecoverySource.INSTANCE, unassignedInfo));
            }
        }
        shards.put(shardNumber, indexShardRoutingBuilder.build());
    }
    return this;
}
 

/**
 * Initializes an index, to be restored from snapshot
 */
private Builder initializeAsRestore(IndexMetaData indexMetaData, RestoreSource restoreSource, IntSet ignoreShards, boolean asNew, UnassignedInfo unassignedInfo) {
    if (!shards.isEmpty()) {
        throw new IllegalStateException("trying to initialize an index with fresh shards, but already has shards created");
    }
    for (int shardId = 0; shardId < indexMetaData.getNumberOfShards(); shardId++) {
        IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(new ShardId(indexMetaData.getIndex(), shardId));
        for (int i = 0; i <= indexMetaData.getNumberOfReplicas(); i++) {
            if (asNew && ignoreShards.contains(shardId)) {
                // This shards wasn't completely snapshotted - restore it as new shard
                indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(index, shardId, null, i == 0, unassignedInfo));
            } else {
                indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(index, shardId, i == 0 ? restoreSource : null, i == 0, unassignedInfo));
            }
        }
        shards.put(shardId, indexShardRoutingBuilder.build());
    }
    return this;
}
 

@Test
public void testKeybasedGraphPartitioning() {
    Object[] options = {option(GraphDatabaseConfiguration.IDS_FLUSH), false,
                        option(VertexIDAssigner.PLACEMENT_STRATEGY), PropertyPlacementStrategy.class.getName(),
            option(PropertyPlacementStrategy.PARTITION_KEY), "clusterId"};
    clopen(options);

    int[] groupDegrees = {5,5,5,5,5,5,5,5};
    int numVertices = setupGroupClusters(groupDegrees,CommitMode.PER_VERTEX);

    IntSet partitionIds = new IntHashSet(numVertices); //to track the "spread" of partition ids
    for (int i=0;i<groupDegrees.length;i++) {
        TitanVertex g = getOnlyVertex(tx.query().has("groupid","group"+i));
        int partitionId = -1;
        for (TitanVertex v : g.query().direction(Direction.IN).labels("member").vertices()) {
            if (partitionId<0) partitionId = getPartitionID(v);
            assertEquals(partitionId,getPartitionID(v));
            partitionIds.add(partitionId);
        }
    }
    assertTrue(partitionIds.size()>numPartitions/2); //This is a probabilistic test that might fail
}
 

public FetchProjection(int collectPhaseId,
                       Map<TableIdent, FetchSource> fetchSources,
                       List<Symbol> outputSymbols,
                       Map<String, IntSet> nodeReaders,
                       TreeMap<Integer, String> readerIndices,
                       Map<String, TableIdent> indicesToIdents) {
    this.collectPhaseId = collectPhaseId;
    this.fetchSources = fetchSources;
    this.outputSymbols = outputSymbols;
    this.nodeReaders = nodeReaders;
    this.readerIndices = readerIndices;
    this.indicesToIdents = indicesToIdents;
}
 

public IntObjectHashMap<IntContainer> generateToFetch(IntSet readerIds) {
    IntObjectHashMap<IntContainer> toFetch = new IntObjectHashMap<>(readerIds.size());
    for (IntCursor readerIdCursor : readerIds) {
        ReaderBucket readerBucket = readerBuckets.get(readerIdCursor.value);
        if (readerBucket != null && readerBucket.docs.size() > 0) {
            toFetch.put(readerIdCursor.value, readerBucket.docs.keys());
        }
    }
    return toFetch;
}
 

/**
 * Initializes a new empty index, to be restored from a snapshot
 */
public Builder initializeAsNewRestore(IndexMetaData indexMetaData, SnapshotRecoverySource recoverySource, IntSet ignoreShards) {
    final UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NEW_INDEX_RESTORED,
                                                             "restore_source[" + recoverySource.snapshot().getRepository() + "/" +
                                                                 recoverySource.snapshot().getSnapshotId().getName() + "]");
    return initializeAsRestore(indexMetaData, recoverySource, ignoreShards, true, unassignedInfo);
}
 

private void testPartitionSpread(boolean flush, boolean batchCommit) {
    Object[] options = {option(GraphDatabaseConfiguration.IDS_FLUSH), flush};
    clopen(options);

    int[] groupDegrees = {10,15,10,17,10,4,7,20,11};
    int numVertices = setupGroupClusters(groupDegrees,batchCommit?CommitMode.BATCH:CommitMode.PER_VERTEX);

    IntSet partitionIds = new IntHashSet(numVertices); //to track the "spread" of partition ids
    for (int i=0;i<groupDegrees.length;i++) {
        TitanVertex g = getOnlyVertex(tx.query().has("groupid","group"+i));
        assertCount(groupDegrees[i],g.edges(Direction.OUT,"contain"));
        assertCount(groupDegrees[i],g.edges(Direction.IN,"member"));
        assertCount(groupDegrees[i],g.query().direction(Direction.OUT).edges());
        assertCount(groupDegrees[i],g.query().direction(Direction.IN).edges());
        assertCount(groupDegrees[i]*2,g.query().edges());
        for (TitanVertex v : g.query().direction(Direction.IN).labels("member").vertices()) {
            int pid = getPartitionID(v);
            partitionIds.add(pid);
            assertEquals(g, getOnlyElement(v.query().direction(Direction.OUT).labels("member").vertices()));
            VertexList vlist = v.query().direction(Direction.IN).labels("contain").vertexIds();
            assertEquals(1,vlist.size());
            assertEquals(pid,idManager.getPartitionId(vlist.getID(0)));
            assertEquals(g,vlist.get(0));
        }
    }
    if (flush || !batchCommit) { //In these cases we would expect significant spread across partitions
        assertTrue(partitionIds.size()>numPartitions/2); //This is a probabilistic test that might fail
    } else {
        assertEquals(1,partitionIds.size()); //No spread in this case
    }
}
 

public FetchProjector(TransportFetchNodeAction transportFetchNodeAction,
                      ThreadPool threadPool,
                      Functions functions,
                      UUID jobId,
                      int collectPhaseId,
                      Map<TableIdent, FetchSource> fetchSources,
                      List<Symbol> outputSymbols,
                      Map<String, IntSet> nodeReaders,
                      TreeMap<Integer, String> readerIndices,
                      Map<String, TableIdent> indicesToIdents) {
    this.transportFetchNodeAction = transportFetchNodeAction;
    this.threadPool = threadPool;
    this.jobId = jobId;
    this.collectPhaseId = collectPhaseId;
    this.fetchSources = fetchSources;
    this.nodeReaders = nodeReaders;
    this.readerIndices = readerIndices;
    this.indicesToIdents = indicesToIdents;

    FetchRowInputSymbolVisitor rowInputSymbolVisitor = new FetchRowInputSymbolVisitor(functions);

    this.collectRowContext = new FetchRowInputSymbolVisitor.Context(fetchSources);

    List<Input<?>> inputs = new ArrayList<>(outputSymbols.size());
    for (Symbol symbol : outputSymbols) {
        inputs.add(rowInputSymbolVisitor.process(symbol, collectRowContext));
    }

    outputRow = new InputRow(inputs);
}
 

public Builder addAsNewRestore(IndexMetaData indexMetaData, SnapshotRecoverySource recoverySource, IntSet ignoreShards) {
    IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetaData.getIndex())
            .initializeAsNewRestore(indexMetaData, recoverySource, ignoreShards);
    add(indexRoutingBuilder);
    return this;
}
 

/**
 * Initializes a new empty index, to be restored from a snapshot
 */
public Builder initializeAsNewRestore(IndexMetaData indexMetaData, RestoreSource restoreSource, IntSet ignoreShards) {
    return initializeAsRestore(indexMetaData, restoreSource, ignoreShards, true, new UnassignedInfo(UnassignedInfo.Reason.NEW_INDEX_RESTORED, "restore_source[" + restoreSource.snapshotId().getRepository() + "/" + restoreSource.snapshotId().getSnapshot() + "]"));
}
 

public Builder addAsNewRestore(IndexMetaData indexMetaData, RestoreSource restoreSource, IntSet ignoreShards) {
    IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetaData.getIndex())
            .initializeAsNewRestore(indexMetaData, restoreSource, ignoreShards);
    add(indexRoutingBuilder);
    return this;
}
 

public Map<String, IntSet> nodeReaders() {
    return nodeReaders;
}
 

public IntSet primaryKeyColumnIndices() {
    return primaryKeyColumnIndices;
}
 

public Map<String, IntSet> nodeReaders() {
    return nodeReaders;
}
 

public IntSet partitionedByIndices() {
    return partitionedByColumnsIndices;
}
 

public Map<String, IntSet> nodeReaders() {
    return nodeReaders;
}
 

public FetchMapper(FetchOperation fetchOperation, Map<String, IntSet> readerIdsByNode) {
    this.fetchOperation = fetchOperation;
    this.readerIdsByNode = readerIdsByNode;
}
 

public Map<String, IntSet> nodeReaders() {
    return nodeReaders;
}