com.carrotsearch.hppc.IntArrayList Java Examples

/**
 * Releases shard targets that are not used in the docsIdsToLoad.
 */
protected void releaseIrrelevantSearchContexts(AtomicArray<? extends QuerySearchResultProvider> queryResults,
                                               AtomicArray<IntArrayList> docIdsToLoad) {
    if (docIdsToLoad == null) {
        return;
    }
    // we only release search context that we did not fetch from if we are not scrolling
    if (request.scroll() == null) {
        for (AtomicArray.Entry<? extends QuerySearchResultProvider> entry : queryResults.asList()) {
            final TopDocs topDocs = entry.value.queryResult().queryResult().topDocs();
            if (topDocs != null && topDocs.scoreDocs.length > 0 // the shard had matches
                    && docIdsToLoad.get(entry.index) == null) { // but none of them made it to the global top docs
                try {
                    DiscoveryNode node = nodes.get(entry.value.queryResult().shardTarget().nodeId());
                    sendReleaseSearchContext(entry.value.queryResult().id(), node);
                } catch (Throwable t1) {
                    logger.trace("failed to release context", t1);
                }
            }
        }
    }
}
 

private static RoutedCollectPhase createRemoteCollectPhase(UUID childJobId,
                                                           RoutedCollectPhase collectPhase,
                                                           ShardId shardId,
                                                           String nodeId) {

    Routing routing = new Routing(
        Map.of(
            nodeId, Map.of(shardId.getIndexName(), IntArrayList.from(shardId.getId()))
        )
    );
    return new RoutedCollectPhase(
        childJobId,
        SENDER_PHASE_ID,
        collectPhase.name(),
        routing,
        collectPhase.maxRowGranularity(),
        collectPhase.toCollect(),
        new ArrayList<>(Projections.shardProjections(collectPhase.projections())),
        collectPhase.where(),
        DistributionInfo.DEFAULT_BROADCAST
    );
}
 

@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    int size = in.readVInt();
    locations = new IntArrayList(size);
    failures = new ArrayList<>(size);
    for (int i = 0; i < size; i++) {
        locations.add(in.readVInt());
        if (in.readBoolean()) {
            failures.add(Failure.readFailure(in));
        } else {
            failures.add(null);
        }
    }
    if (in.readBoolean()) {
        failure = in.readThrowable();
    }
}
 

public static void ensureStringTypesAreStrings(DataType[] dataTypes, Object[][] rows) {
    if (rows.length == 0) {
        return;
    }

    // NOTE: currently BytesRef inside Maps aren't converted here because
    // if the map is coming from a ESSearchTask/EsGetTask they already contain strings
    // and we have no case in which another Task returns a Map with ByteRefs/Strings inside.
    final IntArrayList stringColumns = new IntArrayList();
    final IntArrayList stringCollectionColumns = new IntArrayList();
    int idx = 0;
    for (DataType dataType : dataTypes) {
        if (BYTES_REF_TYPES.contains(dataType)) {
            stringColumns.add(idx);
        } else if ((DataTypes.isCollectionType(dataType)
                && (BYTES_REF_TYPES.contains(((CollectionType)dataType).innerType())))) {
            stringCollectionColumns.add(idx);
        }
        idx++;
    }

    for (Object[] row : rows) {
        convertStringColumns(row, stringColumns);
        convertStringCollectionColumns(row, stringCollectionColumns);
    }
}
 

public void update(ShardResponse response) {
    IntArrayList itemIndices = response.itemIndices();
    List<Failure> failures = response.failures();
    for (int i = 0; i < itemIndices.size(); i++) {
        int location = itemIndices.get(i);
        ShardResponse.Failure failure = failures.get(i);
        if (failure == null) {
            successfulWrites.set(location, true);
        } else {
            failureLocations.set(location, true);
        }
    }
    List<Object[]> resultRows = response.getResultRows();
    if (resultRows != null) {
        this.resultRows.addAll(resultRows);
    }
}
 

@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    int size = in.readVInt();
    locations = new IntArrayList(size);
    responses = new ArrayList<>(size);
    failures = new ArrayList<>(size);
    for (int i = 0; i < size; i++) {
        locations.add(in.readVInt());
        if (in.readBoolean()) {
            GetResponse response = new GetResponse();
            response.readFrom(in);
            responses.add(response);
        } else {
            responses.add(null);
        }
        if (in.readBoolean()) {
            failures.add(MultiGetResponse.Failure.readFailure(in));
        } else {
            failures.add(null);
        }
    }
}
 

/**
 * Returns all transformations that do not have the given property and sorts the resulting array
 * according to the strategy.
 *
 * @param level The level which is to be sorted
 * @param triggerSkip The trigger to be used for limiting the number of nodes to be sorted
 * @return A sorted array of nodes remaining on this level
 */
private int[] getSortedUnprocessedNodes(int level, DependentAction triggerSkip) {

    // Create
    IntArrayList list = new IntArrayList();
    for (ObjectIterator<Long> iter = ((SolutionSpaceLong)solutionSpace).unsafeGetLevel(level); iter.hasNext();) {
        long id = iter.next();
        if (!skip(triggerSkip, ((SolutionSpaceLong)solutionSpace).getTransformation(id))) {
            list.add((int)id);
        }            
    }

    // Copy & sort
    int[] array = new int[list.size()];
    System.arraycopy(list.buffer, 0, array, 0, list.elementsCount);
    sort(array);
    return array;
}
 

/**
 * Returns a new data subset, only containing those rows that are included in the subset
 * @param rowset
 * @return
 */
protected DataSubset getSubsetInstance(RowSet rowset) {
    int index = -1;
    RowSet newset = RowSet.create(rowset.size());
    IntArrayList list = new IntArrayList();
    for (int row = 0; row < this.set.length(); row++) {
        if (rowset.contains(row)) {
            index++;
            if (this.set.contains(row)) {
                newset.add(index);
                list.add(index);
            }
        }
    }
    return new DataSubset(newset, list.toArray());
}
 

@Test
public void testDoubleIndex() throws Exception {
    IndexBaseBuilder builder = new IndexBaseBuilder();

    builder.allocate("i1", IntArrayList.from(1, 4));
    builder.allocate("i2", IntArrayList.from(1, 2));
    builder.allocate("i1", IntArrayList.from(1, 5));
    builder.allocate("i1", IntArrayList.from(1, 3));
    builder.allocate("i3", IntArrayList.from(1, 3));

    TreeMap<String, Integer> bases = builder.build();
    assertThat(bases.size(), is(3));
    assertThat(bases.get("i1"), is(0));
    assertThat(bases.get("i2"), is(6));
    assertThat(bases.get("i3"), is(9));
}
 

protected static int[] convertToInt(IntArrayList[] setToSplit, int exclude) {
  int count = 0;

  for (int i = 0; i < setToSplit.length; i++) {
    if (i != exclude) {
      count += setToSplit[i].size();
    }
  }

  int[] setData = new int[count];
  int a = 0;
  for (int i = 0; i < setToSplit.length; i++) {
    if (i != exclude) {
      for (IntCursor d : setToSplit[i]) {
        setData[a++] = d.value;
      }
    }
  }

  return setData;
}
 

public Routing(StreamInput in) throws IOException {
    int numLocations = in.readVInt();
    if (numLocations == 0) {
        locations = Map.of();
    } else {
        locations = new TreeMap<>();

        for (int i = 0; i < numLocations; i++) {
            String nodeId = in.readString();
            int numInner = in.readVInt();
            Map<String, IntIndexedContainer> shardsByIndex = new TreeMap<>();

            locations.put(nodeId, shardsByIndex);
            for (int j = 0; j < numInner; j++) {
                String indexName = in.readString();
                int numShards = in.readVInt();
                IntArrayList shardIds = new IntArrayList(numShards);
                for (int k = 0; k < numShards; k++) {
                    shardIds.add(in.readVInt());
                }
                shardsByIndex.put(indexName, shardIds);
            }
        }
    }
}
 

/**
 * gallic plus just delegates to the string semiring plus + the primitive semiring plus
 * NOTE this isn't the Union ('general') Gallic Plus from openfst (i have split this out for sanity at the expense of
 * elegance).
 */
@Override
public GallicWeight plus(GallicWeight a, GallicWeight b) {
  if (isZero(a)) return b;
  if (isZero(b)) return a;
  if (mode == RESTRICT_GALLIC) {
    if (SHORTLEX_ORDERING.compare(a, b) != 0) {
      throw new IllegalArgumentException("Trying to plus two different gallic weights, which isn't allowed in " +
        "this context. Did you pass a non-functional FST where a functional one was required? a = " + a +
        "; b = " + b);
    }
    double newWeight = this.weightSemiring.plus(a.getWeight(), b.getWeight());
    return GallicWeight.create(new IntArrayList(a.getLabels()), newWeight);
  } else {
    Preconditions.checkState(mode == MIN_GALLIC);
    return this.weightSemiring.naturalLess(a.getWeight(), b.getWeight()) ? a : b;
  }
}
 

@Override
public void getDocumentsWithWord(String word, IntArrayList documents) {
    DocsEnum docs = null;
    Term term = new Term(fieldName, word);
    try {
        int baseDocId;
        for (int i = 0; i < reader.length; i++) {
            docs = reader[i].termDocsEnum(term);
            baseDocId = contexts[i].docBase;
            if (docs != null) {
                while (docs.nextDoc() != DocsEnum.NO_MORE_DOCS) {
                    documents.add(docs.docID() + baseDocId);
                }
            }
        }
    } catch (IOException e) {
        LOGGER.error("Error while requesting documents for word \"" + word + "\".", e);
    }
}
 

public NodeFetchRequest(StreamInput in) throws IOException {
    super(in);
    jobId = new UUID(in.readLong(), in.readLong());
    fetchPhaseId = in.readVInt();
    closeContext = in.readBoolean();
    int numReaders = in.readVInt();
    if (numReaders > 0) {
        IntObjectHashMap<IntArrayList> toFetch = new IntObjectHashMap<>(numReaders);
        for (int i = 0; i < numReaders; i++) {
            int readerId = in.readVInt();
            int numDocs = in.readVInt();
            IntArrayList docs = new IntArrayList(numDocs);
            toFetch.put(readerId, docs);
            for (int j = 0; j < numDocs; j++) {
                docs.add(in.readInt());
            }
        }
        this.toFetch = toFetch;
    } else {
        this.toFetch = null;
    }
}
 

@Test
public void test() {
    ContextWindowFrequencyDeterminer determiner = new ContextWindowFrequencyDeterminer(this,
            windowSize);
    IntArrayList lists[] = new IntArrayList[positions.length];
    for (int i = 0; i < lists.length; ++i) {
        if (positions[i] != null) {
            lists[i] = new IntArrayList(positions[i].length);
            lists[i].add(positions[i]);
        }
    }
    int counts[] = determiner.determineCounts(
            new String[1][lists.length]/* new String[][] { { "A", "B", "C" } } */,
            new SegmentationDefinition[] { new SegmentationDefinition(
                    new int[0], new int[0][0], null) })[0].counts;
    Assert.assertArrayEquals(expectedCounts, counts);
}
 

@Test
public void test() {
    BooleanSlidingWindowFrequencyDeterminer determiner = new BooleanSlidingWindowFrequencyDeterminer(this,
            windowSize);
    IntArrayList lists[] = new IntArrayList[positions.length];
    for (int i = 0; i < lists.length; ++i) {
        if (positions[i] != null) {
            lists[i] = new IntArrayList(positions[i].length);
            lists[i].add(positions[i]);
        }
    }
    int counts[] = determiner.determineCounts(
            new String[1][lists.length]/* new String[][] { { "A", "B", "C" } } */,
            new SegmentationDefinition[] { new SegmentationDefinition(
                    new int[0], new int[0][0], null) })[0].counts;
    Assert.assertArrayEquals(expectedCounts, counts);
}
 

/**
 * Update arc offsets assuming the given goto length.
 */
private int emitNodes(FSA fsa, OutputStream os, IntArrayList linearized) throws IOException {
  int offset = 0;

  // Add epsilon state.
  offset += emitNodeData(os, 0);
  if (fsa.getRootNode() != 0)
    offset += emitArc(os, BIT_LAST_ARC, (byte) '^', offsets.get(fsa.getRootNode()));
  else
    offset += emitArc(os, BIT_LAST_ARC, (byte) '^', 0);

  boolean offsetsChanged = false;
  final int max = linearized.size();
  for (IntCursor c : linearized) {
    final int state = c.value;
    final int nextState = c.index + 1 < max ? linearized.get(c.index + 1) : NO_STATE;

    if (os == null) {
      offsetsChanged |= (offsets.get(state) != offset);
      offsets.put(state, offset);
    } else {
      assert offsets.get(state) == offset : state + " " + offsets.get(state) + " " + offset;
    }

    offset += emitNodeData(os, withNumbers ? numbers.get(state) : 0);
    offset += emitNodeArcs(fsa, os, state, nextState);
  }

  return offsetsChanged ? offset : 0;
}
 

private int[] createCounts(ObjectObjectOpenHashMap<String, IntArrayList> wordDocMapping, String[] wordset) {
    int counts[] = new int[(1 << wordset.length)];
    IntArrayList wordDocuments[] = new IntArrayList[wordset.length];
    for (int i = 0; i < wordDocuments.length; ++i) {
        wordDocuments[i] = wordDocMapping.get(wordset[i]);
        Arrays.sort(wordDocuments[i].buffer, 0, wordDocuments[i].elementsCount);
    }

    int posInList[] = new int[wordDocuments.length];
    int nextDocId;
    int documentSignature = 0;
    counts[0] = -1;
    do {
        ++counts[documentSignature];
        nextDocId = Integer.MAX_VALUE;
        for (int i = 0; i < wordDocuments.length; ++i) {
            if ((posInList[i] < wordDocuments[i].elementsCount)
                    && (wordDocuments[i].buffer[posInList[i]] <= nextDocId)) {
                if (wordDocuments[i].buffer[posInList[i]] < nextDocId) {
                    nextDocId = wordDocuments[i].buffer[posInList[i]];
                    documentSignature = 0;
                }
                documentSignature |= 1 << i;
            }
        }
        for (int i = 0; i < posInList.length; ++i) {
            if ((documentSignature & (1 << i)) > 0) {
                ++posInList[i];
            }
        }
    } while (nextDocId != Integer.MAX_VALUE);
    return counts;
}
 

@Override
public void getDocumentsWithWords(ObjectObjectOpenHashMap<String, IntArrayList> wordDocMapping) {
    Object keys[] = (Object[]) wordDocMapping.keys;
    Object values[] = (Object[]) wordDocMapping.values;
    for (int i = 0; i < wordDocMapping.allocated.length; ++i) {
        if (wordDocMapping.allocated[i]) {
            getDocumentsWithWord((String) keys[i], (IntArrayList) values[i]);
        }
    }
}
 

private static void convertStringCollectionColumns(Object[] row, IntArrayList stringCollectionColumns) {
    for (IntCursor stringCollectionColumn : stringCollectionColumns) {
        Object value = row[stringCollectionColumn.value];
        if (value == null) {
            continue;
        }
        if (value instanceof Set) {
            row[stringCollectionColumn.value] = setToStringArray(((Set<BytesRef>) value));
        } else if (value instanceof BytesRef[]) {
            row[stringCollectionColumn.value] = objectArrayToStringArray(((BytesRef[]) value));
        } else if (value instanceof Object[]) {
            row[stringCollectionColumn.value] = objectArrayToStringArray(((Object[]) value));
        }
    }
}
 

@Override
public IntObjectOpenHashMap<IntArrayList[]> requestWordPositionsInDocuments(String[] words,
        IntIntOpenHashMap docLengths) {
    IntObjectOpenHashMap<IntArrayList[]> positionsInDocs = new IntObjectOpenHashMap<IntArrayList[]>();
    for (int i = 0; i < words.length; ++i) {
        requestDocumentsWithWord(words[i], positionsInDocs, docLengths, i, words.length);
    }
    return positionsInDocs;
}
 

protected static int[] convertToInt(IntArrayList trainSet) {
  int[] train = new int[trainSet.size()];
  int a = 0;
  for (IntCursor i : trainSet) {
    train[a++] = i.value;
  }
  return train;
}
 

public static Routing forTableOnAllNodes(RelationName relationName, DiscoveryNodes nodes) {
    TreeMap<String, Map<String, IntIndexedContainer>> indicesByNode = new TreeMap<>();
    Map<String, IntIndexedContainer> shardsByIndex = Collections.singletonMap(
        relationName.indexNameOrAlias(),
        IntArrayList.from(IntArrayList.EMPTY_ARRAY)
    );
    for (DiscoveryNode node : nodes) {
        indicesByNode.put(node.getId(), shardsByIndex);
    }
    return new Routing(indicesByNode);
}
 

protected void generateIndices(TimeSeries[] samples) {
  IntArrayList[] sets = getStratifiedTrainTestSplitIndices(samples, folds);
  this.testIndices = new int[folds][];
  this.trainIndices = new int[folds][];
  for (int s = 0; s < folds; s++) {
    this.testIndices[s] = convertToInt(sets[s]);
    this.trainIndices[s] = convertToInt(sets, s);
  }
}
 

protected void addCountsFromSmallDocument(IntArrayList[] positions, int[] counts) {
    int signature = 0;
    for (int i = 0; i < positions.length; ++i) {
        if ((positions[i] != null) && (positions[i].size() > 0)) {
            signature |= 1 << i;
        }
    }
    ++counts[signature];
}
 

protected int[] determineCounts(String wordset[]) {
    int counts[] = new int[(1 << wordset.length)];
    IntArrayList positions[];
    IntIntOpenHashMap docLengths = new IntIntOpenHashMap();
    IntObjectOpenHashMap<IntArrayList[]> positionsInDocs = corpusAdapter.requestWordPositionsInDocuments(wordset,
            docLengths);
    for (int i = 0; i < positionsInDocs.keys.length; ++i) {
        if (positionsInDocs.allocated[i]) {
            positions = ((IntArrayList[]) ((Object[]) positionsInDocs.values)[i]);
            addCountsFromDocument(positions, counts, docLengths.get(positionsInDocs.keys[i]));
        }
    }
    return counts;
}
 

/**
 * Java serialization
 *
 * @param in
 * @throws IOException
 * @throws ClassNotFoundException
 */
private void readObject(java.io.ObjectInputStream in) throws IOException, ClassNotFoundException {
  in.defaultReadObject();

  if (isLeaf()) {
    int[] elem = (int[]) in.readUnshared();
    this.elementIds = new IntArrayList(elem.length);
    elementIds.add(elem);
  }
}
 

public SFANode(byte[] word, int length) {
  this.type = NodeType.Leaf;
  this.word = word;

  this.minValues = new double[length];
  this.maxValues = new double[length];
  Arrays.fill(this.minValues, Double.MAX_VALUE);
  Arrays.fill(this.maxValues, Double.MIN_VALUE);

  this.elementIds = new IntArrayList(leafThreshold / 2);
  this.approximationIds = new IntArrayList(leafThreshold / 2);
}
 

@Test
public void testSearcherIsAcquiredForShard() throws Exception {
    IntArrayList shards = IntArrayList.from(1, 2);
    Routing routing = new Routing(Map.of("dummy", Map.of("i1", shards)));
    IndexBaseBuilder ibb = new IndexBaseBuilder();
    ibb.allocate("i1", shards);

    HashMultimap<RelationName, String> tableIndices = HashMultimap.create();
    tableIndices.put(new RelationName(Schemas.DOC_SCHEMA_NAME, "i1"), "i1");

    MetaData metaData = MetaData.builder()
        .put(IndexMetaData.builder("i1")
            .settings(Settings.builder()
                .put(SETTING_NUMBER_OF_SHARDS, 1)
                .put(SETTING_NUMBER_OF_REPLICAS, 0)
                .put(SETTING_VERSION_CREATED, Version.CURRENT))
            .build(), true)
        .build();
    final FetchTask context = new FetchTask(
        UUID.randomUUID(),
        new FetchPhase(
            1,
            null,
            ibb.build(),
            tableIndices,
            ImmutableList.of(createReference("i1", new ColumnIdent("x"), DataTypes.STRING))),
        "dummy",
        new SharedShardContexts(mock(IndicesService.class, RETURNS_MOCKS), UnaryOperator.identity()),
        metaData,
        relationName -> null,
        ImmutableList.of(routing));

    context.prepare();

    assertThat(context.searcher(1), Matchers.notNullValue());
    assertThat(context.searcher(2), Matchers.notNullValue());
}
 

public ShardResponse(StreamInput in) throws IOException {
    super(in);
    int size = in.readVInt();
    locations = new IntArrayList(size);
    failures = new ArrayList<>(size);
    for (int i = 0; i < size; i++) {
        locations.add(in.readVInt());
        if (in.readBoolean()) {
            failures.add(new Failure(in));
        } else {
            failures.add(null);
        }
    }
    if (in.readBoolean()) {
        failure = in.readException();
    }
    if (in.getVersion().onOrAfter(Version.V_4_2_0)) {
        int resultColumnsSize = in.readVInt();
        if (resultColumnsSize > 0) {
            resultColumns = new Symbol[resultColumnsSize];
            for (int i = 0; i < resultColumnsSize; i++) {
                Symbol symbol = Symbols.fromStream(in);
                resultColumns[i] = symbol;
            }
            Streamer[] resultRowStreamers = Symbols.streamerArray(List.of(resultColumns));
            int resultRowsSize = in.readVInt();
            if (resultRowsSize > 0) {
                resultRows = new ArrayList<>(resultRowsSize);
                int rowLength = in.readVInt();
                for (int i = 0; i < resultRowsSize; i++) {
                    Object[] row = new Object[rowLength];
                    for (int j = 0; j < rowLength; j++) {
                        row[j] = resultRowStreamers[j].readValueFrom(in);
                    }
                    resultRows.add(row);
                }
            }
        }
    }
}