Java Code Examples for com.carrotsearch.hppc.IntArrayList

The following examples show how to use com.carrotsearch.hppc.IntArrayList. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Elasticsearch   Source File: BytesRefUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static void ensureStringTypesAreStrings(DataType[] dataTypes, Object[][] rows) {
    if (rows.length == 0) {
        return;
    }

    // NOTE: currently BytesRef inside Maps aren't converted here because
    // if the map is coming from a ESSearchTask/EsGetTask they already contain strings
    // and we have no case in which another Task returns a Map with ByteRefs/Strings inside.
    final IntArrayList stringColumns = new IntArrayList();
    final IntArrayList stringCollectionColumns = new IntArrayList();
    int idx = 0;
    for (DataType dataType : dataTypes) {
        if (BYTES_REF_TYPES.contains(dataType)) {
            stringColumns.add(idx);
        } else if ((DataTypes.isCollectionType(dataType)
                && (BYTES_REF_TYPES.contains(((CollectionType)dataType).innerType())))) {
            stringCollectionColumns.add(idx);
        }
        idx++;
    }

    for (Object[] row : rows) {
        convertStringColumns(row, stringColumns);
        convertStringCollectionColumns(row, stringCollectionColumns);
    }
}
 
Example 2
Source Project: crate   Source File: RemoteCollectorFactory.java    License: Apache License 2.0 6 votes vote down vote up
private static RoutedCollectPhase createRemoteCollectPhase(UUID childJobId,
                                                           RoutedCollectPhase collectPhase,
                                                           ShardId shardId,
                                                           String nodeId) {

    Routing routing = new Routing(
        Map.of(
            nodeId, Map.of(shardId.getIndexName(), IntArrayList.from(shardId.getId()))
        )
    );
    return new RoutedCollectPhase(
        childJobId,
        SENDER_PHASE_ID,
        collectPhase.name(),
        routing,
        collectPhase.maxRowGranularity(),
        collectPhase.toCollect(),
        new ArrayList<>(Projections.shardProjections(collectPhase.projections())),
        collectPhase.where(),
        DistributionInfo.DEFAULT_BROADCAST
    );
}
 
Example 3
Source Project: Elasticsearch   Source File: ShardResponse.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    int size = in.readVInt();
    locations = new IntArrayList(size);
    failures = new ArrayList<>(size);
    for (int i = 0; i < size; i++) {
        locations.add(in.readVInt());
        if (in.readBoolean()) {
            failures.add(Failure.readFailure(in));
        } else {
            failures.add(null);
        }
    }
    if (in.readBoolean()) {
        failure = in.readThrowable();
    }
}
 
Example 4
Source Project: Elasticsearch   Source File: MultiGetShardResponse.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    int size = in.readVInt();
    locations = new IntArrayList(size);
    responses = new ArrayList<>(size);
    failures = new ArrayList<>(size);
    for (int i = 0; i < size; i++) {
        locations.add(in.readVInt());
        if (in.readBoolean()) {
            GetResponse response = new GetResponse();
            response.readFrom(in);
            responses.add(response);
        } else {
            responses.add(null);
        }
        if (in.readBoolean()) {
            failures.add(MultiGetResponse.Failure.readFailure(in));
        } else {
            failures.add(null);
        }
    }
}
 
Example 5
Source Project: arx   Source File: FLASHAlgorithmImpl.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Returns all transformations that do not have the given property and sorts the resulting array
 * according to the strategy.
 *
 * @param level The level which is to be sorted
 * @param triggerSkip The trigger to be used for limiting the number of nodes to be sorted
 * @return A sorted array of nodes remaining on this level
 */
private int[] getSortedUnprocessedNodes(int level, DependentAction triggerSkip) {

    // Create
    IntArrayList list = new IntArrayList();
    for (ObjectIterator<Long> iter = ((SolutionSpaceLong)solutionSpace).unsafeGetLevel(level); iter.hasNext();) {
        long id = iter.next();
        if (!skip(triggerSkip, ((SolutionSpaceLong)solutionSpace).getTransformation(id))) {
            list.add((int)id);
        }            
    }

    // Copy & sort
    int[] array = new int[list.size()];
    System.arraycopy(list.buffer, 0, array, 0, list.elementsCount);
    sort(array);
    return array;
}
 
Example 6
Source Project: Elasticsearch   Source File: AbstractSearchAsyncAction.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Releases shard targets that are not used in the docsIdsToLoad.
 */
protected void releaseIrrelevantSearchContexts(AtomicArray<? extends QuerySearchResultProvider> queryResults,
                                               AtomicArray<IntArrayList> docIdsToLoad) {
    if (docIdsToLoad == null) {
        return;
    }
    // we only release search context that we did not fetch from if we are not scrolling
    if (request.scroll() == null) {
        for (AtomicArray.Entry<? extends QuerySearchResultProvider> entry : queryResults.asList()) {
            final TopDocs topDocs = entry.value.queryResult().queryResult().topDocs();
            if (topDocs != null && topDocs.scoreDocs.length > 0 // the shard had matches
                    && docIdsToLoad.get(entry.index) == null) { // but none of them made it to the global top docs
                try {
                    DiscoveryNode node = nodes.get(entry.value.queryResult().shardTarget().nodeId());
                    sendReleaseSearchContext(entry.value.queryResult().id(), node);
                } catch (Throwable t1) {
                    logger.trace("failed to release context", t1);
                }
            }
        }
    }
}
 
Example 7
Source Project: arx   Source File: DataSubset.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Returns a new data subset, only containing those rows that are included in the subset
 * @param rowset
 * @return
 */
protected DataSubset getSubsetInstance(RowSet rowset) {
    int index = -1;
    RowSet newset = RowSet.create(rowset.size());
    IntArrayList list = new IntArrayList();
    for (int row = 0; row < this.set.length(); row++) {
        if (rowset.contains(row)) {
            index++;
            if (this.set.contains(row)) {
                newset.add(index);
                list.add(index);
            }
        }
    }
    return new DataSubset(newset, list.toArray());
}
 
Example 8
Source Project: crate   Source File: ShardResponse.java    License: Apache License 2.0 6 votes vote down vote up
public void update(ShardResponse response) {
    IntArrayList itemIndices = response.itemIndices();
    List<Failure> failures = response.failures();
    for (int i = 0; i < itemIndices.size(); i++) {
        int location = itemIndices.get(i);
        ShardResponse.Failure failure = failures.get(i);
        if (failure == null) {
            successfulWrites.set(location, true);
        } else {
            failureLocations.set(location, true);
        }
    }
    List<Object[]> resultRows = response.getResultRows();
    if (resultRows != null) {
        this.resultRows.addAll(resultRows);
    }
}
 
Example 9
Source Project: crate   Source File: IndexBaseBuilderTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDoubleIndex() throws Exception {
    IndexBaseBuilder builder = new IndexBaseBuilder();

    builder.allocate("i1", IntArrayList.from(1, 4));
    builder.allocate("i2", IntArrayList.from(1, 2));
    builder.allocate("i1", IntArrayList.from(1, 5));
    builder.allocate("i1", IntArrayList.from(1, 3));
    builder.allocate("i3", IntArrayList.from(1, 3));

    TreeMap<String, Integer> bases = builder.build();
    assertThat(bases.size(), is(3));
    assertThat(bases.get("i1"), is(0));
    assertThat(bases.get("i2"), is(6));
    assertThat(bases.get("i3"), is(9));
}
 
Example 10
Source Project: SFA   Source File: Classifier.java    License: GNU General Public License v3.0 6 votes vote down vote up
protected static int[] convertToInt(IntArrayList[] setToSplit, int exclude) {
  int count = 0;

  for (int i = 0; i < setToSplit.length; i++) {
    if (i != exclude) {
      count += setToSplit[i].size();
    }
  }

  int[] setData = new int[count];
  int a = 0;
  for (int i = 0; i < setToSplit.length; i++) {
    if (i != exclude) {
      for (IntCursor d : setToSplit[i]) {
        setData[a++] = d.value;
      }
    }
  }

  return setData;
}
 
Example 11
Source Project: crate   Source File: Routing.java    License: Apache License 2.0 6 votes vote down vote up
public Routing(StreamInput in) throws IOException {
    int numLocations = in.readVInt();
    if (numLocations == 0) {
        locations = Map.of();
    } else {
        locations = new TreeMap<>();

        for (int i = 0; i < numLocations; i++) {
            String nodeId = in.readString();
            int numInner = in.readVInt();
            Map<String, IntIndexedContainer> shardsByIndex = new TreeMap<>();

            locations.put(nodeId, shardsByIndex);
            for (int j = 0; j < numInner; j++) {
                String indexName = in.readString();
                int numShards = in.readVInt();
                IntArrayList shardIds = new IntArrayList(numShards);
                for (int k = 0; k < numShards; k++) {
                    shardIds.add(in.readVInt());
                }
                shardsByIndex.put(indexName, shardIds);
            }
        }
    }
}
 
Example 12
Source Project: jopenfst   Source File: GallicSemiring.java    License: MIT License 6 votes vote down vote up
/**
 * gallic plus just delegates to the string semiring plus + the primitive semiring plus
 * NOTE this isn't the Union ('general') Gallic Plus from openfst (i have split this out for sanity at the expense of
 * elegance).
 */
@Override
public GallicWeight plus(GallicWeight a, GallicWeight b) {
  if (isZero(a)) return b;
  if (isZero(b)) return a;
  if (mode == RESTRICT_GALLIC) {
    if (SHORTLEX_ORDERING.compare(a, b) != 0) {
      throw new IllegalArgumentException("Trying to plus two different gallic weights, which isn't allowed in " +
        "this context. Did you pass a non-functional FST where a functional one was required? a = " + a +
        "; b = " + b);
    }
    double newWeight = this.weightSemiring.plus(a.getWeight(), b.getWeight());
    return GallicWeight.create(new IntArrayList(a.getLabels()), newWeight);
  } else {
    Preconditions.checkState(mode == MIN_GALLIC);
    return this.weightSemiring.naturalLess(a.getWeight(), b.getWeight()) ? a : b;
  }
}
 
Example 13
@Override
public void getDocumentsWithWord(String word, IntArrayList documents) {
    DocsEnum docs = null;
    Term term = new Term(fieldName, word);
    try {
        int baseDocId;
        for (int i = 0; i < reader.length; i++) {
            docs = reader[i].termDocsEnum(term);
            baseDocId = contexts[i].docBase;
            if (docs != null) {
                while (docs.nextDoc() != DocsEnum.NO_MORE_DOCS) {
                    documents.add(docs.docID() + baseDocId);
                }
            }
        }
    } catch (IOException e) {
        LOGGER.error("Error while requesting documents for word \"" + word + "\".", e);
    }
}
 
Example 14
Source Project: crate   Source File: NodeFetchRequest.java    License: Apache License 2.0 6 votes vote down vote up
public NodeFetchRequest(StreamInput in) throws IOException {
    super(in);
    jobId = new UUID(in.readLong(), in.readLong());
    fetchPhaseId = in.readVInt();
    closeContext = in.readBoolean();
    int numReaders = in.readVInt();
    if (numReaders > 0) {
        IntObjectHashMap<IntArrayList> toFetch = new IntObjectHashMap<>(numReaders);
        for (int i = 0; i < numReaders; i++) {
            int readerId = in.readVInt();
            int numDocs = in.readVInt();
            IntArrayList docs = new IntArrayList(numDocs);
            toFetch.put(readerId, docs);
            for (int j = 0; j < numDocs; j++) {
                docs.add(in.readInt());
            }
        }
        this.toFetch = toFetch;
    } else {
        this.toFetch = null;
    }
}
 
Example 15
@Test
public void test() {
    ContextWindowFrequencyDeterminer determiner = new ContextWindowFrequencyDeterminer(this,
            windowSize);
    IntArrayList lists[] = new IntArrayList[positions.length];
    for (int i = 0; i < lists.length; ++i) {
        if (positions[i] != null) {
            lists[i] = new IntArrayList(positions[i].length);
            lists[i].add(positions[i]);
        }
    }
    int counts[] = determiner.determineCounts(
            new String[1][lists.length]/* new String[][] { { "A", "B", "C" } } */,
            new SegmentationDefinition[] { new SegmentationDefinition(
                    new int[0], new int[0][0], null) })[0].counts;
    Assert.assertArrayEquals(expectedCounts, counts);
}
 
Example 16
@Test
public void test() {
    BooleanSlidingWindowFrequencyDeterminer determiner = new BooleanSlidingWindowFrequencyDeterminer(this,
            windowSize);
    IntArrayList lists[] = new IntArrayList[positions.length];
    for (int i = 0; i < lists.length; ++i) {
        if (positions[i] != null) {
            lists[i] = new IntArrayList(positions[i].length);
            lists[i].add(positions[i]);
        }
    }
    int counts[] = determiner.determineCounts(
            new String[1][lists.length]/* new String[][] { { "A", "B", "C" } } */,
            new SegmentationDefinition[] { new SegmentationDefinition(
                    new int[0], new int[0][0], null) })[0].counts;
    Assert.assertArrayEquals(expectedCounts, counts);
}
 
Example 17
Source Project: Elasticsearch   Source File: BytesRefUtils.java    License: Apache License 2.0 5 votes vote down vote up
private static void convertStringCollectionColumns(Object[] row, IntArrayList stringCollectionColumns) {
    for (IntCursor stringCollectionColumn : stringCollectionColumns) {
        Object value = row[stringCollectionColumn.value];
        if (value == null) {
            continue;
        }
        if (value instanceof Set) {
            row[stringCollectionColumn.value] = setToStringArray(((Set<BytesRef>) value));
        } else if (value instanceof BytesRef[]) {
            row[stringCollectionColumn.value] = objectArrayToStringArray(((BytesRef[]) value));
        } else if (value instanceof Object[]) {
            row[stringCollectionColumn.value] = objectArrayToStringArray(((Object[]) value));
        }
    }
}
 
Example 18
Source Project: crate   Source File: Routing.java    License: Apache License 2.0 5 votes vote down vote up
public static Routing forTableOnAllNodes(RelationName relationName, DiscoveryNodes nodes) {
    TreeMap<String, Map<String, IntIndexedContainer>> indicesByNode = new TreeMap<>();
    Map<String, IntIndexedContainer> shardsByIndex = Collections.singletonMap(
        relationName.indexNameOrAlias(),
        IntArrayList.from(IntArrayList.EMPTY_ARRAY)
    );
    for (DiscoveryNode node : nodes) {
        indicesByNode.put(node.getId(), shardsByIndex);
    }
    return new Routing(indicesByNode);
}
 
Example 19
Source Project: Elasticsearch   Source File: ShardRequest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    routing = in.readOptionalString();
    jobId = new UUID(in.readLong(), in.readLong());

    int size = in.readVInt();
    locations = new IntArrayList(size);
    for (int i = 0; i < size; i++) {
        locations.add(in.readVInt());
    }
}
 
Example 20
Source Project: Elasticsearch   Source File: ShardFetchRequest.java    License: Apache License 2.0 5 votes vote down vote up
public ShardFetchRequest(SearchScrollRequest request, long id, IntArrayList list, ScoreDoc lastEmittedDoc) {
    super(request);
    this.id = id;
    this.docIds = list.buffer;
    this.size = list.size();
    this.lastEmittedDoc = lastEmittedDoc;
}
 
Example 21
Source Project: crate   Source File: FetchRows.java    License: Apache License 2.0 5 votes vote down vote up
public FetchRows(IntArrayList fetchIdPositions,
                 List<Input<?>> outputExpressions,
                 UnsafeArrayRow inputRow,
                 IntObjectHashMap<UnsafeArrayRow> fetchedRows,
                 ArrayList<Object[]> nullRows) {
    this.fetchedRows = fetchedRows;
    this.nullRows = nullRows;
    this.fetchIdPositions = fetchIdPositions.toArray();
    this.output = new InputRow(outputExpressions);
    this.inputRow = inputRow;
}
 
Example 22
Source Project: crate   Source File: IndexBaseBuilderTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testMaxShard() throws Exception {
    IndexBaseBuilder builder = new IndexBaseBuilder();

    builder.allocate("i1", IntArrayList.from(1, 4));
    builder.allocate("i2", IntArrayList.from(1, 5));
    TreeMap<String, Integer> bases = builder.build();
    assertThat(bases.size(), is(2));
    assertThat(bases.get("i1"), is(0));
    assertThat(bases.get("i2"), is(5));
}
 
Example 23
Source Project: Elasticsearch   Source File: MultiGetShardRequest.java    License: Apache License 2.0 5 votes vote down vote up
MultiGetShardRequest(MultiGetRequest multiGetRequest, String index, int shardId) {
    super(multiGetRequest, index);
    this.shardId = shardId;
    locations = new IntArrayList();
    items = new ArrayList<>();
    preference = multiGetRequest.preference;
    realtime = multiGetRequest.realtime;
    refresh = multiGetRequest.refresh;
    ignoreErrorsOnGeneratedFields = multiGetRequest.ignoreErrorsOnGeneratedFields;
}
 
Example 24
Source Project: Elasticsearch   Source File: AbstractSearchAsyncAction.java    License: Apache License 2.0 5 votes vote down vote up
protected ShardFetchSearchRequest createFetchRequest(QuerySearchResult queryResult, AtomicArray.Entry<IntArrayList> entry, ScoreDoc[] lastEmittedDocPerShard) {
    if (lastEmittedDocPerShard != null) {
        ScoreDoc lastEmittedDoc = lastEmittedDocPerShard[entry.index];
        return new ShardFetchSearchRequest(request, queryResult.id(), entry.value, lastEmittedDoc);
    } else {
        return new ShardFetchSearchRequest(request, queryResult.id(), entry.value);
    }
}
 
Example 25
/**
 * Add a state to linearized list.
 */
private void linearizeState(final FSA fsa, IntStack nodes, IntArrayList linearized, BitSet visited, int node) {
  linearized.add(node);
  visited.set(node);
  for (int arc = fsa.getFirstArc(node); arc != 0; arc = fsa.getNextArc(arc)) {
    if (!fsa.isArcTerminal(arc)) {
      final int target = fsa.getEndNode(arc);
      if (!visited.get(target))
        nodes.push(target);
    }
  }
}
 
Example 26
@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    int size = in.readVInt();
    locations = new IntArrayList(size);
    requests = new ArrayList<>(size);
    for (int i = 0; i < size; i++) {
        locations.add(in.readVInt());
        requests.add(TermVectorsRequest.readTermVectorsRequest(in));
    }

    preference = in.readOptionalString();
}
 
Example 27
@Override
public Set<Long> getErrorDocIds() throws IOException
{
    Set<Long> errorDocIds = new HashSet<>();
    RefCounted<SolrIndexSearcher> refCounted = null;
    try
    {
        refCounted = this.core.getSearcher();
        SolrIndexSearcher searcher = refCounted.get();
        TermQuery errorQuery = new TermQuery(new Term(FIELD_DOC_TYPE, DOC_TYPE_ERROR_NODE));
        DocListCollector docListCollector = new DocListCollector();
        searcher.search(errorQuery, docListCollector);
        IntArrayList docList = docListCollector.getDocs();
        int size = docList.size();

        for (int i = 0; i < size; ++i)
        {
            int doc = docList.get(i);
            Document document = searcher.doc(doc, REQUEST_ONLY_ID_FIELD);
            IndexableField id = document.getField(FIELD_SOLR4_ID);
            String idString = id.stringValue();

            if (idString.startsWith(PREFIX_ERROR))
            {
                idString = idString.substring(PREFIX_ERROR.length());
            }

            errorDocIds.add(Long.valueOf(idString));
        }
    }
    finally
    {
        ofNullable(refCounted).ifPresent(RefCounted::decref);
    }
    return errorDocIds;
}
 
Example 28
Source Project: crate   Source File: Routing.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Return a routing for the given table on the given node id.
 */
public static Routing forTableOnSingleNode(RelationName relationName, String nodeId) {
    Map<String, Map<String, IntIndexedContainer>> locations = new TreeMap<>();
    Map<String, IntIndexedContainer> tableLocation = new TreeMap<>();
    tableLocation.put(relationName.fqn(), IntArrayList.from(IntArrayList.EMPTY_ARRAY));
    locations.put(nodeId, tableLocation);
    return new Routing(locations);
}
 
Example 29
Source Project: lucene-solr   Source File: CollapsingQParserPlugin.java    License: Apache License 2.0 5 votes vote down vote up
public OrdFieldValueStrategy(int maxDoc,
                             int valueCount,
                             int nullPolicy,
                             boolean needsScores,
                             IntIntHashMap boostDocsMap,
                             SortedDocValues values) {
  this.ords = new IntIntDynamicMap(valueCount, -1);
  this.nullPolicy = nullPolicy;
  this.needsScores = needsScores;
  this.collapsedSet = new FixedBitSet(maxDoc);
  if(boostDocsMap != null) {
    this.boosts = true;
    this.boostOrds = new IntArrayList();
    this.boostDocs = new IntArrayList();
    int[] bd = new int[boostDocsMap.size()];
    Iterator<IntIntCursor> it =  boostDocsMap.iterator();
    int index = -1;
    while(it.hasNext()) {
      IntIntCursor cursor = it.next();
      bd[++index] = cursor.key;
    }

    Arrays.sort(bd);
    this.mergeBoost = new MergeBoost(bd);
    this.boosted = true;
  }

  if (this.needsScores) {
    this.scores = new IntFloatDynamicMap(valueCount, 0.0f);
    if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
      nullScores = new FloatArrayList();
    }
  }
}
 
Example 30
Source Project: lucene-solr   Source File: CollapsingQParserPlugin.java    License: Apache License 2.0 5 votes vote down vote up
public IntFieldValueStrategy(int maxDoc,
                             int size,
                             String collapseField,
                             int nullValue,
                             int nullPolicy,
                             boolean needsScores,
                             IntIntHashMap boostDocsMap) {
  this.collapseField = collapseField;
  this.nullValue = nullValue;
  this.nullPolicy = nullPolicy;
  this.needsScores = needsScores;
  this.collapsedSet = new FixedBitSet(maxDoc);
  this.cmap = new IntIntHashMap(size);
  this.docs = new IntIntDynamicMap(size, 0);
  if(boostDocsMap != null) {
    this.boosts = true;
    this.boostDocs = new IntArrayList();
    this.boostKeys = new IntArrayList();
    int[] bd = new int[boostDocsMap.size()];
    Iterator<IntIntCursor> it =  boostDocsMap.iterator();
    int index = -1;
    while(it.hasNext()) {
      IntIntCursor cursor = it.next();
      bd[++index] = cursor.key;
    }

    Arrays.sort(bd);
    this.mergeBoost = new MergeBoost(bd);
  }

  if(needsScores) {
    this.scores = new IntFloatDynamicMap(size, 0.0f);
    if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
      nullScores = new FloatArrayList();
    }
  }
}