com.carrotsearch.hppc.ObjectIntHashMap Java Examples

The following examples show how to use com.carrotsearch.hppc.ObjectIntHashMap. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: JoinOrdering.java    From crate with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
static Set<RelationName> findAndRemoveFirstJoinPair(ObjectIntHashMap<RelationName> occurrences,
                                                    Collection<Set<RelationName>> joinPairs) {
    Iterator<Set<RelationName>> setsIterator = joinPairs.iterator();
    while (setsIterator.hasNext()) {
        Set<RelationName> set = setsIterator.next();
        for (RelationName name : set) {
            int count = occurrences.getOrDefault(name, 0);
            if (count > 1) {
                setsIterator.remove();
                return set;
            }
        }
    }
    return joinPairs.iterator().next();
}
 
Example #2
Source File: HppcMaps.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
public static <V> ObjectIntHashMap<V> ensureNoNullKeys(int capacity, float loadFactor) {
    return new ObjectIntHashMap<V>(capacity, loadFactor) {
        @Override
        public int put(V key, int value) {
            if (key == null) {
                throw new IllegalArgumentException("Map key must not be null");
            }
            return super.put(key, value);
        }
    };
}
 
Example #3
Source File: JoinOrderingTest.java    From crate with Apache License 2.0 5 votes vote down vote up
@Test
public void testFindFirstJoinPairOnlyOneOccurrence() {
    // SELECT * FROM t1, t2, t3 WHERE t1.id = t2.id AND t3.id = t4.id
    ObjectIntHashMap<RelationName> occurrences = new ObjectIntHashMap<>(4);
    occurrences.put(T3.T1, 1);
    occurrences.put(T3.T2, 1);
    occurrences.put(T3.T3, 1);
    occurrences.put(T3.T4, 1);
    Set<Set<RelationName>> sets = Sets.newLinkedHashSet();
    sets.add(Set.of(T3.T1, T3.T2));
    sets.add(Set.of(T3.T2, T3.T3));
    sets.add(Set.of(T3.T3, T3.T4));
    assertThat(JoinOrdering.findAndRemoveFirstJoinPair(occurrences, sets), is(Set.of(T3.T1, T3.T2)));
}
 
Example #4
Source File: JoinOrderingTest.java    From crate with Apache License 2.0 5 votes vote down vote up
@Test
public void testFindFirstJoinPair() {
    // SELECT * FROM t1, t2, t3 WHERE t1.id = t2.id AND t2.id = t3.id AND t3.id = t4.id
    ObjectIntHashMap<RelationName> occurrences = new ObjectIntHashMap<>(4);
    occurrences.put(T3.T1, 1);
    occurrences.put(T3.T2, 1);
    occurrences.put(T3.T3, 3);
    occurrences.put(T3.T4, 1);
    ArrayList<Set<RelationName>> joinPairs = new ArrayList<>();
    joinPairs.add(Set.of(T3.T1, T3.T2));
    joinPairs.add(Set.of(T3.T2, T3.T3));
    joinPairs.add(Set.of(T3.T3, T3.T4));
    assertThat(JoinOrdering.findAndRemoveFirstJoinPair(occurrences, joinPairs), is(Set.of(T3.T2, T3.T3)));
}
 
Example #5
Source File: JoinOrdering.java    From crate with Apache License 2.0 5 votes vote down vote up
private static ObjectIntHashMap<RelationName> getOccurrencesInJoinConditions(
    int numberOfRelations,
    Set<? extends Set<RelationName>> explicitJoinedRelations,
    Set<? extends Set<RelationName>> implicitJoinedRelations) {

    ObjectIntHashMap<RelationName> occurrences = new ObjectIntHashMap<>(numberOfRelations);
    explicitJoinedRelations.forEach(o -> o.forEach(qName -> occurrences.putOrAdd(qName, 1, 1)));
    implicitJoinedRelations.forEach(o -> o.forEach(qName -> occurrences.putOrAdd(qName, 1, 1)));
    return occurrences;
}
 
Example #6
Source File: CombinedDeletionPolicy.java    From crate with Apache License 2.0 5 votes vote down vote up
CombinedDeletionPolicy(Logger logger, TranslogDeletionPolicy translogDeletionPolicy,
                       SoftDeletesPolicy softDeletesPolicy, LongSupplier globalCheckpointSupplier) {
    this.logger = logger;
    this.translogDeletionPolicy = translogDeletionPolicy;
    this.softDeletesPolicy = softDeletesPolicy;
    this.globalCheckpointSupplier = globalCheckpointSupplier;
    this.snapshottedCommits = new ObjectIntHashMap<>();
}
 
Example #7
Source File: HppcObjectIntMapTest.java    From hashmapTest with The Unlicense 5 votes vote down vote up
@Override
public int test() {
    final ObjectIntHashMap<Integer> m_map = new ObjectIntHashMap<>( m_keys.length / 2 + 1, m_fillFactor );
    int add = 0, remove = 0;
    while ( add < m_keys.length )
    {
        m_map.put( m_keys[ add ], add );
        ++add;
        m_map.put( m_keys[ add ], add );
        ++add;
        m_map.remove( m_keys[ remove++ ] );
    }
    return m_map.size();
}
 
Example #8
Source File: HppcObjectIntMapTest.java    From hashmapTest with The Unlicense 5 votes vote down vote up
@Override
public int test() {
    final ObjectIntHashMap<Integer> m_map = new ObjectIntHashMap<>( m_keys.length, m_fillFactor );
    for ( int i = 0; i < m_keys.length; ++i )
        m_map.put( m_keys[ i ], i );
    for ( int i = 0; i < m_keys2.length; ++i )
        m_map.put( m_keys2[ i ], i );
    return m_map.size();
}
 
Example #9
Source File: SFA.java    From SFA with GNU General Public License v3.0 5 votes vote down vote up
protected int moveElement(
    List<ValueLabel> element,
    ObjectIntHashMap<Double> cIn, ObjectIntHashMap<Double> cOut,
    int pos) {
  cIn.putOrAdd(element.get(pos).label, 1, 1);
  cOut.putOrAdd(element.get(pos).label, -1, -1);
  return 1;
}
 
Example #10
Source File: SFA.java    From SFA with GNU General Public License v3.0 5 votes vote down vote up
protected static double calculateInformationGain(
    ObjectIntHashMap<Double> cIn, ObjectIntHashMap<Double> cOut,
    double class_entropy,
    double total_c_in,
    double total) {
  double total_c_out = (total - total_c_in);
  return class_entropy
      - total_c_in / total * entropy(cIn, total_c_in)
      - total_c_out / total * entropy(cOut, total_c_out);
}
 
Example #11
Source File: ClusterStatsNodes.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public void readFrom(StreamInput in) throws IOException {
    int size = in.readVInt();
    versions = new ObjectIntHashMap<>(size);
    for (; size > 0; size--) {
        versions.addTo(JvmVersion.readJvmVersion(in), in.readVInt());
    }
    threads = in.readVLong();
    maxUptime = in.readVLong();
    heapUsed = in.readVLong();
    heapMax = in.readVLong();
}
 
Example #12
Source File: ClusterStatsNodes.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
JvmStats() {
    versions = new ObjectIntHashMap<>();
    threads = 0;
    maxUptime = 0;
    heapMax = 0;
    heapUsed = 0;
}
 
Example #13
Source File: HppcObjectIntMapTest.java    From hashmapTest with The Unlicense 4 votes vote down vote up
@Override
public void setup(int[] keys, float fillFactor, final int oneFailureOutOf ) {
    super.setup(keys, fillFactor, oneFailureOutOf);
    m_map = new ObjectIntHashMap<>( keys.length, fillFactor );
    for ( Integer key : keys ) m_map.put( new Integer( key % oneFailureOutOf == 0 ? key+1 : key), key );
}
 
Example #14
Source File: DictionaryBuilder.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up
public DictionaryBuilder(final Class<T> clazzType) {
  this.clazzType = clazzType;
  this.valueAccumulator = new ObjectIntHashMap<>();
  this.lookupIndex = -1;
  this.isBuilt = false;
}
 
Example #15
Source File: AwarenessAllocationDecider.java    From crate with Apache License 2.0 4 votes vote down vote up
private Decision underCapacity(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation, boolean moveToNode) {
    if (awarenessAttributes.isEmpty()) {
        return allocation.decision(Decision.YES, NAME,
            "allocation awareness is not enabled, set cluster setting [%s] to enable it",
            CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey());
    }

    IndexMetaData indexMetaData = allocation.metaData().getIndexSafe(shardRouting.index());
    int shardCount = indexMetaData.getNumberOfReplicas() + 1; // 1 for primary
    for (String awarenessAttribute : awarenessAttributes) {
        // the node the shard exists on must be associated with an awareness attribute
        if (!node.node().getAttributes().containsKey(awarenessAttribute)) {
            return allocation.decision(Decision.NO, NAME,
                "node does not contain the awareness attribute [%s]; required attributes cluster setting [%s=%s]",
                awarenessAttribute, CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(),
                allocation.debugDecision() ? Strings.collectionToCommaDelimitedString(awarenessAttributes) : null);
        }

        // build attr_value -> nodes map
        ObjectIntHashMap<String> nodesPerAttribute = allocation.routingNodes().nodesPerAttributesCounts(awarenessAttribute);

        // build the count of shards per attribute value
        ObjectIntHashMap<String> shardPerAttribute = new ObjectIntHashMap<>();
        for (ShardRouting assignedShard : allocation.routingNodes().assignedShards(shardRouting.shardId())) {
            if (assignedShard.started() || assignedShard.initializing()) {
                // Note: this also counts relocation targets as that will be the new location of the shard.
                // Relocation sources should not be counted as the shard is moving away
                RoutingNode routingNode = allocation.routingNodes().node(assignedShard.currentNodeId());
                shardPerAttribute.addTo(routingNode.node().getAttributes().get(awarenessAttribute), 1);
            }
        }

        if (moveToNode) {
            if (shardRouting.assignedToNode()) {
                String nodeId = shardRouting.relocating() ? shardRouting.relocatingNodeId() : shardRouting.currentNodeId();
                if (!node.nodeId().equals(nodeId)) {
                    // we work on different nodes, move counts around
                    shardPerAttribute.putOrAdd(allocation.routingNodes().node(nodeId).node().getAttributes().get(awarenessAttribute),
                            0, -1);
                    shardPerAttribute.addTo(node.node().getAttributes().get(awarenessAttribute), 1);
                }
            } else {
                shardPerAttribute.addTo(node.node().getAttributes().get(awarenessAttribute), 1);
            }
        }

        int numberOfAttributes = nodesPerAttribute.size();
        List<String> fullValues = forcedAwarenessAttributes.get(awarenessAttribute);
        if (fullValues != null) {
            for (String fullValue : fullValues) {
                if (!shardPerAttribute.containsKey(fullValue)) {
                    numberOfAttributes++;
                }
            }
        }
        // TODO should we remove ones that are not part of full list?

        int averagePerAttribute = shardCount / numberOfAttributes;
        int totalLeftover = shardCount % numberOfAttributes;
        int requiredCountPerAttribute;
        if (averagePerAttribute == 0) {
            // if we have more attributes values than shard count, no leftover
            totalLeftover = 0;
            requiredCountPerAttribute = 1;
        } else {
            requiredCountPerAttribute = averagePerAttribute;
        }
        int leftoverPerAttribute = totalLeftover == 0 ? 0 : 1;

        int currentNodeCount = shardPerAttribute.get(node.node().getAttributes().get(awarenessAttribute));
        // if we are above with leftover, then we know we are not good, even with mod
        if (currentNodeCount > (requiredCountPerAttribute + leftoverPerAttribute)) {
            return allocation.decision(Decision.NO, NAME,
                    "there are too many copies of the shard allocated to nodes with attribute [%s], there are [%d] total configured " +
                    "shard copies for this shard id and [%d] total attribute values, expected the allocated shard count per " +
                    "attribute [%d] to be less than or equal to the upper bound of the required number of shards per attribute [%d]",
                    awarenessAttribute,
                    shardCount,
                    numberOfAttributes,
                    currentNodeCount,
                    requiredCountPerAttribute + leftoverPerAttribute);
        }
        // all is well, we are below or same as average
        if (currentNodeCount <= requiredCountPerAttribute) {
            continue;
        }
    }

    return allocation.decision(Decision.YES, NAME, "node meets all awareness attribute requirements");
}
 
Example #16
Source File: ClusterStatsNodes.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
public ObjectIntHashMap<JvmVersion> getVersions() {
    return versions;
}
 
Example #17
Source File: ClusterStatsNodes.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
public OsStats() {
    names = new ObjectIntHashMap<>();
}
 
Example #18
Source File: AwarenessAllocationDecider.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
private Decision underCapacity(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation, boolean moveToNode) {
    if (awarenessAttributes.length == 0) {
        return allocation.decision(Decision.YES, NAME, "no allocation awareness enabled");
    }

    IndexMetaData indexMetaData = allocation.metaData().index(shardRouting.index());
    int shardCount = indexMetaData.getNumberOfReplicas() + 1; // 1 for primary
    for (String awarenessAttribute : awarenessAttributes) {
        // the node the shard exists on must be associated with an awareness attribute
        if (!node.node().attributes().containsKey(awarenessAttribute)) {
            return allocation.decision(Decision.NO, NAME, "node does not contain awareness attribute: [%s]", awarenessAttribute);
        }

        // build attr_value -> nodes map
        ObjectIntHashMap<String> nodesPerAttribute = allocation.routingNodes().nodesPerAttributesCounts(awarenessAttribute);

        // build the count of shards per attribute value
        ObjectIntHashMap<String> shardPerAttribute = new ObjectIntHashMap<>();
        for (ShardRouting assignedShard : allocation.routingNodes().assignedShards(shardRouting)) {
            if (assignedShard.started() || assignedShard.initializing()) {
                // Note: this also counts relocation targets as that will be the new location of the shard.
                // Relocation sources should not be counted as the shard is moving away
                RoutingNode routingNode = allocation.routingNodes().node(assignedShard.currentNodeId());
                shardPerAttribute.addTo(routingNode.node().attributes().get(awarenessAttribute), 1);
            }
        }

        if (moveToNode) {
            if (shardRouting.assignedToNode()) {
                String nodeId = shardRouting.relocating() ? shardRouting.relocatingNodeId() : shardRouting.currentNodeId();
                if (!node.nodeId().equals(nodeId)) {
                    // we work on different nodes, move counts around
                    shardPerAttribute.putOrAdd(allocation.routingNodes().node(nodeId).node().attributes().get(awarenessAttribute), 0, -1);
                    shardPerAttribute.addTo(node.node().attributes().get(awarenessAttribute), 1);
                }
            } else {
                shardPerAttribute.addTo(node.node().attributes().get(awarenessAttribute), 1);
            }
        }

        int numberOfAttributes = nodesPerAttribute.size();
        String[] fullValues = forcedAwarenessAttributes.get(awarenessAttribute);
        if (fullValues != null) {
            for (String fullValue : fullValues) {
                if (!shardPerAttribute.containsKey(fullValue)) {
                    numberOfAttributes++;
                }
            }
        }
        // TODO should we remove ones that are not part of full list?

        int averagePerAttribute = shardCount / numberOfAttributes;
        int totalLeftover = shardCount % numberOfAttributes;
        int requiredCountPerAttribute;
        if (averagePerAttribute == 0) {
            // if we have more attributes values than shard count, no leftover
            totalLeftover = 0;
            requiredCountPerAttribute = 1;
        } else {
            requiredCountPerAttribute = averagePerAttribute;
        }
        int leftoverPerAttribute = totalLeftover == 0 ? 0 : 1;

        int currentNodeCount = shardPerAttribute.get(node.node().attributes().get(awarenessAttribute));
        // if we are above with leftover, then we know we are not good, even with mod
        if (currentNodeCount > (requiredCountPerAttribute + leftoverPerAttribute)) {
            return allocation.decision(Decision.NO, NAME,
                    "too many shards on node for attribute: [%s], required per attribute: [%d], node count: [%d], leftover: [%d]",
                    awarenessAttribute, requiredCountPerAttribute, currentNodeCount, leftoverPerAttribute);
        }
        // all is well, we are below or same as average
        if (currentNodeCount <= requiredCountPerAttribute) {
            continue;
        }
    }

    return allocation.decision(Decision.YES, NAME, "node meets awareness requirements");
}