Java Code Examples for it.unimi.dsi.fastutil.ints.IntArrayList#size()

The following examples show how to use it.unimi.dsi.fastutil.ints.IntArrayList#size() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: PLIBuilder.java From winter with Apache License 2.0

6 votes

protected static List<PositionListIndex> fetchPositionListIndexesStatic(List<HashMap<String, IntArrayList>> clusterMaps, boolean isNullEqualNull) {
	List<PositionListIndex> clustersPerAttribute = new ArrayList<>();
	for (int columnId = 0; columnId < clusterMaps.size(); columnId++) {
		List<IntArrayList> clusters = new ArrayList<>();
		HashMap<String, IntArrayList> clusterMap = clusterMaps.get(columnId);
		
		if (!isNullEqualNull)
			clusterMap.remove(null);
		
		for (IntArrayList cluster : clusterMap.values())
			if (cluster.size() > 1)
				clusters.add(cluster);
		
		clustersPerAttribute.add(new PositionListIndex(columnId, clusters));
	}
	return clustersPerAttribute;
}

Example 2

Source File: CFDFinder.java From metanome-algorithms with Apache License 2.0

6 votes

public static int findViolationsFor(final IntArrayList cluster, final int[] invertedRhsPLI) {
	int maxSize = 0;
	Int2IntAVLTreeMap clusterMap = new Int2IntAVLTreeMap();
	clusterMap.defaultReturnValue(0);
	for (int tuple : cluster) {
		int clusterId = invertedRhsPLI[tuple];
		if (clusterId == -1) {
			// single-tuple cluster, skip
			continue;
		}
		int count = clusterMap.get(clusterId);
		count += 1;
		clusterMap.put(clusterId, count);
		if (count > maxSize) {
			maxSize = count;
		}
	}
	if (maxSize > 0) {
		return cluster.size() - maxSize;
	}
	// if there is no cluster in the list, there are only single-tuple clusters on the rhs and thus, only one keeper
	return cluster.size() - 1;
}

Example 3

Source File: PLIBuilder.java From metanome-algorithms with Apache License 2.0

6 votes

protected static List<PositionListIndex> fetchPositionListIndexesStatic(List<HashMap<String, IntArrayList>> clusterMaps, boolean isNullEqualNull) {
	List<PositionListIndex> clustersPerAttribute = new ArrayList<>();
	for (int columnId = 0; columnId < clusterMaps.size(); columnId++) {
		List<IntArrayList> clusters = new ArrayList<>();
		HashMap<String, IntArrayList> clusterMap = clusterMaps.get(columnId);
		
		if (!isNullEqualNull)
			clusterMap.remove(null);
		
		for (IntArrayList cluster : clusterMap.values())
			if (cluster.size() > 1)
				clusters.add(cluster);
		
		clustersPerAttribute.add(new PositionListIndex(columnId, clusters));
	}
	return clustersPerAttribute;
}

Example 4

Source File: HyFD.java From winter with Apache License 2.0

6 votes

private void fetchNonFdsWindowingOverClusters(Set<OpenBitSet> negCover, int[][] compressedRecords, List<PositionListIndex> plis) {
	System.out.println("\tMoving window over small clusters ...");
	for (PositionListIndex pli : plis) {
		boolean selectSmallClustersOnly = pli.getClusters().size() < this.attributeThreshold; 	// If there are too few clusters, then the clusters are large and we have already executed sufficient comparisons between the records of these clusters
		
		for (IntArrayList cluster : pli.getClusters()) {
			if (selectSmallClustersOnly && (cluster.size() > this.windowSize))					// But if the current cluster is very small, we should still use it for comparisons (the other cluster(s) must be very large)
				continue;
			
			for (int recordIndex = 0; recordIndex < cluster.size(); recordIndex++) {
				int recordId = cluster.getInt(recordIndex);
				
				for (int partnerRecordIndex = recordIndex + 1; partnerRecordIndex < Math.min(recordIndex + this.windowSize, cluster.size()); partnerRecordIndex++) {
					int partnerRecordId = cluster.getInt(partnerRecordIndex);
					
					negCover.add(this.getViolatedFds(compressedRecords[recordId], compressedRecords[partnerRecordId]));
				}
			}
		}
	}
}

Example 5

Source File: PositionListIndex.java From metanome-algorithms with Apache License 2.0

6 votes

public boolean refines(int[][] lhsInvertedPlis, int[] rhs) {
	for (IntArrayList cluster : this.clusters) {
		Object2IntOpenHashMap<IntArrayList> clustersMap = new Object2IntOpenHashMap<>(cluster.size());
		
		// Check if all subclusters of this cluster point into the same other clusters
		for (int recordId : cluster) {
			IntArrayList additionalLhsCluster = this.buildClusterIdentifier(recordId, lhsInvertedPlis);
			if (additionalLhsCluster == null)
				continue;
			
			if (clustersMap.containsKey(additionalLhsCluster)) {
				if ((rhs[recordId] == -1) || (clustersMap.getInt(additionalLhsCluster) != rhs[recordId]))
					return false;
			}
			else {
				clustersMap.put(additionalLhsCluster, rhs[recordId]);
			}
		}
	}
	return true;
}

Example 6

Source File: HyFD.java From metanome-algorithms with Apache License 2.0

6 votes

private void fetchNonFdsFromClustersTopsAndBottoms(Set<BitSet> negCover, int[][] compressedRecords, List<PositionListIndex> plis) {
	System.out.println("\tComparing window on clusters tops and bottoms ...");
	for (PositionListIndex pli : plis) {
		for (IntArrayList cluster : pli.getClusters()) {
			if (cluster.size() < this.windowSize)
				continue;
			
			for (int recordIndex = 0; recordIndex < this.windowSize; recordIndex++) {
				int recordId = cluster.getInt(recordIndex);
				
				for (int partnerRecordIndex = cluster.size() - 1; partnerRecordIndex > cluster.size() - this.windowSize; partnerRecordIndex--) {
					int partnerRecordId = cluster.getInt(partnerRecordIndex);
					
					if (recordId == partnerRecordId)
						continue;
					
					negCover.add(this.getViolatedFds(compressedRecords[recordId], compressedRecords[partnerRecordId]));
				}
			}
		}
	}
}

Example 7

Source File: PLIBuilder.java From winter with Apache License 2.0

6 votes

protected List<PositionListIndex> fetchPositionListIndexes(List<HashMap<String, IntArrayList>> clusterMaps, boolean isNullEqualNull) {
	List<PositionListIndex> clustersPerAttribute = new ArrayList<>();
	for (int columnId = 0; columnId < clusterMaps.size(); columnId++) {
		List<IntArrayList> clusters = new ArrayList<>();
		HashMap<String, IntArrayList> clusterMap = clusterMaps.get(columnId);
		
		if (!isNullEqualNull)
			clusterMap.remove(null);
		
		for (IntArrayList cluster : clusterMap.values())
			if (cluster.size() > 1)
				clusters.add(cluster);
		
		clustersPerAttribute.add(new PositionListIndex(columnId, clusters));
	}
	return clustersPerAttribute;
}

Example 8

Source File: PositionListIndex.java From metanome-algorithms with Apache License 2.0

6 votes

public boolean refines(int[][] lhsInvertedPlis, int[] rhs) {
	for (IntArrayList cluster : this.clusters) {
		Object2IntOpenHashMap<IntArrayList> clustersMap = new Object2IntOpenHashMap<>(cluster.size());
		
		// Check if all subclusters of this cluster point into the same other clusters
		for (int recordId : cluster) {
			IntArrayList additionalLhsCluster = this.buildClusterIdentifier(recordId, lhsInvertedPlis);
			if (additionalLhsCluster == null)
				continue;
			
			if (clustersMap.containsKey(additionalLhsCluster)) {
				if ((rhs[recordId] == -1) || (clustersMap.getInt(additionalLhsCluster) != rhs[recordId]))
					return false;
			}
			else {
				clustersMap.put(additionalLhsCluster, rhs[recordId]);
			}
		}
	}
	return true;
}

Example 9

Source File: PositionListIndex.java From metanome-algorithms with Apache License 2.0

5 votes

public PositionListIndex intersect(PositionListIndex otherPLI) {
	Int2IntOpenHashMap hashedPLI = otherPLI.asHashMap();
	Int2ObjectMap<Int2ObjectMap<IntArrayList>> intersectMap = this.buildIntersectMap(this, hashedPLI);
	
	List<IntArrayList> clusters = new ArrayList<>();
	for (Int2ObjectMap<IntArrayList> cluster1 : intersectMap.values())
		for (IntArrayList cluster2 : cluster1.values())
			if (cluster2.size() > 1)
				clusters.add(cluster2);
	
	return new PositionListIndex(-1, clusters);
}

Example 10

Source File: PositionListIndex.java From metanome-algorithms with Apache License 2.0

5 votes

public PositionListIndex intersect(int[] otherPLI) {
	Int2ObjectMap<Int2ObjectMap<IntArrayList>> intersectMap = this.buildIntersectMap(otherPLI);
	
	List<IntArrayList> clusters = new ArrayList<>();
	for (Int2ObjectMap<IntArrayList> cluster1 : intersectMap.values())
		for (IntArrayList cluster2 : cluster1.values())
			if (cluster2.size() > 1)
				clusters.add(cluster2);
	
	return new PositionListIndex(-1, clusters);
}

Example 11

Source File: MToNChain.java From twister2 with Apache License 2.0

5 votes

private boolean sendSyncs() {
  boolean sent = false;
  IntArrayList sendingGroup = sendingGroupsWorkers.get(sendGroupIndex);
  for (int j = 0; j < sendingGroup.size(); j++) {
    int worker = sendingGroup.getInt(j);
    IntArrayList workerTargets = workerToTargets.get(worker);
    int targetIndex = sendWorkerTaskIndex.get(worker);

    for (int i = targetIndex; i < workerTargets.size(); i++) {
      int target = workerTargets.getInt(i);
      RoutingParameters parameters = targetRoutes.get(target);
      byte[] message = new byte[1];
      int flags = MessageFlags.SYNC_EMPTY;
      if (delegate.sendMessage(representSource, message, target, flags, parameters)) {
        sent = true;
        // advance the index
        targetIndex++;
        syncSent.add(target);
        sendWorkerTaskIndex.put(worker, targetIndex);
      } else {
        // no point in going further
        return false;
      }
    }
  }


  if (sent) {
    if (syncSent.size() == targetsArray.length) {
      for (int source : thisWorkerSources) {
        sourceStates.put(source, ReceiverState.SYNCED);
      }
    }
    finishedSendGroups.add(sendGroupIndex);
  }
  return true;
}

Example 12

Source File: HyFD.java From metanome-algorithms with Apache License 2.0

5 votes

private void fetchNonFdsFromClustersTopsAndBottomsProgressive(Set<BitSet> negCover, int[][] compressedRecords, List<PositionListIndex> plis) {
	System.out.println("\tComparing window on clusters tops and bottoms ...");
	for (PositionListIndex pli : plis) {
		int currentWindowDistance = 1;
		int newNonFDs = 0;
		do {
			newNonFDs = 0;
			
			for (IntArrayList cluster : pli.getClusters()) {
				int recordIndex = currentWindowDistance;
				int partnerRecordIndex = cluster.size() - currentWindowDistance; 
				
				if ((recordIndex >= cluster.size()) || (partnerRecordIndex < 0) || (recordIndex == partnerRecordIndex))
					continue;
				
				int recordId = cluster.getInt(recordIndex);
				int partnerRecordId = cluster.getInt(partnerRecordIndex);
				
				if (negCover.add(this.getViolatedFds(compressedRecords[recordId], compressedRecords[partnerRecordId])))
					newNonFDs++;
			}
			
			currentWindowDistance++;
		}
		while (newNonFDs > this.progressiveThreshold);
	}
}

Example 13

Source File: Sampler.java From metanome-algorithms with Apache License 2.0

5 votes

public void runNext(UCCList newNonUCCs, int[][] compressedRecords) {
	this.windowDistance++;
	int numNewNonFds = 0;
	int numComparisons = 0;
	BitSet equalAttrs = new BitSet(this.posCover.getNumAttributes());
	
	int previousNewNonFdsSize = newNonUCCs.size();
	Iterator<IntArrayList> clusterIterator = this.clusters.iterator();
	while (clusterIterator.hasNext()) {
		IntArrayList cluster = clusterIterator.next();
		
		if (cluster.size() <= this.windowDistance) {
			clusterIterator.remove();
			continue;
		}
		
		for (int recordIndex = 0; recordIndex < (cluster.size() - this.windowDistance); recordIndex++) {
			int recordId = cluster.getInt(recordIndex);
			int partnerRecordId = cluster.getInt(recordIndex + this.windowDistance);
			
			this.sampler.match(equalAttrs, compressedRecords[recordId], compressedRecords[partnerRecordId]);
			
			if (!this.negCover.contains(equalAttrs)) {
				BitSet equalAttrsCopy = (BitSet) equalAttrs.clone();
				this.negCover.add(equalAttrsCopy);
				newNonUCCs.add(equalAttrsCopy);
				
				this.memoryGuardian.memoryChanged(1);
				this.memoryGuardian.match(this.negCover, this.posCover, newNonUCCs);
			}
			numComparisons++;
		}
	}
	numNewNonFds = newNonUCCs.size() - previousNewNonFdsSize;
	
	this.numNewNonFds.add(numNewNonFds);
	this.numComparisons.add(numComparisons);
}

Example 14

Source File: PositionListIndex.java From winter with Apache License 2.0

5 votes

public PositionListIndex intersect(int[] otherPLI) {
	Int2ObjectMap<Int2ObjectMap<IntArrayList>> intersectMap = this.buildIntersectMap(otherPLI);
	
	List<IntArrayList> clusters = new ArrayList<>();
	for (Int2ObjectMap<IntArrayList> cluster1 : intersectMap.values())
		for (IntArrayList cluster2 : cluster1.values())
			if (cluster2.size() > 1)
				clusters.add(cluster2);
	
	return new PositionListIndex(-1, clusters);
}

Example 15

Source File: Bubble.java From kourami with BSD 3-Clause "New" or "Revised" License

5 votes

public int removeUnsupportedUniqueEdgeOnly(){
//HashSet<Integer> readHash;
//ArrayList<Integer> removalList = new ArrayList<Integer>();
CustomHashMap readHash;
IntArrayList removalList = new IntArrayList();

for(int i=0; i<this.paths.size(); i++){//Path p: this.paths){
    Path p = this.paths.get(i);
    ArrayList<CustomWeightedEdge> eList = p.getOrderedEdgeList();
    boolean inited = false;
    readHash = null;
    int numUnique = 0;
    for(CustomWeightedEdge e : eList){
	HLA.log.append("|" + e.getNumActivePath() + "|");
	if(e.isUniqueEdge()){ //we only care about uniq edges
	    
	    numUnique++;
	    if(!inited){
		readHash = e.getReadHashSet().clone();//e.getReadHashSetDeepCopy();
		inited = true;
	    }else{
		//update with union after checking intersection
		//if null, we remove this path
		if(e.unionAfterCheckingIntersection(readHash) == null){
		    removalList.add(i);//new Integer(i));
		    break;
		}
	    }
	}
    }
    HLA.log.append("[" + numUnique +  "]");
}
for(int i=removalList.size() - 1; i >= 0; i--){
    this.paths.get(removalList.getInt(i)).excludePath();
    this.paths.remove(removalList.getInt(i));
    
}
return removalList.size();
   }

Example 16

Source File: HyFD.java From winter with Apache License 2.0

5 votes

public boolean runNext(Set<OpenBitSet> negCover, int[][] compressedRecords) {
	this.windowDistance++;
	int numNewNonFds = 0;
	int numComparisons = 0;
	
	int previousNegCoverSize = negCover.size();
	Iterator<IntArrayList> clusterIterator = this.clusters.iterator();
	while (clusterIterator.hasNext()) {
		IntArrayList cluster = clusterIterator.next();
		
		if (cluster.size() <= this.windowDistance) {
			clusterIterator.remove();
			continue;
		}
		
		for (int recordIndex = 0; recordIndex < (cluster.size() - this.windowDistance); recordIndex++) {
			int recordId = cluster.getInt(recordIndex);
			int partnerRecordId = cluster.getInt(recordIndex + this.windowDistance);
			negCover.add(this.getViolatedFds(compressedRecords[recordId], compressedRecords[partnerRecordId]));
			numComparisons++;
		}
	}
	numNewNonFds = negCover.size() - previousNegCoverSize;
	
	this.numNewNonFds.add(numNewNonFds);
	this.numComparisons.add(numComparisons);
	
	if (numComparisons == 0)
		return false;
	return true;
}

Example 17

Source File: PositionListIndex.java From metanome-algorithms with Apache License 2.0

5 votes

public PositionListIndex intersect(int[] otherPLI) {
	Int2ObjectMap<Int2ObjectMap<IntArrayList>> intersectMap = this.buildIntersectMap(otherPLI);
	
	List<IntArrayList> clusters = new ArrayList<>();
	for (Int2ObjectMap<IntArrayList> cluster1 : intersectMap.values())
		for (IntArrayList cluster2 : cluster1.values())
			if (cluster2.size() > 1)
				clusters.add(cluster2);
	
	return new PositionListIndex(-1, clusters);
}

Example 18

Source File: PartitioningExchanger.java From presto with Apache License 2.0

5 votes

@Override
public synchronized void accept(Page page)
{
    // reset the assignment lists
    for (IntList partitionAssignment : partitionAssignments) {
        partitionAssignment.clear();
    }

    // assign each row to a partition
    for (int position = 0; position < page.getPositionCount(); position++) {
        int partition = partitionGenerator.getPartition(page, position);
        partitionAssignments[partition].add(position);
    }

    // build a page for each partition
    Block[] outputBlocks = new Block[page.getChannelCount()];
    for (int partition = 0; partition < buffers.size(); partition++) {
        IntArrayList positions = partitionAssignments[partition];
        if (!positions.isEmpty()) {
            for (int i = 0; i < page.getChannelCount(); i++) {
                outputBlocks[i] = page.getBlock(i).copyPositions(positions.elements(), 0, positions.size());
            }

            Page pageSplit = new Page(positions.size(), outputBlocks);
            memoryManager.updateMemoryUsage(pageSplit.getRetainedSizeInBytes());
            buffers.get(partition).accept(new PageReference(pageSplit, 1, () -> memoryManager.updateMemoryUsage(-pageSplit.getRetainedSizeInBytes())));
        }
    }
}

Example 19

Source File: ArbiterStatusListener.java From deeplearning4j with Apache License 2.0

4 votes

@Override
public void onCandidateIteration(CandidateInfo candidateInfo, Object candidate, int iteration) {
    double score;
    long numParams;
    int numLayers;
    String modelConfigJson;
    int totalNumUpdates;
    if(candidate instanceof MultiLayerNetwork){
        MultiLayerNetwork m = (MultiLayerNetwork)candidate;
        score = m.score();
        numParams = m.numParams();
        numLayers = m.getnLayers();
        modelConfigJson = m.getLayerWiseConfigurations().toJson();
        totalNumUpdates = m.getLayerWiseConfigurations().getIterationCount();
    } else if(candidate instanceof ComputationGraph) {
        ComputationGraph cg = (ComputationGraph)candidate;
        score = cg.score();
        numParams = cg.numParams();
        numLayers = cg.getNumLayers();
        modelConfigJson = cg.getConfiguration().toJson();
        totalNumUpdates = cg.getConfiguration().getIterationCount();
    } else {
        score = 0;
        numParams = 0;
        numLayers = 0;
        totalNumUpdates = 0;
        modelConfigJson = "";
    }

    int idx = candidateInfo.getIndex();

    Pair<IntArrayList, FloatArrayList> pair = candidateScoreVsIter.computeIfAbsent(idx, k -> new Pair<>(new IntArrayList(), new FloatArrayList()));

    IntArrayList iter = pair.getFirst();
    FloatArrayList scores = pair.getSecond();

    //Do we need subsampling to avoid having too many data points?
    int subsamplingFreq = candidateScoreVsIterSubsampleFreq.computeIfAbsent(idx, k -> 1);
    if(iteration / subsamplingFreq > MAX_SCORE_VS_ITER_PTS){
        //Double subsampling frequency and re-parse data
        subsamplingFreq *= 2;
        candidateScoreVsIterSubsampleFreq.put(idx, subsamplingFreq);

        IntArrayList newIter = new IntArrayList();
        FloatArrayList newScores = new FloatArrayList();
        for( int i=0; i<iter.size(); i++ ){
            int it = iter.get(i);
            if(it % subsamplingFreq == 0){
                newIter.add(it);
                newScores.add(scores.get(i));
            }
        }

        iter = newIter;
        scores = newScores;
        candidateScoreVsIter.put(idx, new Pair<>(iter, scores));
    }

    if(iteration % subsamplingFreq == 0) {
        iter.add(iteration);
        scores.add((float) score);
    }


    int[] iters = iter.toIntArray();
    float[] fScores = new float[iters.length];
    for( int i=0; i<iters.length; i++ ){
        fScores[i] = scores.get(i);
    }

    ModelInfoPersistable p = new ModelInfoPersistable.Builder()
            .timestamp(candidateInfo.getCreatedTime())
            .sessionId(sessionId)
            .workerId(String.valueOf(candidateInfo.getIndex()))
            .modelIdx(candidateInfo.getIndex())
            .score(candidateInfo.getScore())
            .status(candidateInfo.getCandidateStatus())
            .scoreVsIter(iters, fScores)
            .lastUpdateTime(System.currentTimeMillis())
            .numParameters(numParams)
            .numLayers(numLayers)
            .totalNumUpdates(totalNumUpdates)
            .paramSpaceValues(candidateInfo.getFlatParams())
            .modelConfigJson(modelConfigJson)
            .exceptionStackTrace(candidateInfo.getExceptionStackTrace())
            .build();


    lastModelInfoPersistable.put(candidateInfo.getIndex(), p);
    statsStorage.putUpdate(p);
}

Example 20

Source File: PositionListIndex.java From winter with Apache License 2.0

4 votes

protected int countNonUniqueValuesIn(List<IntArrayList> clusters) {
	int numNonUniqueValues = 0;
	for (IntArrayList cluster : clusters)
		numNonUniqueValues += cluster.size();
	return numNonUniqueValues;
}