Java Code Examples for it.unimi.dsi.fastutil.ints.IntSet#add()

public static KeyTestInfo generateRandomKeys(Random random, int maxNumKeys) {
  int maxKeyOrValue = maxNumKeys << 2;
  int[] keysAndValues = new int[maxNumKeys * 3];
  int[] nonKeys = new int[maxNumKeys];
  IntSet keySet = new IntOpenHashBigSet(maxNumKeys);
  for (int i = 0; i < maxNumKeys; i++) {
    int entry;
    do {
      entry = random.nextInt(maxKeyOrValue);
    } while (keySet.contains(entry));
    keysAndValues[i * 3] = entry;
    keysAndValues[i * 3 + 1] = random.nextInt(maxKeyOrValue);
    keysAndValues[i * 3 + 2] = random.nextInt(maxKeyOrValue);
    keySet.add(entry);
  }
  for (int i = 0; i < maxNumKeys; i++) {
    int nonKey;
    do {
      nonKey = random.nextInt(maxKeyOrValue);
    } while (keySet.contains(nonKey));
    nonKeys[i] = nonKey;
  }
  return new KeyTestInfo(keysAndValues, nonKeys);
}
 

public ArrayList<Transition<A>> getBatch(int size) {
    ArrayList<Transition<A>> batch = new ArrayList<>(size);
    int storageSize = storage.size();
    int actualBatchSize = Math.min(storageSize, size);

    int[] actualIndex = new int[actualBatchSize];
    IntSet set = new IntOpenHashSet();
    for( int i=0; i<actualBatchSize; i++ ){
        int next = rnd.nextInt(storageSize);
        while(set.contains(next)){
            next = rnd.nextInt(storageSize);
        }
        set.add(next);
        actualIndex[i] = next;
    }

    for (int i = 0; i < actualBatchSize; i ++) {
        Transition<A> trans = storage.get(actualIndex[i]);
        batch.add(trans.dup());
    }

    return batch;
}
 

@Test
public void testIntSet() {
  for (int i = 0; i < NUM_ITERATIONS; i++) {
    int size = RANDOM.nextInt(100);
    IntSet expected = new IntOpenHashSet(size);
    for (int j = 0; j < size; j++) {
      expected.add(RANDOM.nextInt());
    }

    byte[] bytes = ObjectSerDeUtils.serialize(expected);
    IntSet actual = ObjectSerDeUtils.deserialize(bytes, ObjectSerDeUtils.ObjectType.IntSet);

    // NOTE: use Object comparison instead of Collection comparison because the order might be different
    assertEquals((Object) actual, expected, ERROR_MESSAGE);
  }
}
 

private void initStatistics() {
  IntSet _values = new IntOpenHashSet();
  for (int v : getArray()) {
    _values.add(v);
    if (v < lowValue) {
      lowValue = v;
    }
    if (v > highValue) {
      highValue = v;
    }
  }
  valueCount = _values.size();
}
 

@Override
public IntColumn unique() {
  final IntSet values = new IntOpenHashSet();
  for (int i = 0; i < size(); i++) {
    values.add(getInt(i));
  }
  final IntColumn column = IntColumn.create(name() + " Unique values");
  for (int value : values) {
    column.append(value);
  }
  return column;
}
 

@Override
public DateColumn unique() {
  IntSet ints = new IntOpenHashSet(data.size());
  for (int i = 0; i < size(); i++) {
    ints.add(data.getInt(i));
  }
  DateColumn copy = emptyCopy(ints.size());
  copy.setName(name() + " Unique values");
  copy.data = IntArrayList.wrap(ints.toIntArray());
  return copy;
}
 

@Override
public int countUnique() {
  IntSet ints = new IntOpenHashSet(size());
  for (int i = 0; i < size(); i++) {
    ints.add(data.getInt(i));
  }
  return ints.size();
}
 

@Benchmark
public IntSet givenFastUtilsIntSetWithInitialSizeSet_whenPopulated_checkTimeTaken() {
  IntSet intSet = new IntOpenHashSet(setSize);
  for(int i = 0; i < setSize; i++){
    intSet.add(i);
  }
  return intSet;
}
 

@Override
protected int[][] parseSet() throws IOException {
	final Int2ObjectMap<IntSet> map = new Int2ObjectOpenHashMap<IntSet>(3000000);
	final IntSet hidden= DatasetLoader.get(new HiddenCategoriesWIDs(lang));
	File input = WikipediaFiles.CAT_LINKS.getSourceFile(lang);
	final Object2IntMap<String> categories=DatasetLoader.get(new CategoriesToWIDMap(lang));
	
	SQLWikiParser parser = new SQLWikiParser(log) {
		@Override
		public boolean compute(ArrayList<String> values) throws IOException {
			String c_title=cleanPageName(values.get(SQLWikiParser.CATLINKS_TITLE_TO));
			int id=Integer.parseInt(values.get(SQLWikiParser.CATLINKS_ID_FROM));
			if(categories.containsKey(c_title) && !hidden.contains(categories.get(c_title).intValue())){
				if(map.containsKey(id)){
					map.get(id).add(categories.get(c_title).intValue());
				}else{
					IntSet set = new IntOpenHashSet();
					set.add(categories.get(c_title).intValue());
					map.put(id, set);
				}
				return true;
			} else return false;
		}
		
	};
	InputStreamReader reader = new InputStreamReader(new FileInputStream(input), Charset.forName("UTF-8"));
	parser.compute(reader);
	reader.close();
	return createDump(map);
}
 

@Override
public int countUnique() {
  IntSet ints = new IntOpenHashSet(size());
  for (int i = 0; i < size(); i++) {
    ints.add(data.getInt(i));
  }
  return ints.size();
}
 

@Test
public void testMerge() {
	IntSet set1 = mutableSingleton(1);
	IntSet set2 = mutableSingleton(2);
	IntSet merged = merge(set1, set2);
	assertThat(merged).hasSize(2);
	assertThat(merged).contains(1, 2);
	set1.add(3);
	assertThat(set1).hasSize(3);
	assertThat(set1).contains(1, 2, 3);
}
 

@Test
public void testAsSet() {
	IntSet set = mutableSingleton(1);
	assertThat(set).hasSize(1);
	assertThat(set).contains(1);
	set.add(2);
	assertThat(set).hasSize(2);
	assertThat(set).contains(1, 2);
}
 

@Override
public IntColumn unique() {
  final IntSet values = new IntOpenHashSet();
  for (int i = 0; i < size(); i++) {
    values.add(getInt(i));
  }
  final IntColumn column = IntColumn.create(name() + " Unique values");
  for (int value : values) {
    column.append(value);
  }
  return column;
}
 

@Override
public int countUnique() {
  IntSet uniqueElements = new IntOpenHashSet();
  for (int i = 0; i < size(); i++) {
    uniqueElements.add(getInt(i));
  }
  return uniqueElements.size();
}
 

@Override
protected IntList recommend(Model recommender, int userIndex) {
    ReferenceModule trainModule = trainModules[userIndex];
    ReferenceModule testModule = testModules[userIndex];
    IntSet itemSet = new IntOpenHashSet();
    for (DataInstance instance : trainModule) {
        itemSet.add(instance.getQualityFeature(itemDimension));
    }
    // TODO 此处代码需要重构
    ArrayInstance copy = new ArrayInstance(trainMarker.getQualityOrder(), trainMarker.getQuantityOrder());
    copy.copyInstance(testModule.getInstance(0));
    copy.setQualityFeature(userDimension, userIndex);

    List<Integer2FloatKeyValue> rankList = new ArrayList<>(itemSize - itemSet.size());
    for (int itemIndex = 0; itemIndex < itemSize; itemIndex++) {
        if (itemSet.contains(itemIndex)) {
            continue;
        }
        copy.setQualityFeature(itemDimension, itemIndex);
        recommender.predict(copy);
        rankList.add(new Integer2FloatKeyValue(itemIndex, copy.getQuantityMark()));
    }
    Collections.sort(rankList, (left, right) -> {
        return Float.compare(right.getValue(), left.getValue());
    });

    IntList recommendList = new IntArrayList(rankList.size());
    for (Integer2FloatKeyValue keyValue : rankList) {
        recommendList.add(keyValue.getKey());
    }
    return recommendList;
}
 

@Override
protected IntSet check(int userIndex) {
    ReferenceModule testModule = testModules[userIndex];
    IntSet itemSet = new IntOpenHashSet();
    for (DataInstance instance : testModule) {
        itemSet.add(instance.getQualityFeature(itemDimension));
    }
    return itemSet;
}
 

@Deprecated
// TODO 此方法准备取消,利用向量的有序性代替
protected List<IntSet> getUserItemSet(SparseMatrix sparseMatrix) {
    List<IntSet> userItemSet = new ArrayList<>(userSize);
    for (int userIndex = 0; userIndex < userSize; userIndex++) {
        SparseVector userVector = sparseMatrix.getRowVector(userIndex);
        IntSet indexes = new IntOpenHashSet();
        for (int position = 0, size = userVector.getElementSize(); position < size; position++) {
            indexes.add(userVector.getIndex(position));
        }
        userItemSet.add(indexes);
    }
    return userItemSet;
}
 

private static void testFilter(DictionaryAwarePageFilter filter, Block block, boolean filterRange)
{
    IntSet actualSelectedPositions = toSet(filter.filter(null, new Page(block)));

    block = block.getLoadedBlock();

    IntSet expectedSelectedPositions = new IntArraySet(block.getPositionCount());
    for (int position = 0; position < block.getPositionCount(); position++) {
        if (isSelected(filterRange, block.getLong(position, 0))) {
            expectedSelectedPositions.add(position);
        }
    }
    assertEquals(actualSelectedPositions, expectedSelectedPositions);
}
 

public List<ObjectNode> expand(List<ObjectNode> initialResult,
                               RequestContext requestContext) {
    int indexSize = model.getKeyMapSize(itemIndex);
    int maxVal = indexSize;
    if (maxIdx != null && maxIdx < indexSize) {
        maxVal = maxIdx;
    }
    for (ObjectNode entity : initialResult) {
        String itemStr = entity.get(itemAttr).asText();
        if (!"".equals(itemStr)) {
            String[] items = itemStr.split(separator, -1);
            IntSet trues = new IntOpenHashSet();
            for (String item : items) {
                String key = FeatureExtractorUtilities.composeKey(keyPrefix, item);
                if (model.containsKey(itemIndex, key)) {
                    trues.add(model.getIndexForKey(itemIndex, key));
                } else {
                    trues.add(0);
                }
            }
            IntList samples = getSampledIndices(trues, maxVal);
            List<String> itemArr = Lists.newArrayList(itemStr);
            List<String> labelArr = Lists.newArrayList(entity.get(labelAttr).asText());
            for (int sample : samples) {
                Map<String, String> key2val = FeatureExtractorUtilities.decomposeKey(
                        (String)model.getKeyForIndex(itemIndex, sample));
                itemArr.add(key2val.get(keyPrefix));
                labelArr.add("0");
            }
            if (fillInAttrs != null && fillInAttrs.size() > 0 && samples.size() > 0) {
                for (int i=0; i<fillInAttrs.size(); i++) {
                    String fillStr = entity.get(fillInAttrs.get(i)).asText();
                    String[] fills = fillStr.split(separator, -1);
                    List<String> fillEls = Lists.newArrayList(fillStr);
                    for (int j=0; j<samples.size(); j++) {
                        fillEls.add(fills[fills.length - 1]);
                    }
                    entity.put(fillInAttrs.get(i), StringUtils.join(fillEls, joiner));
                }
            }
            entity.put(labelAttr, StringUtils.join(labelArr, joiner));
            entity.put(itemAttr, StringUtils.join(itemArr, joiner));
        }
    }
    return initialResult;
}
 

private void processPage(Page newPage, GroupByIdBlock groupIds)
{
    checkArgument(newPage != null);
    checkArgument(groupIds != null);

    // save the new page
    PageReference newPageReference = new PageReference(newPage);
    int newPageId;
    if (emptyPageReferenceSlots.isEmpty()) {
        // all the previous slots are full; create a new one
        pageReferences.ensureCapacity(currentPageCount + 1);
        newPageId = currentPageCount;
        currentPageCount++;
    }
    else {
        // reuse a previously removed page's slot
        newPageId = emptyPageReferenceSlots.dequeueInt();
    }
    verify(pageReferences.get(newPageId) == null, "should not overwrite a non-empty slot");
    pageReferences.set(newPageId, newPageReference);

    // update the affected heaps and record candidate pages that need compaction
    IntSet pagesToCompact = new IntOpenHashSet();
    for (int position = 0; position < newPage.getPositionCount(); position++) {
        long groupId = groupIds.getGroupId(position);
        groupedRows.ensureCapacity(groupId + 1);

        RowHeap rows = groupedRows.get(groupId);
        if (rows == null) {
            // a new group
            rows = new RowHeap(Ordering.from(comparator).reversed());
            groupedRows.set(groupId, rows);
        }
        else {
            // update an existing group;
            // remove the memory usage for this group for now; add it back after update
            memorySizeInBytes -= rows.getEstimatedSizeInBytes();
        }

        if (rows.size() < topN) {
            // still have space for the current group
            Row row = new Row(newPageId, position);
            rows.enqueue(row);
            newPageReference.reference(row);
        }
        else {
            // may compare with the topN-th element with in the heap to decide if update is necessary
            Row previousRow = rows.first();
            Row newRow = new Row(newPageId, position);
            if (comparator.compare(newRow, previousRow) < 0) {
                // update reference and the heap
                rows.dequeue();
                PageReference previousPageReference = pageReferences.get(previousRow.getPageId());
                previousPageReference.dereference(previousRow.getPosition());
                newPageReference.reference(newRow);
                rows.enqueue(newRow);

                // compact a page if it is not the current input page and the reference count is below the threshold
                if (previousPageReference.getPage() != newPage &&
                        previousPageReference.getUsedPositionCount() * COMPACT_THRESHOLD < previousPageReference.getPage().getPositionCount()) {
                    pagesToCompact.add(previousRow.getPageId());
                }
            }
        }

        memorySizeInBytes += rows.getEstimatedSizeInBytes();
    }

    // unreference new page if it was not used
    if (newPageReference.getUsedPositionCount() == 0) {
        pageReferences.set(newPageId, null);
    }
    else {
        // assure new page is loaded
        newPageReference.loadPage();
        memorySizeInBytes += newPageReference.getEstimatedSizeInBytes();

        // may compact the new page as well
        if (newPageReference.getUsedPositionCount() * COMPACT_THRESHOLD < newPage.getPositionCount()) {
            verify(!pagesToCompact.contains(newPageId));
            pagesToCompact.add(newPageId);
        }
    }

    // compact pages
    IntIterator iterator = pagesToCompact.iterator();
    while (iterator.hasNext()) {
        int pageId = iterator.nextInt();
        PageReference pageReference = pageReferences.get(pageId);
        if (pageReference.getUsedPositionCount() == 0) {
            pageReferences.set(pageId, null);
            emptyPageReferenceSlots.enqueue(pageId);
            memorySizeInBytes -= pageReference.getEstimatedSizeInBytes();
        }
        else {
            memorySizeInBytes -= pageReference.getEstimatedSizeInBytes();
            pageReference.compact();
            memorySizeInBytes += pageReference.getEstimatedSizeInBytes();
        }
    }
}