Java Code Examples for it.unimi.dsi.fastutil.longs.LongSet#add()

@Test
public void testBlacklistOnlyTwoAuthors() {
  NodeMetadataLeftIndexedMultiSegmentBipartiteGraph mockGraph = getMockGraph();

  SmallArrayBasedLongToDoubleMap[] socialProofs = {};

  LongSet whitelistAuthors = new LongArraySet();
  LongSet blacklistAuthors = new LongArraySet();
  blacklistAuthors.add(3L);
  blacklistAuthors.add(4L);

  TweetAuthorFilter authorFilter = new TweetAuthorFilter(
      mockGraph, whitelistAuthors, blacklistAuthors, new NullStatsReceiver());

  // 10L and 20L are authored by user 1 and 2. Do not filter them
  assertEquals(false, authorFilter.filterResult(10, socialProofs));
  assertEquals(false, authorFilter.filterResult(20, socialProofs));
  // 30L and 40L are authored by user 3 and 4. Filter them
  assertEquals(true, authorFilter.filterResult(30, socialProofs));
  assertEquals(true, authorFilter.filterResult(40, socialProofs));
}
 

@Test
public void testBlacklistWhitelistNoOverlap() {
  NodeMetadataLeftIndexedMultiSegmentBipartiteGraph mockGraph = getMockGraph();

  SmallArrayBasedLongToDoubleMap[] socialProofs = {};

  LongSet whitelistAuthors = new LongArraySet();
  whitelistAuthors.add(1L);
  whitelistAuthors.add(2L);

  LongSet blacklistAuthors = new LongArraySet();
  blacklistAuthors.add(3L);
  blacklistAuthors.add(4L);

  TweetAuthorFilter authorFilter = new TweetAuthorFilter(
      mockGraph, whitelistAuthors, blacklistAuthors, new NullStatsReceiver());

  // Should only return whitelisted 10 and 20. 30 and 40 are filtered by blacklist
  assertEquals(false, authorFilter.filterResult(10, socialProofs));
  assertEquals(false, authorFilter.filterResult(20, socialProofs));
  assertEquals(true, authorFilter.filterResult(30, socialProofs));
  assertEquals(true, authorFilter.filterResult(40, socialProofs));
}
 

/**
 * Return the list of tweets authored by the input list of users
 */
private LongSet getTweetsByAuthors(
    LeftIndexedMultiSegmentBipartiteGraph leftIndexedBipartiteGraph,
    LongSet tweetAuthors) {
  LongSet authoredTweets = new LongOpenHashSet();
  for (long authorId: tweetAuthors) {
    EdgeIterator edgeIterator = leftIndexedBipartiteGraph.getLeftNodeEdges(authorId);
    if (edgeIterator == null) {
      continue;
    }

    // Sequentially iterating through the latest MAX_EDGES_PER_NODE edges per node
    int numEdgesPerNode = 0;
    while (edgeIterator.hasNext() && numEdgesPerNode++ < RecommendationRequest.MAX_EDGES_PER_NODE) {
      long rightNode = edgeIterator.nextLong();
      byte edgeType = edgeIterator.currentEdgeType();
      if (edgeType == RecommendationRequest.AUTHOR_SOCIAL_PROOF_TYPE) {
        authoredTweets.add(rightNode);
      }
    }
  }
  return authoredTweets;
}
 

private boolean doTruncateInvalidTxBefore(long time) throws InvalidTruncateTimeException {
  LOG.info("Removing tx ids before {} from invalid list", time);
  long truncateWp = time * TxConstants.MAX_TX_PER_MS;
  // Check if there any in-progress transactions started earlier than truncate time
  if (inProgress.lowerKey(truncateWp) != null) {
    throw new InvalidTruncateTimeException("Transactions started earlier than " + time + " are in-progress");
  }
  
  // Find all invalid transactions earlier than truncateWp
  LongSet toTruncate = new LongArraySet();
  LongIterator it = invalidTxList.toRawList().iterator();
  while (it.hasNext()) {
    long wp = it.nextLong();
    if (wp < truncateWp) {
      toTruncate.add(wp);
    }
  }
  LOG.info("Removing tx ids {} from invalid list", toTruncate);
  return invalidTxList.removeAll(toTruncate);
}
 

@Test
public void testBlacklistWhitelistPartialOverlap() {
  NodeMetadataLeftIndexedMultiSegmentBipartiteGraph mockGraph = getMockGraph();

  SmallArrayBasedLongToDoubleMap[] socialProofs = {};

  LongSet whitelistAuthors = new LongArraySet();
  whitelistAuthors.add(1L);
  whitelistAuthors.add(2L);
  whitelistAuthors.add(3L);

  LongSet blacklistAuthors = new LongArraySet();
  blacklistAuthors.add(2L);
  blacklistAuthors.add(3L);
  blacklistAuthors.add(4L);

  TweetAuthorFilter authorFilter = new TweetAuthorFilter(
      mockGraph, whitelistAuthors, blacklistAuthors, new NullStatsReceiver());

  // Only return 10, since it is whitelisted and not blacklisted
  assertEquals(false, authorFilter.filterResult(10, socialProofs));
  assertEquals(true, authorFilter.filterResult(20, socialProofs));
  assertEquals(true, authorFilter.filterResult(30, socialProofs));
  assertEquals(true, authorFilter.filterResult(40, socialProofs));
}
 

@Override
public int countUnique() {
  LongSet uniqueElements = new LongOpenHashSet();
  for (int i = 0; i < size(); i++) {
    uniqueElements.add(getLong(i));
  }
  return uniqueElements.size();
}
 

private static boolean isLessThanMinUserSocialProofSizeCombined(
  SmallArrayBasedLongToDoubleMap[] socialProofs,
  int minUserSocialProofSize,
  Set<byte[]> socialProofTypeUnions) {
  if (socialProofTypeUnions.isEmpty() ||
    // check if the size of any social proof union is greater than minUserSocialProofSize before dedupping
    isSocialProofUnionSizeLessThanMin(socialProofs, minUserSocialProofSize, socialProofTypeUnions)) {
    return true;
  }

  LongSet uniqueNodes = new LongOpenHashSet(minUserSocialProofSize);

  for (byte[] socialProofTypeUnion: socialProofTypeUnions) {
    // Clear removes all elements, but does not change the size of the set.
    // Thus, we only use one LongOpenHashSet with at most a size of 2*minUserSocialProofSize
    uniqueNodes.clear();
    for (byte socialProofType: socialProofTypeUnion) {
      if (socialProofs[socialProofType] != null) {
        for (int i = 0; i < socialProofs[socialProofType].size(); i++) {
          uniqueNodes.add(socialProofs[socialProofType].keys()[i]);
          if (uniqueNodes.size() >= minUserSocialProofSize) {
            return false;
          }
        }
      }
    }
  }
  return true;
}
 

@Override
public InstantColumn unique() {
  LongSet ints = new LongOpenHashSet(data.size());
  for (long i : data) {
    ints.add(i);
  }
  InstantColumn column = emptyCopy(ints.size());
  column.setName(name() + " Unique values");
  column.data = LongArrayList.wrap(ints.toLongArray());
  return column;
}
 

@Override
public DateTimeColumn unique() {
  LongSet ints = new LongOpenHashSet(data.size());
  for (long i : data) {
    ints.add(i);
  }
  DateTimeColumn column = emptyCopy(ints.size());
  column.setName(name() + " Unique values");
  column.data = LongArrayList.wrap(ints.toLongArray());
  return column;
}
 

@Override
public int countUnique() {
  LongSet ints = new LongOpenHashSet(data.size());
  for (long i : data) {
    ints.add(i);
  }
  return ints.size();
}
 

@Override
public int countUnique() {
  LongSet ints = new LongOpenHashSet(data.size());
  for (long i : data) {
    ints.add(i);
  }
  return ints.size();
}
 

@Override
public int countUnique() {
  LongSet uniqueElements = new LongOpenHashSet();
  for (int i = 0; i < size(); i++) {
    uniqueElements.add(getLong(i));
  }
  return uniqueElements.size();
}
 

@Override
public LongColumn unique() {
  final LongSet values = new LongOpenHashSet();
  for (int i = 0; i < size(); i++) {
    values.add(getLong(i));
  }
  final LongColumn column = LongColumn.create(name() + " Unique values");
  for (long value : values) {
    column.append(value);
  }
  return column;
}
 

private void intersect(LongSet positions, LongSet indexSet) {

        LongSet toRemove = new LongArraySet();
        for (long l : positions) {
            if (!indexSet.contains(l)) {
                toRemove.add(l);
            }
        }
        positions.removeAll(toRemove);
    }
 

@Override
public int countUnique() {
  LongSet ints = new LongOpenHashSet(data.size());
  for (long i : data) {
    ints.add(i);
  }
  return ints.size();
}
 

private Boolean findLog(Block pos, LongSet visited, Integer distance, Integer check, BlockFace fromSide) {
    ++check;
    long index = Hash.hashBlock((int) pos.x, (int) pos.y, (int) pos.z);
    if (visited.contains(index)) return false;
    if (pos.getId() == WOOD || pos.getId() == WOOD2) return true;
    if ((pos.getId() == LEAVES || pos.getId() == LEAVES2) && distance <= 4) {
        visited.add(index);
        int down = pos.down().getId();
        if (down == WOOD || down == WOOD2) {
            return true;
        }
        if (fromSide == null) {
            //North, East, South, West
            for (int side = 2; side <= 5; ++side) {
                if (this.findLog(pos.getSide(BlockFace.fromIndex(side)), visited, distance + 1, check, BlockFace.fromIndex(side)))
                    return true;
            }
        } else { //No more loops
            switch (fromSide) {
                case NORTH:
                    if (this.findLog(pos.getSide(BlockFace.NORTH), visited, distance + 1, check, fromSide))
                        return true;
                    if (this.findLog(pos.getSide(BlockFace.WEST), visited, distance + 1, check, fromSide))
                        return true;
                    if (this.findLog(pos.getSide(BlockFace.EAST), visited, distance + 1, check, fromSide))
                        return true;
                    break;
                case SOUTH:
                    if (this.findLog(pos.getSide(BlockFace.SOUTH), visited, distance + 1, check, fromSide))
                        return true;
                    if (this.findLog(pos.getSide(BlockFace.WEST), visited, distance + 1, check, fromSide))
                        return true;
                    if (this.findLog(pos.getSide(BlockFace.EAST), visited, distance + 1, check, fromSide))
                        return true;
                    break;
                case WEST:
                    if (this.findLog(pos.getSide(BlockFace.NORTH), visited, distance + 1, check, fromSide))
                        return true;
                    if (this.findLog(pos.getSide(BlockFace.SOUTH), visited, distance + 1, check, fromSide))
                        return true;
                    if (this.findLog(pos.getSide(BlockFace.WEST), visited, distance + 1, check, fromSide))
                        return true;
                case EAST:
                    if (this.findLog(pos.getSide(BlockFace.NORTH), visited, distance + 1, check, fromSide))
                        return true;
                    if (this.findLog(pos.getSide(BlockFace.SOUTH), visited, distance + 1, check, fromSide))
                        return true;
                    if (this.findLog(pos.getSide(BlockFace.EAST), visited, distance + 1, check, fromSide))
                        return true;
                    break;
            }
        }
    }
    return false;
}
 

@Override
protected void writeColumnsAndSample(RelationalInput input) throws InputIterationException, IOException {
    List<List<String>> alternativeSamples = new ArrayList<>();
    final List<LongSet> sampledColumnValues = new ArrayList<>();
    for (int i = 0; i < this.getNumberOfColumns(); i++) {
        sampledColumnValues.add(new LongOpenHashSet(this.sampleGoal < 0 ? 1000 : this.sampleGoal));
    }
    // Keeps track of the first value in each column if it is the only value seen.
    Long[] firstColumnValues = new Long[this.getNumberOfColumns()];

    int rowCounter = 0;
    DebugCounter counter = new DebugCounter();
    while (input.hasNext()) {
        List<String> row = input.next();
        boolean isSampleCompleted = true;
        boolean rowHasUnseenValue = false;
        for (int i = 0; i < this.getNumberOfColumns(); i++) {
            // Write the hash to the column.
            String str = row.get(i);
            long hash = getHash(str, i);

            // Keep track of the first column value and delete it if multiple values are observed.
            if (rowCounter == 0) firstColumnValues[i] = Long.valueOf(hash);
            else if (firstColumnValues[i] != null && firstColumnValues[i].longValue() != hash) firstColumnValues[i] = null;

            // Check if the value requests to put the row into the sample.
            if (hash != NULLHASH) {
                final LongSet sampledValues = sampledColumnValues.get(i);
                boolean shouldSample = this.sampleGoal < 0 || sampledValues.size() < this.sampleGoal;
                isSampleCompleted &= !shouldSample;
                if (shouldSample && sampledValues.add(hash)) {
                    rowHasUnseenValue = true;
                }
            }
        }

        if (rowHasUnseenValue) {
            alternativeSamples.add(row);
        }

        counter.countUp();
        rowCounter++;

        if (isSampleCompleted) break;
    }
    counter.done();
    writeSample(alternativeSamples);

    // Check for constant and null columns.
    for (int i = 0; i < firstColumnValues.length; i++) {
        this.isNullColumn[i] = firstColumnValues[i] != null && firstColumnValues[i].longValue() == NULLHASH;
        this.isConstantColumn[i] = firstColumnValues[i] != null && firstColumnValues[i].longValue() != NULLHASH;
    }
}
 

@Override
protected void writeColumnsAndSample(RelationalInput input) throws InputIterationException, IOException {
    boolean isWritingAnyColumn = false;
    FileOutputStream[] out = new FileOutputStream[columnFiles.length];
    FileChannel[] channel = new FileChannel[columnFiles.length];
    ByteBuffer[] bb = new ByteBuffer[columnFiles.length];
    for (int i = 0; i < columnFiles.length; i++) {
        if (!isNew[i]) continue;
        out[i] = new FileOutputStream(columnFiles[i]);
        channel[i] = out[i].getChannel();
        bb[i] = ByteBuffer.allocateDirect(BUFFERSIZE);
        isWritingAnyColumn = true;
    }

    List<List<String>> alternativeSamples = new ArrayList<>();
    final List<LongSet> sampledColumnValues = new ArrayList<>();
    for (int i = 0; i < columnFiles.length; i++) {
        sampledColumnValues.add(new LongOpenHashSet(this.sampleGoal < 0 ? 1000 : this.sampleGoal));
    }
    Long[] firstColumnValues = new Long[columnFiles.length];

    int rowCounter = 0;
    DebugCounter counter = new DebugCounter();
    while (input.hasNext()) {
        List<String> row = input.next();
        boolean isSampleCompleted = true;
        boolean rowHasUnseenValue = false;
        for (int i = 0; i < columnFiles.length; i++) {
            // Write the hash to the column.
            String str = row.get(i);
            long hash = getHash(str, i);
            if (isNew[i]) {
                if (bb[i].remaining() == 0) {
                    bb[i].flip();
                    channel[i].write(bb[i]);
                    bb[i].clear();
                }
                bb[i].putLong(hash);
            }

            // Keep track of the first column value and delete it if multiple values are observed.
            if (rowCounter == 0) firstColumnValues[i] = Long.valueOf(hash);
            else if (firstColumnValues[i] != null && firstColumnValues[i].longValue() != hash) firstColumnValues[i] = null;

            // Check if the value requests to put the row into the sample.
            if (hash != NULLHASH) {
                final LongSet sampledValues = sampledColumnValues.get(i);
                boolean shouldSample = this.sampleGoal < 0 || sampledValues.size() < this.sampleGoal;
                isSampleCompleted &= !shouldSample;
                if (shouldSample && sampledValues.add(hash)) {
                    rowHasUnseenValue = true;
                }
            }
        }

        if (rowHasUnseenValue) {
            alternativeSamples.add(row);
        }

        counter.countUp();
        rowCounter++;

        if (!isWritingAnyColumn && isSampleCompleted) break;
    }
    counter.done();
    writeSample(alternativeSamples);

    // Check for constant and null columns.
    for (int i = 0; i < firstColumnValues.length; i++) {
        this.isNullColumn[i] = firstColumnValues[i] != null && firstColumnValues[i].longValue() == NULLHASH;
        this.isConstantColumn[i] = firstColumnValues[i] != null && firstColumnValues[i].longValue() != NULLHASH;
    }

    for (int i = 0; i < columnFiles.length; i++) {
        if (!isNew[i]) continue;
        bb[i].flip();
        channel[i].write(bb[i]);
        out[i].close();
    }
}
 

@Override
protected void insertRowIntoAD(SimpleColumnCombination combination, long hash, LongSet longSet) {
    longSet.add(hash);
}
 

@VisibleForTesting
protected void seedLeftSideForFirstIteration() {
  long queryNode = salsaInternalState.getSalsaRequest().getQueryNode();
  salsaStats.setNumDirectNeighbors(
      salsaInternalState.getBipartiteGraph().getLeftNodeDegree(queryNode));

  Long2DoubleMap seedNodesWithWeight =
      salsaInternalState.getSalsaRequest().getLeftSeedNodesWithWeight();
  LongSet nonZeroSeedSet = salsaInternalState.getNonZeroSeedSet();

  double totalWeight = 0.0;
  for (Long2DoubleMap.Entry entry : seedNodesWithWeight.long2DoubleEntrySet()) {
    if (salsaInternalState.getBipartiteGraph().getLeftNodeDegree(entry.getLongKey())
        > 0) {
      totalWeight += entry.getDoubleValue();
      nonZeroSeedSet.add(entry.getLongKey());
    }
  }

  // If there is a pre-specified weight, we let it take precedence, but if not, then we reset
  // weights in accordance with the fraction of weight requested for the query node.
  if (!seedNodesWithWeight.containsKey(queryNode)
      && salsaInternalState.getBipartiteGraph().getLeftNodeDegree(queryNode) > 0) {
    double queryNodeWeight = 1.0;
    if (totalWeight > 0.0) {
      queryNodeWeight =
          totalWeight * salsaInternalState.getSalsaRequest().getQueryNodeWeightFraction()
              / (1.0 - salsaInternalState.getSalsaRequest().getQueryNodeWeightFraction());
    }
    seedNodesWithWeight.put(queryNode, queryNodeWeight);
    totalWeight += queryNodeWeight;
    nonZeroSeedSet.add(queryNode);
  }

  for (long leftNode : nonZeroSeedSet) {
    int numWalksToStart = (int) Math.ceil(
        seedNodesWithWeight.get(leftNode) / totalWeight
            * salsaInternalState.getSalsaRequest().getNumRandomWalks());
      salsaInternalState.getCurrentLeftNodes().put(leftNode, numWalksToStart);
  }

  salsaStats.setNumSeedNodes(salsaInternalState.getCurrentLeftNodes().size());
}