com.google.common.collect.ConcurrentHashMultiset Java Examples

@Override
public void end(boolean success) {
  if (watch.isRunning()) {
    watch.stop();
  }
  if (success) {
    OperationStats.this.successCounter.add(op);
    Multiset<Long> latency =
        OperationStats.this.latency.computeIfAbsent(
            op, o -> ConcurrentHashMultiset.create());

    long elapsed = watch.elapsed(TimeUnit.MILLISECONDS);
    latency.add(keyFromRange(elapsed));
  } else {
    OperationStats.this.failureCounter.add(op);
  }
}
 

/**
 * Log the top N action rewind events and clear the history of failed actions' lost inputs and
 * rewind plans.
 */
void reset(ExtendedEventHandler eventHandler) {
  ImmutableList<ActionRewindEvent> topActionRewindEvents =
      rewindPlansStats.stream()
          .collect(
              greatest(
                  MAX_ACTION_REWIND_EVENTS, comparing(RewindPlanStats::invalidatedNodesCount)))
          .stream()
          .map(ActionRewindingStats::toActionRewindEventProto)
          .collect(toImmutableList());
  ActionRewindingStats rewindingStats =
      new ActionRewindingStats(lostInputRecords.size(), topActionRewindEvents);
  eventHandler.post(rewindingStats);
  lostInputRecords = ConcurrentHashMultiset.create();
  rewindPlansStats = new ConcurrentLinkedQueue<>();
}
 

/**
 *
 * @param numClasses
 * @param samples sampled multi-labels; can have duplicates; their empirical probabilities will be estimated
 * @return
 */
public MultiLabel predict(int numClasses, List<MultiLabel> samples){
    Multiset<MultiLabel> multiset = ConcurrentHashMultiset.create();
    for (MultiLabel multiLabel: samples){
        multiset.add(multiLabel);
    }

    int sampleSize = samples.size();
    List<MultiLabel> uniqueOnes = new ArrayList<>();
    List<Double> probs = new ArrayList<>();
    for (Multiset.Entry<MultiLabel> entry: multiset.entrySet()){
        uniqueOnes.add(entry.getElement());
        probs.add((double)entry.getCount()/sampleSize);
    }
    return predict(numClasses,uniqueOnes,probs);
}
 

private static void add(List<String> source, Multiset<Ngram> multiset, String field, int slop, List<Integer> template){
    Multiset<Ngram> multiSetForDoc = ConcurrentHashMultiset.create();
    for (int i=0;i<source.size();i++){
        if(i+template.get(template.size()-1)<source.size()){
            List<String> list = new ArrayList<>();
            for (int j: template){
                list.add(source.get(i+j));
            }
            Ngram ngram = new Ngram();
            ngram.setNgram(Ngram.toNgramString(list));
            ngram.setSlop(slop);
            ngram.setField(field);
            ngram.setInOrder(true);
            multiSetForDoc.setCount(ngram,1);
        }
    }
     multiset.addAll(multiSetForDoc);
}
 

public static Multiset<Ngram> gatherNgram(ESIndex index, String[] ids, NgramTemplate template, int minDF){
    Multiset<Ngram> multiset = ConcurrentHashMultiset.create();
    String field = template.getField();
    Arrays.stream(ids).parallel().forEach(id -> {
        Map<Integer,String> termVector = index.getTermVectorFromIndex(field, id);
        add(termVector,multiset,template);
    });
    Multiset<Ngram> filtered = ConcurrentHashMultiset.create();
    for (Multiset.Entry entry: multiset.entrySet()){
        Ngram ngram = (Ngram)entry.getElement();
        int count = entry.getCount();
        if (count>=minDF){
            filtered.add(ngram,count);
        }
    }
    return filtered;
}
 

public static void main(String[] args) {
    // Разберем текст на слова
    String INPUT_TEXT = "Hello World! Hello All! Hi World!";
    // Создаем Multiset
    Multiset<String> multiset = ConcurrentHashMultiset.create(Arrays.asList(INPUT_TEXT.split(" ")));

    // Выводим кол-вом вхождений слов
    System.out.println(multiset); // напечатает [Hi, Hello x 2, World! x 2, All!] - в произвольном порядке
    // Выводим все уникальные слова
    System.out.println(multiset.elementSet());    // напечатает [Hi, Hello, World!, All!] - в произвольном порядке

    // Выводим количество по каждому слову
    System.out.println("Hello = " + multiset.count("Hello"));    // напечатает 2
    System.out.println("World = " + multiset.count("World!"));    // напечатает 2
    System.out.println("All = " + multiset.count("All!"));    // напечатает 1
    System.out.println("Hi = " + multiset.count("Hi"));    // напечатает 1
    System.out.println("Empty = " + multiset.count("Empty"));    // напечатает 0

    // Выводим общее количества всех слов в тексте
    System.out.println(multiset.size());    //напечатает 6

    // Выводим общее количество всех уникальных слов
    System.out.println(multiset.elementSet().size());    //напечатает 4
}
 

public static void main(String[] args) {
    // Parse text to separate words
    String INPUT_TEXT = "Hello World! Hello All! Hi World!";
    // Create Multiset
    Multiset<String> multiset = ConcurrentHashMultiset.create(Arrays.asList(INPUT_TEXT.split(" ")));

    // Print count words
    System.out.println(multiset); // print [Hi, Hello x 2, World! x 2, All!] - in random orders
    // Print all unique words
    System.out.println(multiset.elementSet());    // print [Hi, Hello, World!, All!] - in random orders

    // Print count occurrences of words
    System.out.println("Hello = " + multiset.count("Hello"));    // print 2
    System.out.println("World = " + multiset.count("World!"));    // print 2
    System.out.println("All = " + multiset.count("All!"));    // print 1
    System.out.println("Hi = " + multiset.count("Hi"));    // print 1
    System.out.println("Empty = " + multiset.count("Empty"));    // print 0

    // Print count all words
    System.out.println(multiset.size());    //print 6

    // Print count unique words
    System.out.println(multiset.elementSet().size());    //print 4
}
 

/**
 * Increment count for a given alignment for a given phrase-pair.
 */
public void incrementAlignmentCount(AlignmentTemplate alTemp) {
  if (storeAlignmentCounts) {
    int idx = alTemp.getKey();
    int alIdx = alTemp.getAKey();
    final ConcurrentHashMultiset<Integer> aCounts;
    if (idx >= 0) {
      assert (idx <= index.size());
      synchronized (aCounter) {
        // assert(idx <= aCounter.size());
        while (idx >= aCounter.size())
          aCounter.add(ConcurrentHashMultiset.create());
        aCounts = aCounter.get(idx);
      }
      aCounts.add(alIdx);
    }
  }
}
 

private int getArgmaxAlignment(int idx) {
  if (idx >= aCounter.size())
    return -1;
  // Linear search:
  ConcurrentHashMultiset<Integer> aCounts = aCounter.get(idx);
  int maxK = -1;
  int maxV = Integer.MIN_VALUE;
  String maxKLex = null;
  for (int k : aCounts.elementSet()) {
    int v = aCounts.count(k);
    if (v == maxV) {
      // If there is a tie, take lexicographic order as defined in Moses:
      String kLex = AlignmentTemplate.alignmentToString(aIndex.get(k));
      if (maxKLex == null)
        maxKLex = AlignmentTemplate.alignmentToString(aIndex.get(maxK));
      if (kLex.compareTo(maxKLex) < 0) {
        maxK = k;
        maxV = v;
        maxKLex = kLex;
      }
    } else if (v > maxV) {
      maxK = k;
      maxV = v;
      maxKLex = null;
    }
  }
  assert (maxK >= 0);
  return maxK;
}
 

private static void addCountToArray(ConcurrentHashMultiset<Integer> counter, int idx) {
  if (idx < 0)
    return;
  counter.add(idx);
  if (DEBUG_LEVEL >= 3)
    System.err.println("Increasing count idx=" + idx + " in vector (" + counter
        + ").");
}
 

private static void addCountToArray(final ConcurrentHashMultiset<Integer> counter, int idx) {
  if (idx < 0)
    return;
  counter.add(idx);
  if (DEBUG_LEVEL >= 3)
    System.err.println("Increasing count idx=" + idx + " in vector (" + counter
        + ").");
}
 

@Test
public void testOverflow() throws Throwable {
    when(pjp.getTarget()).thenReturn(this);
    Field field = findField(ConcurrentHashMultiset.class, "countMap");
    makeAccessible(field);
    ConcurrentMap map = (ConcurrentMap) getField(field, jobs.getStatCounter());
    map.put("java.lang.String", Integer.MAX_VALUE);
    testExecute();
}
 

public static Multiset<Ngram> gatherNgram(ESIndex index, String[] ids, NgramTemplate template){
    Multiset<Ngram> multiset = ConcurrentHashMultiset.create();
    String field = template.getField();
    Arrays.stream(ids).parallel().forEach(id -> {
        Map<Integer,String> termVector = index.getTermVectorFromIndex(field, id);
        add(termVector,multiset,template);
    });
    return multiset;
}
 

@Test
public void testResetAfterLimit() throws InterruptedException {
	rateLimiter = new  LoggingRateLimiter(0);
	ConcurrentHashMultiset freq = rateLimiter.getFrequency();
	
	rateLimiter.resetAfterLimit();
	
	assertTrue (freq != rateLimiter.getFrequency() );
	freq = rateLimiter.getFrequency();
	Thread.sleep(1);
	
	rateLimiter.resetAfterLimit();
	assertThat (freq,is( rateLimiter.getFrequency()));
}
 

private static void test8(){
    NgramTemplate template = new NgramTemplate("body",3,1);
    Multiset<Ngram> multiset = ConcurrentHashMultiset.create();
    List<String> source = new ArrayList<>();
    for (int i=0;i<10;i++){
        source.add(""+i);
    }
    NgramEnumerator.add(source,multiset,template);
    System.out.println(multiset.elementSet().stream().map(Ngram::getNgram).collect(Collectors.toList()));
}
 

public void logResult(String operation, String result) {
  if (running) {
    Multiset<String> multiset =
        opWithResult.computeIfAbsent(operation, op -> ConcurrentHashMultiset.create());
    multiset.add(result);
  }
}
 

static Set<Ngram> gather(Config config, ESIndex index,
                             String[] ids, Logger logger) throws Exception{

        File metaDataFolder = new File(config.getString("output.folder"),"meta_data");
        metaDataFolder.mkdirs();

        Multiset<Ngram> allNgrams = ConcurrentHashMultiset.create();
        List<Integer> ns = config.getIntegers("train.feature.ngram.n");
        double minDf = config.getDouble("train.feature.ngram.minDf");
        int minDFrequency = (int)Math.floor(ids.length*minDf);
        List<String> fields = config.getStrings("train.feature.ngram.extractionFields");
        List<Integer> slops = config.getIntegers("train.feature.ngram.slop");
        boolean inorder = config.getBoolean("train.feature.ngram.inOrder");
        boolean allowDuplicates = config.getBoolean("train.feature.ngram.allowDuplicateWords");
        for (String field: fields){
            for (int n: ns){
                for (int slop:slops){
                    logger.info("gathering "+n+ "-grams from field "+field+" with slop "+slop+" and minDf "+minDf+ ", (actual frequency threshold = "+minDFrequency+")");
                    NgramTemplate template = new NgramTemplate(field,n,slop);
                    Multiset<Ngram> ngrams = NgramEnumerator.gatherNgram(index, ids, template, minDFrequency);
                    logger.info("gathered "+ngrams.elementSet().size()+ " ngrams");
                    int newCounter = 0;
                    for (Multiset.Entry<Ngram> entry: ngrams.entrySet()){
                        Ngram ngram = entry.getElement();
                        ngram.setInOrder(inorder);
                        int count = entry.getCount();
                        if (interesting(allNgrams,ngram,count)){
                            if (allowDuplicates) {
                                allNgrams.add(ngram, count);
                                newCounter += 1;
                            }else{
                                if (!ngram.hasDuplicate()){
                                    allNgrams.add(ngram, count);
                                    newCounter += 1;
                                }
                            }
                        }

                    }
                    logger.info(newCounter+" are really new");
                }
            }
        }
        logger.info("there are "+allNgrams.elementSet().size()+" ngrams in total");
//        BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(new File(metaDataFolder,"all_ngrams.txt")));
//        for (Multiset.Entry<Ngram> ngramEntry: allNgrams.entrySet()){
//            bufferedWriter.write(ngramEntry.getElement().toString());
//            bufferedWriter.write("\t");
//            bufferedWriter.write(""+ngramEntry.getCount());
//            bufferedWriter.newLine();
//        }
//
//        bufferedWriter.close();
//
//        //for serialization
//        Set<Ngram> uniques = new HashSet<>();
//        uniques.addAll(allNgrams.elementSet());
//        Serialization.serialize(uniques, new File(metaDataFolder, "all_ngrams.ser"));
        return allNgrams.elementSet();
    }
 

@Test
public void testDoesNotReset(){
	ConcurrentHashMultiset freq = rateLimiter.getFrequency();
	rateLimiter.resetAfterLimit();
	assertThat (freq,is( rateLimiter.getFrequency()));
}
 

/**
 * Attempts to add a single instance of the given value to the given
 * multiset without exceeding the specified maximum number of values. If
 * the value cannot be added without exceeding the maximum, false is
 * returned.
 *
 * @param <T>
 *     The type of values contained within the multiset.
 *
 * @param multiset
 *     The multiset to attempt to add a value to.
 *
 * @param value
 *     The value to attempt to add.
 *
 * @param max
 *     The maximum number of each distinct value that the given multiset
 *     should hold, or zero if no limit applies.
 *
 * @return
 *     true if the value was successfully added without exceeding the
 *     specified maximum, false if the value could not be added.
 */
private <T> boolean tryAdd(ConcurrentHashMultiset<T> multiset, T value, int max) {

    // Repeatedly attempt to add a new value to the given multiset until we
    // explicitly succeed or explicitly fail
    while (true) {

        // Get current number of values
        int count = multiset.count(value);

        // Bail out if the maximum has already been reached
        if (count >= max && max != 0)
            return false;

        // Attempt to add one more value
        if (multiset.setCount(value, count, count+1))
            return true;

        // Try again if unsuccessful

    }

}
 

public ClassMemberUseCounter(ConcurrentHashMultiset<ClassMemberUse<?, ?, ?>> memberUseCounter) {
  this.memberUseCounter = memberUseCounter;
}
 

public <E extends TypeMappable<? extends E>> ConcurrentHashMultiset<E> map(
    ConcurrentHashMultiset<E> mappableTypes) {
  return mappableTypes.stream()
      .map(e -> e.acceptTypeMapper(this))
      .collect(toCollection(ConcurrentHashMultiset::create));
}
 

@Test
public void testConcurrentRace_AllSameSizedCombinations() throws Exception {
  // When we have n values
  int n = 10;
  ImmutableSet.Builder<String> valsBuilder = ImmutableSet.builder();
  for (int i = 0; i < n; i++) {
    valsBuilder.add("val-" + i);
  }
  ImmutableSet<String> vals = valsBuilder.build();
  int k = 5;
  // And we have all combinations of size k of these n values
  Set<Set<String>> combinations = Sets.combinations(vals, k);
  int numCombinations = combinations.size();
  // And we have a MultisetSemaphore
  final MultisetSemaphore<String> multisetSemaphore = MultisetSemaphore.newBuilder()
      // with K max num unique values,
      .maxNumUniqueValues(k)
      .build();
  // And a ExecutorService with nCk threads,
  ExecutorService executorService = Executors.newFixedThreadPool(numCombinations);
  // And a recorder for thrown exceptions,
  ThrowableRecordingRunnableWrapper wrapper =
      new ThrowableRecordingRunnableWrapper("testConcurrentRace_AllSameSizedCombinations");
  // And a ConcurrentHashMultiset for counting the multiplicities of the values ourselves,
  ConcurrentHashMultiset<String> counts = ConcurrentHashMultiset.create();
  for (Set<String> combination : combinations) {
    // And, for each of the nCk combinations, we submit a Runnable, that
    @SuppressWarnings("unused")
    Future<?> possiblyIgnoredError =
        executorService.submit(
            wrapper.wrap(
                new Runnable() {
                  @Override
                  public void run() {
                    try {
                      // Tries to acquire permits for its set of k values,
                      multisetSemaphore.acquireAll(combination);
                      // And then verifies that the multiplicities are as expected,
                      combination.forEach(counts::add);
                      assertThat(counts.entrySet().size()).isAtMost(k);
                      combination.forEach(counts::remove);
                      // And then releases the permits.
                      multisetSemaphore.releaseAll(combination);
                    } catch (InterruptedException e) {
                      throw new IllegalStateException(e);
                    }
                  }
                }));
  }
  // Then all of our Runnables completed (without deadlock!), as expected,
  boolean interrupted = ExecutorUtil.interruptibleShutdown(executorService);
  // And also none of them threw any Exceptions.
  assertThat(wrapper.getFirstThrownError()).isNull();
  if (interrupted) {
    Thread.currentThread().interrupt();
    throw new InterruptedException();
  }
}
 

/**
 * Attempts to add a single instance of the given value to the given
 * multiset without exceeding the specified maximum number of values. If
 * the value cannot be added without exceeding the maximum, false is
 * returned.
 *
 * @param <T>
 *     The type of values contained within the multiset.
 *
 * @param multiset
 *     The multiset to attempt to add a value to.
 *
 * @param value
 *     The value to attempt to add.
 *
 * @param max
 *     The maximum number of each distinct value that the given multiset
 *     should hold, or zero if no limit applies.
 *
 * @return
 *     true if the value was successfully added without exceeding the
 *     specified maximum, false if the value could not be added.
 */
private <T> boolean tryAdd(ConcurrentHashMultiset<T> multiset, T value, int max) {

    // Repeatedly attempt to add a new value to the given multiset until we
    // explicitly succeed or explicitly fail
    while (true) {

        // Get current number of values
        int count = multiset.count(value);

        // Bail out if the maximum has already been reached
        if (count >= max && max != 0)
            return false;

        // Attempt to add one more value
        if (multiset.setCount(value, count, count+1))
            return true;

        // Try again if unsuccessful

    }

}