Java Code Examples for org.apache.commons.math3.stat.descriptive.moment.Mean#increment()

/**
 * Summarize the phread scores of all the bases piles in the given pileup.
 * This calculates the mean quality score.
 * For bases that disagree, this adds in the error rate.
 * So, if you had AAT with qualities 30,30,10:
 * Mean of : 0.999,0.999,0.1=0.6993333
 * Thus an error rate of 1-0.6993333=0.3006667, which is a phread base score of ~ 5.
 * We want to penalize bases with disagreements.
 * @return
 */
public int getSummarizedPhreadScoreByMeanWithErrors () {
	byte commonBase = getMostCommonBase();

	Mean mean = new Mean();

	for (int i=0; i<this.bases.size(); i++) {
		byte base = this.bases.get(i);
		byte qual = this.qualities.get(i);
		double prob = LikelihoodUtils.getInstance().phredScoreToErrorProbability(qual);

		if (base==commonBase)
			mean.increment(prob);
		else
			mean.increment(1-prob);
	}
	double meanErrorProbability = mean.getResult();
	int phread = LikelihoodUtils.getInstance().errorProbabilityToPhredScore(meanErrorProbability);
	return phread;
}
 

private void setMeanAndStd(){
	Mean mu = new Mean();
	StandardDeviation dev = new StandardDeviation();
	for (String chr : bedgraph.keySet()){
		
		ArrayList<TagNode> inTemp = bedgraph.get(chr);
		Collections.sort(inTemp,  TagNode.basepairComparator);
		for (int i = 0; i < inTemp.size();i++){
			int length = inTemp.get(i).getLength();
			double value = inTemp.get(i).getScore2();
			for (int a = 0; a < length;a++){
				mu.increment(value);
				dev.increment(value);
			}
		}
	}
	mean = mu.getResult();
	std = dev.getResult();
}
 

/**
 * Set the data across a specific region
 * @param node a TagNode representing a specific region for calculation
 * @throws IOException
 */
private void SetMeanAndStd2(TagNode node) throws IOException{
	BBFileReader wigReader = new BBFileReader(wigFile);
	String chrom = node.getChrom();
	int begin = node.getStart();
	int end = node.getStop();
	BigWigIterator iter = wigReader.getBigWigIterator(chrom, begin, chrom, end, false);
	Mean mu = new Mean();
	StandardDeviation dev = new StandardDeviation();
	while(iter.hasNext()){
		WigItem item = iter.next();
		int start = item.getStartBase();
		int stop = item.getEndBase();
		double value = item.getWigValue();
		for (int i = start; i < stop;i++){
			mu.increment(value);
			dev.increment(value);
		}
		
	}
	mean = mu.getResult();
	std = dev.getResult();
	wigReader=null;
}
 

/**
 * Set the data across the entire genome
 * @throws IOException
 */
private void SetMeanAndStd() throws IOException{
	BBFileReader wigReader = new BBFileReader(wigFile);
	BigWigIterator iter = wigReader.getBigWigIterator();
	Mean mu = new Mean();
	StandardDeviation dev = new StandardDeviation();
	while(iter.hasNext()){
		WigItem item = iter.next();
		int start = item.getStartBase();
		int stop = item.getEndBase();
		double value = item.getWigValue();
		for (int i = start; i < stop;i++){
			mu.increment(value);
			dev.increment(value);
		}
		
	}
	mean = mu.getResult();
	std = dev.getResult();
}
 

/**
 * Find the max and the max position
 * @throws IOException
 */
private void find() throws IOException{
	BBFileReader wigReader = new BBFileReader(wig);
	BigWigIterator iter = wigReader.getBigWigIterator(chr,start,chr,stop,false);
	Max m = new Max();
	Mean mu = new Mean();
	ArrayList<Double> values = new ArrayList<Double>();
	double tempMax=0;
	node  = null;
	while (iter.hasNext()){
		WigItem item = iter.next();
		double value = item.getWigValue();
		m.increment(value);
		for (int i = item.getStartBase();i < item.getEndBase();i++){
			values.add(value);
			mu.increment(value);
			if (value > tempMax){
				tempMax = value;
				node = new TagNode(chr,i,i+1);
			}
		}
		
	}
	
	val = values;
	if (values.size()>0){
		score = mu.getResult();
		max = m.getResult();
		median = values.get(values.size()/2);
	}
	else{
		score=0;max=0;median=0;
	}
	wigReader = null;
}
 

public static ScoreNode set(TagNode tag,ArrayList<TagNode> overlaps){
	
	
		Max m = new Max();
		Mean mu = new Mean();
		ArrayList<Double> values = new ArrayList<Double>();
		Collections.sort(overlaps, TagNode.basepairComparator);
		for (int a = 0; a < overlaps.size(); a++) {
			TagNode node2 = overlaps.get(a);

			double value = node2.getScore2();
			m.increment(value);
			for (int i = node2.getStart(); i < node2.getStop(); i++) {
				if (i >= tag.getStart() && i < tag.getStop()) {
					values.add(value);
					mu.increment(value);

				}
			}

		}
		double ave;
		double median;
		double max;
		if (values.size() > 0) {
			ave = mu.getResult();
			max = m.getResult();
			median = values.get(values.size() / 2);
		} else {
			ave = 0;
			max = 0;
			median = 0;
		}
		ScoreNode output = new ScoreNode(ave, median, max, values);
		return output;
	
}
 

public Double evaluate(Collection<?> values){
	Mean statistic = new Mean();

	for(Object value : values){
		Number number = (Number)TypeUtil.parseOrCast(DataType.DOUBLE, value);

		statistic.increment(number.doubleValue());
	}

	return statistic.getResult();
}
 

public EvaluationResult evaluate(File originalDataDir) throws TasteException, IOException, InterruptedException {

    Preconditions.checkArgument(originalDataDir.exists() && originalDataDir.isDirectory(),
                                "%s is not a directory", originalDataDir);
    File tempDir = Files.createTempDir();

    ServerRecommender recommender = null;
    try {

      Multimap<Long,RecommendedItem> data;
      try {
        data = readAndCopyDataFiles(originalDataDir, tempDir);
      } catch (IOException ioe) {
        throw new TasteException(ioe);
      }

      recommender = new ServerRecommender(tempDir);
      recommender.await();

      Generation generation = recommender.getGenerationManager().getCurrentGeneration();
      FastByIDMap<float[]> X = generation.getX();
      FastByIDMap<float[]> Y = generation.getY();

      Mean averageError = new Mean();
      // Only compute average over existing entries...
      for (Map.Entry<Long,RecommendedItem> entry : data.entries()) {
        long userID = entry.getKey();
        long itemID = entry.getValue().getItemID();
        // Each of which was a "1" in the factor P matrix
        double value = SimpleVectorMath.dot(X.get(userID), Y.get(itemID));
        // So store abs(1-value), except, don't penalize for reconstructing > 1. Error is 0 in this case.
        averageError.increment(FastMath.max(0.0, 1.0 - value));
      }

      return new EvaluationResultImpl(averageError.getResult());
    } finally {
      recommender.close();
      IOUtils.deleteRecursively(tempDir);
    }
  }
 

private void Generate(){
	SAMFileReader reader = new SAMFileReader(bam,index);
	Mean mu = new Mean();
	StandardDeviation std = new StandardDeviation();
	for (int i = 0; i < genome.size();i++){
		String chr = genome.get(i).getChrom();
		int start = genome.get(i).getStart();
		int stop = genome.get(i).getStop();
		ArrayList<PileupNode2> temp = new ArrayList<PileupNode2>();
		for (int a = start;a < stop;a++){
			//new
			int begin;
			int end;
			if (a >= 1000){
				begin = a -1000;
			}else{
				begin = 0;
			}
			if (a <= stop - 1000){
				end = a + 1000;
			}else{
				end = stop;
			}
			//above is new
			CloseableIterator<SAMRecord> iter = reader.query(chr, begin, end, false);
			double value = 0;
			while(iter.hasNext()){
				SAMRecord record = iter.next();
				if(!record.getReadUnmappedFlag() && !record.getMateUnmappedFlag() && record.getFirstOfPairFlag()) {
					int tagstart = -1;
					int tagstop = -1;
				
					if(record.getInferredInsertSize() > 0) {
						tagstart = record.getAlignmentStart();
						
						tagstop = record.getAlignmentStart() + record.getInferredInsertSize() - 1;
					}else if (record.getInferredInsertSize() < 0 ) {
						tagstart = record.getAlignmentEnd() + record.getInferredInsertSize() + 1;
						tagstop = record.getAlignmentEnd();
					}
					if (tagstart == a || tagstop == a || (tagstart < a && tagstop > a)){
						value+=1;
					}
				}
				
			}
			iter.close();
			PileupNode2 node = new PileupNode2(a,value,chr);
			temp.add(node);
			mu.increment(value);
			std.increment(value);
		}
		PileupToBedGraph convert = new PileupToBedGraph(temp,1);
		bedgraph.addAll(convert.getBedGraph());
		
	}
	reader.close();
	mean = mu.getResult();
	stddev = std.getResult();
}
 

@Test
public void testLSH() {
  System.setProperty("model.lsh.sampleRatio", "0.1");
  System.setProperty("model.lsh.numHashes", "20");
  RandomGenerator random = RandomManager.getRandom();

  Mean avgPercentTopRecsConsidered = new Mean();
  Mean avgNDCG = new Mean();
  Mean avgPercentAllItemsConsidered= new Mean();

  for (int iteration = 0; iteration < ITERATIONS; iteration++) {

    FastByIDMap<float[]> Y = new FastByIDMap<float[]>();
    for (int i = 0; i < NUM_ITEMS; i++) {
      Y.put(i, RandomUtils.randomUnitVector(NUM_FEATURES, random));
    }
    float[] userVec = RandomUtils.randomUnitVector(NUM_FEATURES, random);

    double[] results = doTestRandomVecs(Y, userVec);
    double percentTopRecsConsidered = results[0];
    double ndcg = results[1];
    double percentAllItemsConsidered = results[2];

    log.info("Considered {}% of all candidates, {} nDCG, got {}% recommendations correct",
             100 * percentAllItemsConsidered,
             ndcg,
             100 * percentTopRecsConsidered);

    avgPercentTopRecsConsidered.increment(percentTopRecsConsidered);
    avgNDCG.increment(ndcg);
    avgPercentAllItemsConsidered.increment(percentAllItemsConsidered);
  }

  log.info("{}", avgPercentTopRecsConsidered.getResult());
  log.info("{}", avgNDCG.getResult());
  log.info("{}", avgPercentAllItemsConsidered.getResult());

  assertTrue(avgPercentTopRecsConsidered.getResult() > 0.55);
  assertTrue(avgNDCG.getResult() > 0.55);
  assertTrue(avgPercentAllItemsConsidered.getResult() < 0.075);
}
 

private static HashMap<String, MeanSd> calculatePathayMeansOfAnnotatedGenes(DoubleMatrixDataset<String, String> predictionMatrixSignificant, DoubleMatrixDataset<String, String> annotationMatrixSignificant) {

		HashMap<String, MeanSd> pathwayMeanSdMap = new HashMap<>(predictionMatrixSignificant.columns());

		Mean meanCalculator = new Mean();
		Variance varianceCalculator = new Variance();

		for (String pathway : predictionMatrixSignificant.getColObjects()) {

			meanCalculator.clear();
			varianceCalculator.clear();

			DoubleMatrix1D pathwayPredictions = predictionMatrixSignificant.getCol(pathway);
			DoubleMatrix1D pathwayAnnotations = annotationMatrixSignificant.getCol(pathway);

			for (int g = 0; g < pathwayPredictions.size(); ++g) {
				if (pathwayAnnotations.get(g) != 0) {
					meanCalculator.increment(pathwayPredictions.getQuick(g));
					varianceCalculator.increment(pathwayPredictions.getQuick(g));
				}
			}

			double v = varianceCalculator.getResult();

			pathwayMeanSdMap.put(pathway, new MeanSd(meanCalculator.getResult(), v * v));

		}

		return pathwayMeanSdMap;

	}