org.apache.commons.lang3.mutable.MutableLong Java Examples

The following examples show how to use org.apache.commons.lang3.mutable.MutableLong. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FileDataInterface.java    From count-db with MIT License 6 votes vote down vote up
@Override
public long freeMemory() {
    MutableLong totalBytesReleased = new MutableLong(0);
    ifNotClosed(() -> {
        for (FileBucket bucket : fileBuckets) {
            bucket.lockRead();
            for (FileInfo fileInfo : bucket.getFiles()) {
                long bytesReleased = fileInfo.discardFileContents();
                updateSizeOfCachedFileContents(-bytesReleased);
                totalBytesReleased.add(bytesReleased);
            }
            bucket.unlockRead();
        }
    });
    return totalBytesReleased.longValue();
}
 
Example #2
Source File: SessionComponentRegressionTest.java    From sakai with Educational Community License v2.0 6 votes vote down vote up
protected Session newSessionWithBlockableGetLastAccessedTimeImpl(final CountDownLatch opStarted,
		final CountDownLatch opBlocker, final CountDownLatch opCompleted) {
	// unfortunately, the getActiveUserCount() implementation compels us to
	// use MySession rather than an interface.
	String uuid = nextUuid();
	final MySession session = new MySession(sessionComponent,uuid,threadLocalManager,idManager,
			sessionComponent,sessionListener,sessionComponent.getInactiveInterval(),new MyNonPortableSession(),
			new MutableLong(System.currentTimeMillis()), null) {
		private long superGetLastAccessedTime() {
			return super.getLastAccessedTime();
		}
		@Override
		public long getLastAccessedTime()
		{
			Callable<Long> callback = new Callable<Long>() {
				public Long call() throws Exception {
					return superGetLastAccessedTime();
				}
			};
			Long result = 
				execBlockableSessionOp(opStarted, opBlocker, opCompleted, callback);
			return result;
		}
	};
	return session;
}
 
Example #3
Source File: WEQueryQueueManagerTest.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
@Test
public void testSimpleRemoveEmpty()
{
  WindowEndQueueManager<Query, Void> wqqm = new WindowEndQueueManager<>();

  wqqm.setup(null);
  wqqm.beginWindow(0);

  QueryBundle<Query, Void, MutableLong> qb = wqqm.dequeue();
  Query queryD = qb == null ? null : qb.getQuery();
  Assert.assertEquals("The queries must match.", null, queryD);

  qb = wqqm.dequeue();
  queryD = qb == null ? null : qb.getQuery();
  Assert.assertEquals("The queries must match.", null, queryD);

  wqqm.endWindow();
  wqqm.teardown();
}
 
Example #4
Source File: WEQueryQueueManagerTest.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
@Test
public void testSimpleAddOneRemove()
{
  WindowEndQueueManager<Query, Void> wqqm = new WindowEndQueueManager<>();

  wqqm.setup(null);
  wqqm.beginWindow(0);

  Query query = new MockQuery("1");
  wqqm.enqueue(query, null, new MutableLong(1L));

  Query queryD = wqqm.dequeue().getQuery();
  QueryBundle<Query, Void, MutableLong> qb = wqqm.dequeue();
  Query queryD1 = qb == null ? null : qb.getQuery();

  wqqm.endWindow();
  wqqm.teardown();

  Assert.assertEquals("The queries must match.", query, queryD);
  Assert.assertEquals("The queries must match.", null, queryD1);
}
 
Example #5
Source File: SerialReplicationChecker.java    From hbase with Apache License 2.0 6 votes vote down vote up
public boolean canPush(Entry entry, Cell firstCellInEdit) throws IOException {
  String encodedNameAsString = Bytes.toString(entry.getKey().getEncodedRegionName());
  long seqId = entry.getKey().getSequenceId();
  Long canReplicateUnderSeqId = canPushUnder.getIfPresent(encodedNameAsString);
  if (canReplicateUnderSeqId != null) {
    if (seqId < canReplicateUnderSeqId.longValue()) {
      LOG.trace("{} is before the end barrier {}, pass", entry, canReplicateUnderSeqId);
      return true;
    }
    LOG.debug("{} is beyond the previous end barrier {}, remove from cache", entry,
      canReplicateUnderSeqId);
    // we are already beyond the last safe point, remove
    canPushUnder.invalidate(encodedNameAsString);
  }
  // This is for the case where the region is currently opened on us, if the sequence id is
  // continuous then we are safe to replicate. If there is a breakpoint, then maybe the region
  // has been moved to another RS and then back, so we need to check the barrier.
  MutableLong previousPushedSeqId = pushed.getUnchecked(encodedNameAsString);
  if (seqId == previousPushedSeqId.longValue() + 1) {
    LOG.trace("The sequence id for {} is continuous, pass", entry);
    previousPushedSeqId.increment();
    return true;
  }
  return canPush(entry, CellUtil.cloneRow(firstCellInEdit));
}
 
Example #6
Source File: ApexWindowedStreamImpl.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
@Override
public <STREAM extends WindowedStream<Tuple.WindowedTuple<Long>>> STREAM count(Option... opts)
{
  Function.MapFunction<T, Tuple<Long>> kVMap = new Function.MapFunction<T, Tuple<Long>>()
  {
    @Override
    public Tuple<Long> f(T input)
    {
      if (input instanceof Tuple.TimestampedTuple) {
        return new Tuple.TimestampedTuple<>(((Tuple.TimestampedTuple)input).getTimestamp(), 1L);
      } else {
        return new Tuple.TimestampedTuple<>(System.currentTimeMillis(), 1L);
      }
    }
  };

  WindowedStream<Tuple<Long>> innerstream = map(kVMap);
  WindowedOperatorImpl<Long, MutableLong, Long> windowedOperator = createWindowedOperator(new SumLong());
  return innerstream.addOperator(windowedOperator, windowedOperator.input, windowedOperator.output, opts);
}
 
Example #7
Source File: Application.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
@Override
public void populateDAG(DAG dag, Configuration configuration)
{
  RandomNumberPairGenerator inputOperator = new RandomNumberPairGenerator();
  WindowedOperatorImpl<MutablePair<Double, Double>, MutablePair<MutableLong, MutableLong>, Double> windowedOperator = new WindowedOperatorImpl<>();
  Accumulation<MutablePair<Double, Double>, MutablePair<MutableLong, MutableLong>, Double> piAccumulation = new PiAccumulation();

  windowedOperator.setAccumulation(piAccumulation);
  windowedOperator.setDataStorage(new InMemoryWindowedStorage<MutablePair<MutableLong, MutableLong>>());
  windowedOperator.setWindowStateStorage(new InMemoryWindowedStorage<WindowState>());
  windowedOperator.setWindowOption(new WindowOption.GlobalWindow());
  windowedOperator.setTriggerOption(TriggerOption.AtWatermark().withEarlyFiringsAtEvery(Duration.millis(1000)).accumulatingFiredPanes());

  ConsoleOutputOperator outputOperator = new ConsoleOutputOperator();
  dag.addOperator("inputOperator", inputOperator);
  dag.addOperator("windowedOperator", windowedOperator);
  dag.addOperator("outputOperator", outputOperator);
  dag.addStream("input_windowed", inputOperator.output, windowedOperator.input);
  dag.addStream("windowed_output", windowedOperator.output, outputOperator.input);
}
 
Example #8
Source File: SumTest.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
@Test
public void SumTest()
{
  SumInt si = new SumInt();
  SumLong sl = new SumLong();
  SumFloat sf = new SumFloat();
  SumDouble sd = new SumDouble();

  Assert.assertEquals(new MutableInt(10), si.accumulate(si.defaultAccumulatedValue(), 10));
  Assert.assertEquals(new MutableInt(11), si.accumulate(new MutableInt(1), 10));
  Assert.assertEquals(new MutableInt(22), si.merge(new MutableInt(1), new MutableInt(21)));

  Assert.assertEquals(new MutableLong(10L), sl.accumulate(sl.defaultAccumulatedValue(), 10L));
  Assert.assertEquals(new MutableLong(22L), sl.accumulate(new MutableLong(2L), 20L));
  Assert.assertEquals(new MutableLong(41L), sl.merge(new MutableLong(32L), new MutableLong(9L)));

  Assert.assertEquals(new MutableFloat(9.0F), sf.accumulate(sf.defaultAccumulatedValue(), 9.0F));
  Assert.assertEquals(new MutableFloat(22.5F), sf.accumulate(new MutableFloat(2.5F), 20F));
  Assert.assertEquals(new MutableFloat(41.0F), sf.merge(new MutableFloat(33.1F), new MutableFloat(7.9F)));

  Assert.assertEquals(new MutableDouble(9.0), sd.accumulate(sd.defaultAccumulatedValue(), 9.0));
  Assert.assertEquals(new MutableDouble(22.5), sd.accumulate(new MutableDouble(2.5), 20.0));
  Assert.assertEquals(new MutableDouble(41.0), sd.merge(new MutableDouble(33.1), new MutableDouble(7.9)));
}
 
Example #9
Source File: Application.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
@Override
public void populateDAG(DAG dag, Configuration configuration)
{
  WordGenerator inputOperator = new WordGenerator();
  KeyedWindowedOperatorImpl<String, Long, MutableLong, Long> windowedOperator = new KeyedWindowedOperatorImpl<>();
  Accumulation<Long, MutableLong, Long> sum = new SumAccumulation();

  windowedOperator.setAccumulation(sum);
  windowedOperator.setDataStorage(new InMemoryWindowedKeyedStorage<String, MutableLong>());
  windowedOperator.setRetractionStorage(new InMemoryWindowedKeyedStorage<String, Long>());
  windowedOperator.setWindowStateStorage(new InMemoryWindowedStorage<WindowState>());
  windowedOperator.setWindowOption(new WindowOption.TimeWindows(Duration.standardMinutes(1)));
  windowedOperator.setTriggerOption(TriggerOption.AtWatermark().withEarlyFiringsAtEvery(Duration.millis(1000)).accumulatingAndRetractingFiredPanes());
  //windowedOperator.setAllowedLateness(Duration.millis(14000));

  ConsoleOutputOperator outputOperator = new ConsoleOutputOperator();
  dag.addOperator("inputOperator", inputOperator);
  dag.addOperator("windowedOperator", windowedOperator);
  dag.addOperator("outputOperator", outputOperator);
  dag.addStream("input_windowed", inputOperator.output, windowedOperator.input);
  dag.addStream("windowed_output", windowedOperator.output, outputOperator.input);
}
 
Example #10
Source File: QueryManagerAsynchronousTest.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
@Override
public void run()
{
  int numLoops = totalTuples / batchSize;

  for (int loopCounter = 0, tupleCounter = 0; loopCounter < numLoops; loopCounter++, tupleCounter++) {
    for (int batchCounter = 0; batchCounter < batchSize; batchCounter++, tupleCounter++) {
      queueManager.enqueue(new MockQuery(tupleCounter + ""), null, new MutableLong(1L));

      if (rand.nextDouble() < waitMillisProb) {
        try {
          Thread.sleep(1);
        } catch (InterruptedException ex) {
          throw new RuntimeException(ex);
        }
      }
    }
  }
}
 
Example #11
Source File: WindowedOperatorTest.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
@Test
public void testSlidingWindowAssignment()
{
  WindowedOperatorImpl<Long, MutableLong, Long> windowedOperator = createDefaultWindowedOperator();
  windowedOperator.setWindowOption(new WindowOption.SlidingTimeWindows(Duration.millis(1000), Duration.millis(200)));
  windowedOperator.setup(testMeta.operatorContext);
  Tuple.WindowedTuple<Long> windowedValue = windowedOperator.getWindowedValue(new Tuple.TimestampedTuple<>(BASE + 1600L, 2L));
  Collection<? extends Window> windows = windowedValue.getWindows();
  Window[] winArray = windows.toArray(new Window[]{});
  Assert.assertEquals(5, winArray.length);
  Assert.assertEquals(BASE + 800, winArray[0].getBeginTimestamp());
  Assert.assertEquals(1000, winArray[0].getDurationMillis());
  Assert.assertEquals(BASE + 1000, winArray[1].getBeginTimestamp());
  Assert.assertEquals(1000, winArray[1].getDurationMillis());
  Assert.assertEquals(BASE + 1200, winArray[2].getBeginTimestamp());
  Assert.assertEquals(1000, winArray[2].getDurationMillis());
  Assert.assertEquals(BASE + 1400, winArray[3].getBeginTimestamp());
  Assert.assertEquals(1000, winArray[3].getDurationMillis());
  Assert.assertEquals(BASE + 1600, winArray[4].getBeginTimestamp());
  Assert.assertEquals(1000, winArray[4].getDurationMillis());
  windowedOperator.teardown();
}
 
Example #12
Source File: WindowedOperatorTest.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
private WindowedOperatorImpl<Long, MutableLong, Long> createDefaultWindowedOperator()
{
  WindowedOperatorImpl<Long, MutableLong, Long> windowedOperator = new WindowedOperatorImpl<>();
  if (useSpillable) {
    sccImpl = new SpillableComplexComponentImpl(testMeta.timeStore);
    // TODO: We don't yet support Spillable data structures for window state storage because SpillableMapImpl does not yet support iterating over all keys.
    windowStateStorage = new InMemoryWindowedStorage<>();
    SpillableWindowedPlainStorage<MutableLong> pds = new SpillableWindowedPlainStorage<>();
    pds.setSpillableComplexComponent(sccImpl);
    plainDataStorage = pds;
    SpillableWindowedPlainStorage<Long> prs = new SpillableWindowedPlainStorage<>();
    prs.setSpillableComplexComponent(sccImpl);
    plainRetractionStorage = prs;
    windowedOperator.addComponent("SpillableComplexComponent", sccImpl);
  } else {
    windowStateStorage = new InMemoryWindowedStorage<>();
    plainDataStorage = new InMemoryWindowedStorage<>();
    plainRetractionStorage = new InMemoryWindowedStorage<>();
  }
  windowedOperator.setDataStorage(plainDataStorage);
  windowedOperator.setRetractionStorage(plainRetractionStorage);
  windowedOperator.setWindowStateStorage(windowStateStorage);
  windowedOperator.setAccumulation(new SumAccumulation());
  return windowedOperator;
}
 
Example #13
Source File: WindowedOperatorTest.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
@Test
public void testValidation() throws Exception
{
  WindowedOperatorImpl<Long, MutableLong, Long> windowedOperator = new WindowedOperatorImpl<>();
  verifyValidationFailure(windowedOperator, "nothing is configured");
  windowedOperator.setWindowStateStorage(new InMemoryWindowedStorage<WindowState>());
  verifyValidationFailure(windowedOperator, "data storage is not set");
  windowedOperator.setDataStorage(new InMemoryWindowedStorage<MutableLong>());
  verifyValidationFailure(windowedOperator, "accumulation is not set");
  windowedOperator.setAccumulation(new SumAccumulation());
  windowedOperator.validate();
  windowedOperator.setTriggerOption(new TriggerOption().accumulatingAndRetractingFiredPanes());
  verifyValidationFailure(windowedOperator, "retracting storage is not set for ACCUMULATING_AND_RETRACTING");
  windowedOperator.setRetractionStorage(new InMemoryWindowedStorage<Long>());
  windowedOperator.validate();
  windowedOperator.setTriggerOption(new TriggerOption().discardingFiredPanes().firingOnlyUpdatedPanes());
  verifyValidationFailure(windowedOperator, "DISCARDING is not valid for option firingOnlyUpdatedPanes");
  windowedOperator.setTriggerOption(new TriggerOption().accumulatingFiredPanes().firingOnlyUpdatedPanes());
  windowedOperator.setRetractionStorage(null);
  verifyValidationFailure(windowedOperator, "retracting storage is not set for option firingOnlyUpdatedPanes");
}
 
Example #14
Source File: MySession.java    From sakai with Educational Community License v2.0 6 votes vote down vote up
public MySession(SessionManager sessionManager, String id, ThreadLocalManager threadLocalManager,
				 IdManager idManager, SessionStore sessionStore, SessionAttributeListener sessionListener,
				 int inactiveInterval, NonPortableSession nonPortableSession, MutableLong expirationTimeSuggestion,
				 RebuildBreakdownService rebuildBreakdownService)
{
	this.sessionManager = sessionManager;
	m_id = id;
	this.threadLocalManager = threadLocalManager;
	this.idManager = idManager;
	this.sessionStore = sessionStore;
	this.sessionListener = sessionListener;
	m_inactiveInterval = inactiveInterval;
	m_nonPortalSession = nonPortableSession;
	m_created = System.currentTimeMillis();
	m_accessed = m_created;
	this.expirationTimeSuggestion = expirationTimeSuggestion;
	resetExpirationTimeSuggestion();
	// set the TERRACOTTA_CLUSTER flag
	resolveTerracottaClusterProperty();
	this.rebuildBreakdownService = rebuildBreakdownService;
}
 
Example #15
Source File: Convol.java    From ij-ridgedetection with GNU General Public License v2.0 6 votes vote down vote up
/**
 * Compute gauss mask 0.
 *
 * @param num
 *            the num
 * @param sigma
 *            the sigma
 * @return the double[]
 */
/*
 * num ist eigentlich pointer - aufrufende Funkion nimmt an, dass num geändert
 * wird. Übergebe es deswegen als MutableDouble aus CommonsLang
 */
public double[] compute_gauss_mask_0(MutableLong num, double sigma) {

	int i, n;
	double limit;
	double[] h;

	limit = LinesUtil.MASK_SIZE(LinesUtil.MAX_SIZE_MASK_0, sigma); /* Error < 0.001 on each side */
	n = (int) limit;
	h = new double[2 * n + 1];
	for (i = -n + 1; i <= n - 1; i++)
		h[n + i] = phi0(-i + 0.5, sigma) - phi0(-i - 0.5, sigma);
	h[0] = 1.0 - phi0(n - 0.5, sigma);
	h[2 * n] = phi0(-n + 0.5, sigma);
	num.setValue(n);
	return h;
}
 
Example #16
Source File: Convol.java    From ij-ridgedetection with GNU General Public License v2.0 6 votes vote down vote up
/**
 * Compute gauss mask 2.
 *
 * @param num
 *            the num
 * @param sigma
 *            the sigma
 * @return the double[]
 */
/*
 * num ist eigentlich pointer - aufrufende Funkion nimmt an, dass num geändert
 * wird. Übergebe es deswegen als MutableDouble aus CommonsLang
 */
public double[] compute_gauss_mask_2(MutableLong num, double sigma) {
	int i, n;
	double limit;
	double[] h;

	limit = LinesUtil.MASK_SIZE(LinesUtil.MAX_SIZE_MASK_2, sigma); /* Error < 0.001 on each side */
	n = (int) limit;
	h = new double[2 * n + 1];

	for (i = -n + 1; i <= n - 1; i++)
		h[n + i] = phi2(-i + 0.5, sigma) - phi2(-i - 0.5, sigma);
	h[0] = -phi2(n - 0.5, sigma);
	h[2 * n] = phi2(-n + 0.5, sigma);
	num.setValue(n);
	return h;
}
 
Example #17
Source File: SessionComponentRegressionTest.java    From sakai with Educational Community License v2.0 6 votes vote down vote up
protected Session newSessionWithBlockableMutableLong(final CountDownLatch opStarted,
		final CountDownLatch opBlocker, final CountDownLatch opCompleted) {
	// unfortunately, the Maintenance implementation compels us to
	// use MySession rather than an interface.
	String uuid = nextUuid();
	final MutableLong expirationTimeSuggestion = new MutableLong(System.currentTimeMillis()) {
		@Override
		public long longValue() {
			Callable<Long> callback = new Callable<Long>() {
				public Long call() throws Exception {
					return superLongValue();
				}
			};
			Long result = 
				execBlockableSessionOp(opStarted, opBlocker, opCompleted, callback);
			return result;
		}
		private long superLongValue() {
			return super.longValue();
		}
	};
	final MySession session = new MySession(sessionComponent,uuid,threadLocalManager,idManager,
			sessionComponent,sessionListener,sessionComponent.getInactiveInterval(),new MyNonPortableSession(),
			expirationTimeSuggestion, null);
	return session;
}
 
Example #18
Source File: SessionComponentRegressionTest.java    From sakai with Educational Community License v2.0 6 votes vote down vote up
protected Session newSessionWithBlockableGetLastAccessedTimeImpl(final CountDownLatch opStarted,
		final CountDownLatch opBlocker, final CountDownLatch opCompleted) {
	// unfortunately, the getActiveUserCount() implementation compels us to
	// use MySession rather than an interface.
	String uuid = nextUuid();
	final MySession session = new MySession(sessionComponent,uuid,threadLocalManager,idManager,
			sessionComponent,sessionListener,sessionComponent.getInactiveInterval(),new MyNonPortableSession(),
			new MutableLong(System.currentTimeMillis()), null) {
		private long superGetLastAccessedTime() {
			return super.getLastAccessedTime();
		}
		@Override
		public long getLastAccessedTime()
		{
			Callable<Long> callback = new Callable<Long>() {
				public Long call() throws Exception {
					return superGetLastAccessedTime();
				}
			};
			Long result = 
				execBlockableSessionOp(opStarted, opBlocker, opCompleted, callback);
			return result;
		}
	};
	return session;
}
 
Example #19
Source File: AbstractFSWAL.java    From hbase with Apache License 2.0 6 votes vote down vote up
protected final long stampSequenceIdAndPublishToRingBuffer(RegionInfo hri, WALKeyImpl key,
  WALEdit edits, boolean inMemstore, RingBuffer<RingBufferTruck> ringBuffer)
  throws IOException {
  if (this.closed) {
    throw new IOException(
      "Cannot append; log is closed, regionName = " + hri.getRegionNameAsString());
  }
  MutableLong txidHolder = new MutableLong();
  MultiVersionConcurrencyControl.WriteEntry we = key.getMvcc().begin(() -> {
    txidHolder.setValue(ringBuffer.next());
  });
  long txid = txidHolder.longValue();
  ServerCall<?> rpcCall = RpcServer.getCurrentCall().filter(c -> c instanceof ServerCall)
    .filter(c -> c.getCellScanner() != null).map(c -> (ServerCall) c).orElse(null);
  try (TraceScope scope = TraceUtil.createTrace(implClassName + ".append")) {
    FSWALEntry entry = new FSWALEntry(txid, key, edits, hri, inMemstore, rpcCall);
    entry.stampRegionSequenceId(we);
    ringBuffer.get(txid).load(entry);
  } finally {
    ringBuffer.publish(txid);
  }
  return txid;
}
 
Example #20
Source File: SessionComponentRegressionTest.java    From sakai with Educational Community License v2.0 6 votes vote down vote up
protected Session newSessionWithBlockableMutableLong(final CountDownLatch opStarted,
		final CountDownLatch opBlocker, final CountDownLatch opCompleted) {
	// unfortunately, the Maintenance implementation compels us to
	// use MySession rather than an interface.
	String uuid = nextUuid();
	final MutableLong expirationTimeSuggestion = new MutableLong(System.currentTimeMillis()) {
		@Override
		public long longValue() {
			Callable<Long> callback = new Callable<Long>() {
				public Long call() throws Exception {
					return superLongValue();
				}
			};
			Long result = 
				execBlockableSessionOp(opStarted, opBlocker, opCompleted, callback);
			return result;
		}
		private long superLongValue() {
			return super.longValue();
		}
	};
	final MySession session = new MySession(sessionComponent,uuid,threadLocalManager,idManager,
			sessionComponent,sessionListener,sessionComponent.getInactiveInterval(),new MyNonPortableSession(),
			expirationTimeSuggestion, null);
	return session;
}
 
Example #21
Source File: WEQueryQueueManagerTest.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Test
public void testSimpleAddAfterStarted()
{
  WindowEndQueueManager<Query, Void> wqqm = new WindowEndQueueManager<>();

  wqqm.setup(null);
  wqqm.beginWindow(0);

  Query query = new MockQuery("0");
  wqqm.enqueue(query, null, new MutableLong(1L));

  Query query1 = new MockQuery("1");
  wqqm.enqueue(query1, null, new MutableLong(1L));

  Query queryD = wqqm.dequeue().getQuery();

  Query query2 = new MockQuery("2");
  wqqm.enqueue(query2, null, new MutableLong(1L));

  Query query1D = wqqm.dequeue().getQuery();
  Query query2D = wqqm.dequeue().getQuery();
  QueryBundle<Query, Void, MutableLong> qb = wqqm.dequeue();
  Query query3D = qb == null ? null : qb.getQuery();

  wqqm.endWindow();
  wqqm.teardown();

  Assert.assertEquals("The queries must match.", query, queryD);
  Assert.assertEquals("The queries must match.", query1, query1D);
  Assert.assertEquals("The queries must match.", query2, query2D);
  Assert.assertEquals("The queries must match.", null, query3D);
}
 
Example #22
Source File: UniformDataTestsMain.java    From count-db with MIT License 5 votes vote down vote up
private void testBatchWritingAndReading(DataInterfaceFactory factory, DatabaseCachingType cachingType, int numberOfThreads, final long numberOfItems) throws FileNotFoundException, InterruptedException {
    final BaseDataInterface dataInterface = createDataInterface(cachingType, factory);
    dataInterface.dropAllData();

    MutableLong numberOfItemsWritten = new MutableLong(0);
    long startOfWrite = System.nanoTime();
    CountDownLatch countDownLatch = new CountDownLatch(numberOfThreads);
    for (int i = 0; i < numberOfThreads; i++) {
        new UniformDataTestsThread(numberOfItemsWritten, numberOfItems, dataInterface, countDownLatch, true).start();
    }
    countDownLatch.await();
    dataInterface.flush();
    long endOfWrite = System.nanoTime();
    double writesPerSecond = numberOfItemsWritten.longValue() * 1e9 / (endOfWrite - startOfWrite);

    countDownLatch = new CountDownLatch(numberOfThreads);
    long startOfRead = System.nanoTime();
    dataInterface.optimizeForReading();
    MutableLong numberOfItemsRead = new MutableLong(0);
    for (int i = 0; i < numberOfThreads; i++) {
        new UniformDataTestsThread(numberOfItemsRead, numberOfItems, dataInterface, countDownLatch, false).start();
    }
    countDownLatch.await();
    long endOfRead = System.nanoTime();
    double readsPerSecond = numberOfItemsRead.longValue() * 1e9 / (endOfRead - startOfRead);

    Log.i(factory.getClass().getSimpleName() + " threads " + numberOfThreads + " items " + numberOfItems + " write " + NumUtils.fmt(writesPerSecond) + " read " + NumUtils.fmt(readsPerSecond));
    dataInterface.close();
}
 
Example #23
Source File: WindowedOperatorTest.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
private KeyedWindowedOperatorImpl<String, Long, MutableLong, Long> createDefaultKeyedWindowedOperator(boolean forSession)
{
  KeyedWindowedOperatorImpl<String, Long, MutableLong, Long> windowedOperator = new KeyedWindowedOperatorImpl<>();
  if (useSpillable) {
    sccImpl = new SpillableComplexComponentImpl(testMeta.timeStore);
    // TODO: We don't yet support Spillable data structures for window state storage because SpillableMapImpl does not yet support iterating over all keys.
    windowStateStorage = new InMemoryWindowedStorage<>();
    if (forSession) {
      SpillableSessionWindowedStorage<String, MutableLong> sws = new SpillableSessionWindowedStorage<>();
      sws.setSpillableComplexComponent(sccImpl);
      keyedDataStorage = sws;
    } else {
      SpillableWindowedKeyedStorage<String, MutableLong> kds = new SpillableWindowedKeyedStorage<>();
      kds.setSpillableComplexComponent(sccImpl);
      keyedDataStorage = kds;
    }
    SpillableWindowedKeyedStorage<String, Long> krs = new SpillableWindowedKeyedStorage<>();
    krs.setSpillableComplexComponent(sccImpl);
    keyedRetractionStorage = krs;
    windowedOperator.addComponent("SpillableComplexComponent", sccImpl);
  } else {
    windowStateStorage = new InMemoryWindowedStorage<>();
    if (forSession) {
      keyedDataStorage = new InMemorySessionWindowedStorage<>();
    } else {
      keyedDataStorage = new InMemoryWindowedKeyedStorage<>();
    }
    keyedRetractionStorage = new InMemoryWindowedKeyedStorage<>();
  }
  windowedOperator.setDataStorage(keyedDataStorage);
  windowedOperator.setRetractionStorage(keyedRetractionStorage);
  windowedOperator.setWindowStateStorage(windowStateStorage);
  windowedOperator.setAccumulation(new SumAccumulation());
  return windowedOperator;
}
 
Example #24
Source File: WindowedOperatorTest.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Test
public void testImplicitWatermarks()
{
  WindowedOperatorImpl<Long, MutableLong, Long> windowedOperator = createDefaultWindowedOperator();
  CollectorTestSink controlSink = new CollectorTestSink();

  windowedOperator.controlOutput.setSink(controlSink);

  windowedOperator.setWindowOption(new WindowOption.TimeWindows(Duration.millis(1000)));
  windowedOperator.setAllowedLateness(Duration.millis(1000));
  windowedOperator.setImplicitWatermarkGenerator(new FixedDiffEventTimeWatermarkGen(100));

  windowedOperator.setup(testMeta.operatorContext);

  windowedOperator.beginWindow(1);
  windowedOperator.endWindow();
  Assert.assertEquals("We should get no watermark tuple", 0, controlSink.getCount(false));

  windowedOperator.beginWindow(2);
  windowedOperator.processTuple(new Tuple.TimestampedTuple<>(BASE + 100L, 2L));
  windowedOperator.endWindow();
  Assert.assertEquals("We should get one watermark tuple", 1, controlSink.getCount(false));
  Assert.assertEquals("Check Watermark value",
      ((ControlTuple.Watermark)controlSink.collectedTuples.get(0)).getTimestamp(), BASE);

  windowedOperator.beginWindow(3);
  windowedOperator.processTuple(new Tuple.TimestampedTuple<>(BASE + 900L, 4L));
  windowedOperator.endWindow();
  Assert.assertEquals("We should get two watermark tuples", 2, controlSink.getCount(false));
  Assert.assertEquals("Check Watermark value",
      ((ControlTuple.Watermark)controlSink.collectedTuples.get(1)).getTimestamp(), BASE + 800);
}
 
Example #25
Source File: WriteHeavyIncrementObserver.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> c, Get get, List<Cell> result)
    throws IOException {
  Scan scan =
      new Scan().withStartRow(get.getRow()).withStopRow(get.getRow(), true).readAllVersions();
  NavigableMap<byte[], NavigableMap<byte[], MutableLong>> sums =
      new TreeMap<>(Bytes.BYTES_COMPARATOR);
  get.getFamilyMap().forEach((cf, cqs) -> {
    NavigableMap<byte[], MutableLong> ss = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    sums.put(cf, ss);
    cqs.forEach(cq -> {
      ss.put(cq, new MutableLong(0));
      scan.addColumn(cf, cq);
    });
  });
  List<Cell> cells = new ArrayList<>();
  try (RegionScanner scanner = c.getEnvironment().getRegion().getScanner(scan)) {
    boolean moreRows;
    do {
      moreRows = scanner.next(cells);
      for (Cell cell : cells) {
        byte[] family = CellUtil.cloneFamily(cell);
        byte[] qualifier = CellUtil.cloneQualifier(cell);
        long value = Bytes.toLong(cell.getValueArray(), cell.getValueOffset());
        sums.get(family).get(qualifier).add(value);
      }
      cells.clear();
    } while (moreRows);
  }
  sums.forEach((cf, m) -> m.forEach((cq, s) -> result
      .add(createCell(get.getRow(), cf, cq, HConstants.LATEST_TIMESTAMP, s.longValue()))));
  c.bypass();
}
 
Example #26
Source File: MovingBoundaryTimeBucketAssignerTest.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Test
public void testTimeBucketKeyExpiry()
{
  final MutableLong purgeLessThanEqualTo = new MutableLong(-2);
  testMeta.timeBucketAssigner.setExpireBefore(Duration.standardSeconds(1));
  testMeta.timeBucketAssigner.setBucketSpan(Duration.standardSeconds(1));
  testMeta.timeBucketAssigner.setPurgeListener(new TimeBucketAssigner.PurgeListener()
  {
    @Override
    public void purgeTimeBucketsLessThanEqualTo(long timeBucket)
    {
      purgeLessThanEqualTo.setValue(timeBucket);
    }
  });

  long referenceTime = testMeta.timeBucketAssigner.getReferenceInstant().getMillis();
  testMeta.timeBucketAssigner.setup(testMeta.mockManagedStateContext);
  Assert.assertEquals("purgeLessThanEqualTo", -2L, purgeLessThanEqualTo.longValue());
  long time0 = Duration.standardSeconds(0).getMillis() + referenceTime;
  Assert.assertEquals("time bucket", 1, testMeta.timeBucketAssigner.getTimeBucket(time0) );
  testMeta.timeBucketAssigner.endWindow();
  Assert.assertEquals("purgeLessThanEqualTo", -1, purgeLessThanEqualTo.longValue());

  long time1 = Duration.standardSeconds(9).getMillis() + referenceTime;
  Assert.assertEquals("time bucket", 10, testMeta.timeBucketAssigner.getTimeBucket(time1) );
  testMeta.timeBucketAssigner.endWindow();
  Assert.assertEquals("purgeLessThanEqualTo", 8, purgeLessThanEqualTo.longValue());

  long time2 = Duration.standardSeconds(10).getMillis()  + referenceTime;
  Assert.assertEquals("time bucket", 11, testMeta.timeBucketAssigner.getTimeBucket(time2) );
  testMeta.timeBucketAssigner.endWindow();
  Assert.assertEquals("purgeLessThanEqualTo", 9, purgeLessThanEqualTo.longValue());

  //Check for expiry of time1 now
  Assert.assertEquals("time bucket", -1, testMeta.timeBucketAssigner.getTimeBucket(time1) );
  testMeta.timeBucketAssigner.endWindow();
  Assert.assertEquals("purgeLessThanEqualTo", 9, purgeLessThanEqualTo.longValue());

  testMeta.timeBucketAssigner.teardown();
}
 
Example #27
Source File: BigramTestsThread.java    From count-db with MIT License 5 votes vote down vote up
public BigramTestsThread(DataType dataType, MutableLong numberOfItemsWritten, long numberOfItems, DataInputStream dis, DataInterface dataInterface, CountDownLatch countDownLatch, boolean readData) {
    super("ReadTextThread", false);
    this.numberOfItemsWritten = numberOfItemsWritten;
    this.numberOfItems = numberOfItems;
    this.inputStream = dis;
    this.dataInterface = dataInterface;
    this.countDownLatch = countDownLatch;
    this.readData = readData;
    this.dataType = dataType;
}
 
Example #28
Source File: WindowedOperatorTest.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Test
public void testGlobalWindowAssignment()
{
  WindowedOperatorImpl<Long, MutableLong, Long> windowedOperator = createDefaultWindowedOperator();
  windowedOperator.setWindowOption(new WindowOption.GlobalWindow());
  windowedOperator.setup(testMeta.operatorContext);
  Tuple.WindowedTuple<Long> windowedValue = windowedOperator.getWindowedValue(new Tuple.TimestampedTuple<>(BASE + 1100L, 2L));
  Collection<? extends Window> windows = windowedValue.getWindows();
  Assert.assertEquals(1, windows.size());
  Assert.assertEquals(Window.GlobalWindow.INSTANCE, windows.iterator().next());
  windowedOperator.teardown();
}
 
Example #29
Source File: WindowedOperatorTest.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Test
public void testTimeWindowAssignment()
{
  WindowedOperatorImpl<Long, MutableLong, Long> windowedOperator = createDefaultWindowedOperator();
  windowedOperator.setWindowOption(new WindowOption.TimeWindows(Duration.millis(1000)));
  windowedOperator.setup(testMeta.operatorContext);
  Tuple.WindowedTuple<Long> windowedValue = windowedOperator.getWindowedValue(new Tuple.TimestampedTuple<>(BASE + 1100L, 2L));
  Collection<? extends Window> windows = windowedValue.getWindows();
  Assert.assertEquals(1, windows.size());
  Window window = windows.iterator().next();
  Assert.assertEquals(BASE + 1000, window.getBeginTimestamp());
  Assert.assertEquals(1000, window.getDurationMillis());
}
 
Example #30
Source File: SessionComponent.java    From sakai with Educational Community License v2.0 5 votes vote down vote up
/**
 * @inheritDoc
 */
public Session startSession(String id)
{
	if (isClosing()) {
		throw new ClosingException();
	}
	// create a non portable session object if this is a clustered environment
	NonPortableSession nPS = new MyNonPortableSession();

	// create a new MutableLong object representing the current time that both
	// the Session and SessionManager can see.
	MutableLong currentTime = currentTimeMutableLong();

	// create a new session
	Session s = new MySession(this,id,threadLocalManager(),idManager(),this,sessionListener,m_defaultInactiveInterval,nPS,currentTime,rebuildBreakdownService());

	// Place session into the main Session Storage, capture any old id
	Session old = m_sessions.put(s.getId(), s);

	// Place an entry in the expirationTimeSuggestionMap that corresponds to the entry in m_sessions
	expirationTimeSuggestionMap.put(id, currentTime);

	// check for id conflict
	if (old != null)
	{
		log.warn("startSession: duplication id: " + s.getId());
	}

	return s;
}